rto_graph/media.rs
1//! Generated media content — a separate artifact store, never a graph fact.
2//!
3//! An ASR transcript (Voxtral) and a VLM description (`SmolVLM`) are **generated**,
4//! not decoded. Asked to transcribe digital silence a model does not return
5//! nothing; it returns fluent, confident prose. That is not a deterministic pure
6//! function of `(path, blob id, bytes)` — change the model, the quantisation or
7//! the sampling parameters and the same blob yields different "facts" — so it is
8//! not a `derived` fact and must not be stored as one (ADR-0015, issue #300).
9//!
10//! It lives here instead: its own table, its own retrieval surface, and never in
11//! `nodes`/`edges`. Two consequences are load-bearing, and both are asserted by
12//! tests rather than assumed:
13//!
14//! - [`crate::Store::export_factset`] — and therefore the published
15//! [`crate::GraphArtifact`] — stays a pure function of the tree **across a
16//! `media build`**, because nothing in this module writes a node or an edge.
17//! - No record acquires the `authored` relevance boost that [`crate::search`]
18//! applies, because generated content is ranked in [a separate
19//! channel](crate::search_channels) by a scorer that has no provenance term at
20//! all.
21//!
22//! Nothing here adds a [`crate::Provenance`] variant.
23//!
24//! # The boundary is generation, not models
25//!
26//! | Content | Nature | Verdict |
27//! |---|---|---|
28//! | Prose, PDF text | deterministic parse | stays `derived` |
29//! | **OCR** (`ocrs-text`) | discriminative; decodes text that is *actually present*; its errors are misreadings, correctable against the image | **stays `derived`** |
30//! | **ASR transcript** (Voxtral) | generative | **lives here** |
31//! | **VLM description** (`SmolVLM`) | generative | **lives here** |
32//!
33//! OCR has ground truth in the artefact. A transcript of silence has no ground
34//! truth to be wrong *against*, and no amount of model improvement changes its
35//! kind.
36//!
37//! # Keying: source blob + producer identity
38//!
39//! A record is keyed by `(blob_id, producer)`, where the producer is the whole
40//! identity of what produced the text — model id and file digest, quantisation,
41//! mmproj digest, prompt, and sampling parameters (see [`Producer`]). So
42//! re-describing the same blob with a better model writes a **new record, not a
43//! mutation**: you can compare the two, and you can discard one producer's output
44//! wholesale when you stop trusting it.
45//!
46//! Records survive [`crate::Store::rebuild`], following the `imports` precedent —
47//! they are expensive to reproduce (a 715 MB projector load per blob, issue #301)
48//! and are not derivable from source alone.
49//!
50//! # Two outcomes, both recorded
51//!
52//! A record holds a [`MediaOutcome`], not a string. Either a model ran and
53//! produced text, or the [pre-generation gate](gate) refused the blob before any
54//! model was loaded — and **the refusal is stored too**, with the value it
55//! measured, so `media status` can distinguish *not generated* from *generated
56//! nothing*. The two cases are variants of one enum rather than a nullable text
57//! column precisely so that a skip cannot carry generated text and a generated
58//! record cannot claim a measurement.
59//!
60//! @rto:0015
61
62use rusqlite::{Connection, OptionalExtension, params};
63use serde::{Deserialize, Serialize};
64
65use crate::store::StoreError;
66
67pub mod gate;
68pub mod producers;
69
70pub use gate::{GateReason, GateThresholds, MediaSkip};
71
72/// The prefix of every rendered [`ProducerId`].
73pub const MEDIA_PRODUCER_PREFIX: &str = "media";
74
75/// Stable schema tag on [`MediaBuildReport`] and [`MediaStatus`], so a
76/// programmatic consumer can depend on the shape.
77pub const MEDIA_SCHEMA: &str = "roteiro.media/v1";
78
79/// Audio files larger than this (compressed bytes) are not transcribed — decode
80/// plus inference time scales with duration, so cap the work one clip imposes.
81///
82/// Unconditional (not behind `audio-transcribe`) because [`build_media`] applies
83/// the cap while *enumerating* candidates, which every build can do.
84pub const MAX_AUDIO_BYTES: usize = 50 * 1024 * 1024;
85
86/// Images larger than this (compressed bytes) are not described. Shared with the
87/// OCR path in [`crate::extract`], which applies the same cap.
88pub const MAX_IMAGE_BYTES: usize = 20 * 1024 * 1024;
89
90/// Longest permitted prompt, in bytes. A prompt is part of the producer identity
91/// and is stored on every row; anything longer is a configuration mistake, not a
92/// prompt.
93pub const MAX_PROMPT: usize = 4096;
94
95/// Longest permitted model id, in characters — the same bound, and the same
96/// character set, the analyzer ids in [`crate::findings`] use, because a model id
97/// is likewise a component of a stored, indexed and printed identity.
98pub const MAX_MODEL_ID: usize = 64;
99
100/// Errors raised when constructing or building generated media content.
101#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
102pub enum MediaError {
103 /// A model id was empty, over-long, or contained a character outside
104 /// lowercase `[a-z0-9._-]`.
105 #[error(
106 "invalid model id {0:?} (expected 1 to {MAX_MODEL_ID} characters of lowercase [a-z0-9._-])"
107 )]
108 InvalidModelId(String),
109 /// A producer field that must be present was empty, or a prompt was longer
110 /// than [`MAX_PROMPT`]. The message names the field.
111 #[error("invalid producer {field}: {reason}")]
112 InvalidProducer {
113 /// The offending field.
114 field: &'static str,
115 /// Why it was refused.
116 reason: String,
117 },
118 /// `media build` was asked for a modality this **binary** cannot produce.
119 /// Names the feature that would provide it, because the fix is a rebuild.
120 #[error(
121 "this build cannot generate {kind} content: rebuild with `--features {feature}` \
122 (generated media content is opt-in, so the default build has no producer)"
123 )]
124 NoProducer {
125 /// The modality asked for.
126 kind: &'static str,
127 /// The cargo feature that provides it.
128 feature: &'static str,
129 },
130 /// The feature is compiled in but the model is not on disk. Names the exact
131 /// command that installs it, rather than degrading to silence.
132 #[error("model `{model}` is not installed: run `roteiro model pull {model}`")]
133 ModelMissing {
134 /// Registry name of the missing model.
135 model: String,
136 },
137 /// The `[models]` key governing this modality names a model that cannot be
138 /// used — an unknown name, the wrong modality, or one that is not installed.
139 ///
140 /// A separate variant from [`MediaError::ModelMissing`] because the fix is
141 /// different in kind: that one is a download, this one is an edit to a config
142 /// file that is currently *appearing* to be honoured. Falling back to the
143 /// default instead would be the worst outcome available — and, given that
144 /// llama.cpp aborts rather than errors when handed a model of the wrong
145 /// architecture, not merely a cosmetic one.
146 #[cfg(feature = "models")]
147 #[error(transparent)]
148 ModelConfig(#[from] crate::model_choice::ModelChoiceError),
149 /// A stored row could not be interpreted (database corruption).
150 #[error("corrupt media record: {0}")]
151 Corrupt(String),
152}
153
154/// Which generative modality produced a record.
155///
156/// Only generative modalities appear here: OCR is discriminative and stays on the
157/// `derived` extraction path, so it has no variant and cannot acquire one by
158/// accident.
159#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
160#[serde(rename_all = "lowercase")]
161pub enum MediaKind {
162 /// Speech transcription of an audio blob.
163 Audio,
164 /// A vision-language model's description of an image blob.
165 Vision,
166}
167
168impl MediaKind {
169 /// Stable string token used in the `SQLite` store and in `--json` output.
170 #[must_use]
171 pub fn as_str(self) -> &'static str {
172 match self {
173 Self::Audio => "audio",
174 Self::Vision => "vision",
175 }
176 }
177
178 /// Parse a modality from its stable token; `None` for an unrecognised value
179 /// (a corrupt row).
180 #[must_use]
181 pub fn from_token(s: &str) -> Option<Self> {
182 match s {
183 "audio" => Some(Self::Audio),
184 "vision" => Some(Self::Vision),
185 _ => None,
186 }
187 }
188
189 /// Whether `path` names a blob this modality can read.
190 #[must_use]
191 pub fn accepts_path(self, path: &str) -> bool {
192 match self {
193 Self::Audio => is_audio(path),
194 Self::Vision => is_image(path),
195 }
196 }
197
198 /// The byte cap this modality applies to a candidate blob.
199 #[must_use]
200 pub fn max_bytes(self) -> usize {
201 match self {
202 Self::Audio => MAX_AUDIO_BYTES,
203 Self::Vision => MAX_IMAGE_BYTES,
204 }
205 }
206
207 /// The cargo feature that compiles this modality's generator in.
208 ///
209 /// Unconditional, so a build *without* the feature can still name it — which
210 /// is the point: telling an operator which flag they lack is only useful from
211 /// the binary that lacks it.
212 #[must_use]
213 pub fn feature(self) -> &'static str {
214 match self {
215 Self::Audio => "audio-transcribe",
216 Self::Vision => "image-vision",
217 }
218 }
219
220 /// Registry name of the model this modality generates with **when nothing
221 /// pins one** — the argument to `roteiro model pull` on a stock setup, and
222 /// the value [`crate::ModelTask::default_model`] reads for this modality.
223 /// Unconditional for the same reason as [`MediaKind::feature`].
224 ///
225 /// Since Stage 33 a project can pin another with `[models] audio` /
226 /// `[models] vision`, so a caller that needs the model *this repository*
227 /// actually uses must ask [`crate::resolve_model`] with [`MediaKind::task`],
228 /// not this. The two differ exactly when a pin is set, which is why this one
229 /// is documented as the default rather than as "the" model.
230 #[must_use]
231 pub const fn model(self) -> &'static str {
232 match self {
233 Self::Audio => "voxtral-mini-3b",
234 Self::Vision => "smolvlm-500m-gguf",
235 }
236 }
237
238 /// The resolver task this modality's generation is (`transcribe` /
239 /// `describe`), so a call site holding a [`MediaKind`] can ask which model
240 /// this repository pinned without restating the mapping.
241 #[cfg(feature = "models")]
242 #[must_use]
243 pub fn task(self) -> crate::model_choice::ModelTask {
244 match self {
245 Self::Audio => crate::model_choice::ModelTask::Transcribe,
246 Self::Vision => crate::model_choice::ModelTask::Describe,
247 }
248 }
249
250 /// Whether *this binary* was compiled with this modality's generator.
251 ///
252 /// Separates the two reasons a modality can be unavailable, which call for
253 /// very different instructions: a rebuild, or a download.
254 #[must_use]
255 pub fn compiled_in(self) -> bool {
256 match self {
257 Self::Audio => cfg!(feature = "audio-transcribe"),
258 Self::Vision => cfg!(feature = "image-vision"),
259 }
260 }
261}
262
263impl std::fmt::Display for MediaKind {
264 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
265 f.write_str(self.as_str())
266 }
267}
268
269/// Whether `path` is an audio file the projector's miniaudio decoder can read
270/// (WAV/MP3/FLAC — the formats llama.cpp bundles support for).
271#[must_use]
272pub fn is_audio(path: &str) -> bool {
273 matches!(
274 crate::extract::extension(path).as_deref(),
275 Some("wav" | "mp3" | "flac")
276 )
277}
278
279/// Whether `path` is an image the OCR and vision paths can read.
280#[must_use]
281pub fn is_image(path: &str) -> bool {
282 matches!(
283 crate::extract::extension(path).as_deref(),
284 Some("png" | "jpg" | "jpeg")
285 )
286}
287
288/// Whether a character may appear in a model id.
289fn is_model_id_char(c: char) -> bool {
290 c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '-')
291}
292
293/// Whether `id` is a well-formed model id: 1..=[`MAX_MODEL_ID`] characters of
294/// lowercase `[a-z0-9._-]`. A `:` is excluded because a model id is a component
295/// of a rendered [`ProducerId`].
296#[must_use]
297pub fn is_valid_model_id(id: &str) -> bool {
298 !id.is_empty() && id.len() <= MAX_MODEL_ID && id.chars().all(is_model_id_char)
299}
300
301/// Everything about *what produced* a piece of generated text — the evidence
302/// chain graph provenance was never designed to hold.
303///
304/// This whole struct is the identity a record is keyed by, via
305/// [`Producer::id`]: change the model, its digest, the quantisation, the
306/// projector, the prompt or a sampling parameter and you have a different
307/// producer, so the next `media build` writes a **new record** rather than
308/// overwriting the old one.
309///
310/// The **tool version** is deliberately *not* part of the identity — it is
311/// recorded on the row ([`MediaRecord::tool_version`]) for forensics, but folding
312/// it in would invalidate every record on every release without the output having
313/// changed.
314#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
315pub struct Producer {
316 /// Which generative modality this producer covers.
317 pub kind: MediaKind,
318 /// Registry name of the model (`voxtral-mini-3b`, `smolvlm-500m-gguf`).
319 pub model: String,
320 /// Digest of the model file as pinned in the registry — the tie between a
321 /// record and the exact weights that produced it.
322 pub model_digest: String,
323 /// Quantisation of those weights (`Q4_K_M`, `Q8_0`, …), read from the pinned
324 /// file name; `unknown` when it cannot be determined.
325 pub quantisation: String,
326 /// Digest of the multimodal projector (`mmproj.gguf`) the model was run with.
327 pub mmproj_digest: String,
328 /// The prompt the model was given.
329 pub prompt: String,
330 /// Sampling temperature.
331 pub temperature: f64,
332 /// Token budget for the generation.
333 pub max_tokens: u32,
334}
335
336impl Producer {
337 /// Validate a producer, refusing an identity that could not be stored or
338 /// rendered unambiguously.
339 ///
340 /// # Errors
341 /// Returns [`MediaError::InvalidModelId`] for a malformed model id, or
342 /// [`MediaError::InvalidProducer`] naming the field for an empty digest, an
343 /// empty or over-long prompt, or a non-finite temperature.
344 pub fn validate(&self) -> Result<(), MediaError> {
345 if !is_valid_model_id(&self.model) {
346 return Err(MediaError::InvalidModelId(self.model.clone()));
347 }
348 let non_empty = |field: &'static str, value: &str| {
349 if value.is_empty() {
350 Err(MediaError::InvalidProducer {
351 field,
352 reason: "it is empty".to_owned(),
353 })
354 } else {
355 Ok(())
356 }
357 };
358 non_empty("model_digest", &self.model_digest)?;
359 non_empty("quantisation", &self.quantisation)?;
360 non_empty("mmproj_digest", &self.mmproj_digest)?;
361 non_empty("prompt", &self.prompt)?;
362 if self.prompt.len() > MAX_PROMPT {
363 return Err(MediaError::InvalidProducer {
364 field: "prompt",
365 reason: format!(
366 "it is {} bytes, over the {MAX_PROMPT}-byte limit",
367 self.prompt.len()
368 ),
369 });
370 }
371 if !self.temperature.is_finite() {
372 return Err(MediaError::InvalidProducer {
373 field: "temperature",
374 reason: format!("{} is not a finite number", self.temperature),
375 });
376 }
377 Ok(())
378 }
379
380 /// The identity token this producer's records are keyed by:
381 /// `media:<kind>:<model>:<fingerprint>`.
382 ///
383 /// The fingerprint is a 64-bit FNV-1a fold of the canonical rendering of
384 /// *every* identity field, in a fixed order. It is a **handle, not a
385 /// digest**: it makes the identity short enough to type at
386 /// `media clear --producer <id>`, while the row itself carries all the fields
387 /// verbatim, so nothing depends on the fold being collision-free. And because
388 /// the kind and the model name are in the token literally, a collision would
389 /// additionally require the *same* model, differing only in digest, prompt or
390 /// sampling parameters.
391 ///
392 /// No hash crate is involved deliberately: this is not a security boundary,
393 /// and the workspace does not take a dependency for one.
394 #[must_use]
395 pub fn id(&self) -> ProducerId {
396 use std::fmt::Write as _;
397
398 // A length-prefixed, ordered rendering, so two producers cannot fold to
399 // the same bytes by moving a `:` from one field into the next.
400 let mut canonical = String::new();
401 for part in [
402 self.kind.as_str(),
403 self.model.as_str(),
404 self.model_digest.as_str(),
405 self.quantisation.as_str(),
406 self.mmproj_digest.as_str(),
407 self.prompt.as_str(),
408 ] {
409 // Writing to a `String` is infallible.
410 let _ = write!(canonical, "{}:{part}", part.len());
411 }
412 // `{:?}` on an f64 round-trips exactly, so two distinct temperatures can
413 // never render identically.
414 let _ = write!(canonical, "t{:?}m{}", self.temperature, self.max_tokens);
415 ProducerId(format!(
416 "{MEDIA_PRODUCER_PREFIX}:{}:{}:{:016x}",
417 self.kind.as_str(),
418 self.model,
419 fnv1a(canonical.as_bytes())
420 ))
421 }
422}
423
424/// 64-bit FNV-1a. Deterministic, dependency-free, and used only for the
425/// [`ProducerId`] handle — see [`Producer::id`] for why that is sufficient.
426fn fnv1a(bytes: &[u8]) -> u64 {
427 let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
428 for b in bytes {
429 hash ^= u64::from(*b);
430 hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
431 }
432 hash
433}
434
435/// A rendered [`Producer`] identity — the token `media clear --producer` takes
436/// and `media status` prints.
437#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
438pub struct ProducerId(String);
439
440impl ProducerId {
441 /// The token.
442 #[must_use]
443 pub fn as_str(&self) -> &str {
444 &self.0
445 }
446}
447
448impl std::fmt::Display for ProducerId {
449 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
450 f.write_str(&self.0)
451 }
452}
453
454/// What a [`MediaProducer`] returns for one blob.
455#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
456pub struct GeneratedContent {
457 /// The generated text.
458 pub text: String,
459 /// A confidence signal, when the runtime exposes one. `None` is the honest
460 /// answer for both ASR and VLM today — neither emits a calibrated score — and
461 /// this is *not* the confidence an `inferred` edge carries; it must never be
462 /// read as one.
463 #[serde(default, skip_serializing_if = "Option::is_none")]
464 pub confidence: Option<f64>,
465}
466
467/// What one producer run concluded about one blob: text, or a recorded refusal.
468///
469/// The two cases are a **sum type, not a nullable field**, because the invariant
470/// that matters is mutual exclusion: a gated skip must write no generated text
471/// anywhere, and a generated record must not claim a measurement it never made.
472/// Expressed this way, neither is representable — a `text` column and a
473/// `skip_reason` column would leave both mistakes a `NULL` away. (The store
474/// enforces the same thing again in SQL, because the table outlives this type.)
475///
476/// Serialises internally tagged, so every record's JSON says which it is:
477/// `{"outcome":"generated","text":…}` or
478/// `{"outcome":"skipped","reason":"silence","value":0.0,"threshold":0.0001}`.
479#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
480#[serde(tag = "outcome", rename_all = "lowercase")]
481pub enum MediaOutcome {
482 /// A model ran and returned this text.
483 Generated(GeneratedContent),
484 /// The pre-generation gate refused the blob; **no model was loaded**. See
485 /// [`gate`].
486 Skipped(MediaSkip),
487}
488
489impl MediaOutcome {
490 /// The generated text, or `None` for a gated skip.
491 ///
492 /// The only way to reach a record's text, so every consumer — search,
493 /// the CLI, the explorer — has to acknowledge that a record may have none.
494 #[must_use]
495 pub fn text(&self) -> Option<&str> {
496 match self {
497 Self::Generated(content) => Some(content.text.as_str()),
498 Self::Skipped(_) => None,
499 }
500 }
501
502 /// The recorded refusal, or `None` when a model actually ran.
503 #[must_use]
504 pub fn skip(&self) -> Option<MediaSkip> {
505 match self {
506 Self::Generated(_) => None,
507 Self::Skipped(skip) => Some(*skip),
508 }
509 }
510
511 /// Whether the gate refused this blob.
512 #[must_use]
513 pub fn is_skipped(&self) -> bool {
514 matches!(self, Self::Skipped(_))
515 }
516}
517
518/// One stored record: a blob, the producer that described it, and what it said —
519/// or why nothing was said.
520#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
521pub struct MediaRecord {
522 /// Git blob id of the source media.
523 pub blob_id: String,
524 /// Repository path the blob was seen at. Evidence, not identity — the same
525 /// blob at two paths is one record.
526 pub path: String,
527 /// The rendered producer identity this record is keyed by.
528 pub producer_id: ProducerId,
529 /// The full producer identity, verbatim.
530 pub producer: Producer,
531 /// Version of the tool that wrote the record. Recorded, never part of the
532 /// identity — see [`Producer`].
533 pub tool_version: String,
534 /// Which description of this blob this is, counting from 1 across **all**
535 /// producers. Lets `media status` show that a blob has been re-described
536 /// rather than merely described.
537 ///
538 /// **Strictly increasing per blob while records accumulate**, including
539 /// across a `--force` rebuild: each write takes one more than the highest
540 /// generation currently on record for that blob.
541 ///
542 /// It is derived from the blob's surviving records, so `media clear` does
543 /// reset it — completely, if every record for that blob is discarded. See
544 /// [`record`] for why that limit is accepted rather than engineered around.
545 pub generation: u32,
546 /// When the record was written, as `SQLite`'s `datetime('now')`. Written for
547 /// humans and for `media status`; no ordering or policy depends on it.
548 pub produced_at: String,
549 /// What the run concluded: generated text, or a recorded gate refusal.
550 #[serde(flatten)]
551 pub outcome: MediaOutcome,
552}
553
554/// A narrowing filter for [`crate::Store::media_records`]. All-`None` means
555/// "every record".
556#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
557pub struct MediaFilter<'a> {
558 /// Only records written by this producer id.
559 pub producer: Option<&'a str>,
560 /// Only records of this modality.
561 pub kind: Option<MediaKind>,
562 /// Only records for this source blob.
563 pub blob_id: Option<&'a str>,
564}
565
566/// The values [`crate::Store::record_media_content`] writes.
567#[derive(Debug, Clone, Copy)]
568pub struct MediaWrite<'a> {
569 /// Git blob id of the source media.
570 pub blob_id: &'a str,
571 /// Repository path the blob was seen at.
572 pub path: &'a str,
573 /// Who produced the text — or who *would* have, on a gated skip: a refusal
574 /// belongs to a producer identity too, so changing the model re-evaluates
575 /// the blob instead of inheriting the old identity's skip.
576 pub producer: &'a Producer,
577 /// Version of the tool doing the writing.
578 pub tool_version: &'a str,
579 /// What the run concluded.
580 pub outcome: &'a MediaOutcome,
581 /// Replace an existing record for this exact `(blob, producer)` instead of
582 /// leaving it alone. Only `media build --force` sets this: a *different*
583 /// producer never mutates, it writes a new record.
584 pub replace: bool,
585}
586
587/// What one producer has in the store, for `media status`.
588#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
589pub struct ProducerSummary {
590 /// The producer identity.
591 pub producer_id: ProducerId,
592 /// Its modality.
593 pub kind: MediaKind,
594 /// The model it ran.
595 pub model: String,
596 /// Its quantisation.
597 pub quantisation: String,
598 /// How many records it owns, skips included.
599 pub records: u64,
600 /// How many of those are gate refusals rather than generated text. A
601 /// producer whose records are *all* skips has been run and has said nothing,
602 /// which is a very different report from having no records at all.
603 pub skipped: u64,
604 /// The most recent `produced_at` among them.
605 pub latest: String,
606}
607
608/// One gate refusal, as `media status` lists it.
609#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
610pub struct SkipEntry {
611 /// Git blob id of the refused source media.
612 pub blob_id: String,
613 /// Repository path it was seen at.
614 pub path: String,
615 /// The modality that would have described it.
616 pub kind: MediaKind,
617 /// The producer identity the refusal was recorded under.
618 pub producer_id: ProducerId,
619 /// Why, and what was measured.
620 #[serde(flatten)]
621 pub skip: MediaSkip,
622}
623
624/// The `media status` report.
625#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
626pub struct MediaStatus {
627 /// Stable schema tag ([`MEDIA_SCHEMA`]).
628 pub schema: &'static str,
629 /// Total stored records.
630 pub records: u64,
631 /// One entry per producer, ordered by producer id.
632 pub producers: Vec<ProducerSummary>,
633 /// Media blobs in the current tree, by modality — the denominator a rebuild
634 /// would work against.
635 pub candidates: Vec<CandidateCount>,
636 /// Producers this **binary** could run right now, ordered by id. Empty in a
637 /// build with no media features, or with no model installed; that is what
638 /// makes "0 records" legible as *cannot generate* rather than *nothing to
639 /// generate*.
640 pub available_producers: Vec<ProducerSummaryAvailable>,
641 /// Every blob the [pre-generation gate](gate) refused, with the value it
642 /// measured, ordered by `(producer, blob)`.
643 ///
644 /// This is the field that keeps a skip from being an invisible hole: an
645 /// operator reading `media status` sees *"assets/silence.wav — below silence
646 /// threshold (rms=0)"*, not a blob that silently failed to appear.
647 pub skipped: Vec<SkipEntry>,
648}
649
650/// How many blobs of one modality the current tree holds, and how many of them
651/// already have a record for *some* producer.
652#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
653pub struct CandidateCount {
654 /// The modality.
655 pub kind: MediaKind,
656 /// Distinct media blobs in the tree (within the size cap).
657 pub blobs: u64,
658 /// How many of those have at least one record carrying **generated text**.
659 pub described: u64,
660 /// How many of those the [gate](gate) refused, and no producer has since
661 /// described. Counted apart from `described` deliberately: a skipped blob is
662 /// not a described one, and folding the two together would restore exactly
663 /// the ambiguity the recorded skip exists to remove.
664 pub skipped: u64,
665}
666
667/// A producer this binary could run, as reported by `media status`.
668#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
669pub struct ProducerSummaryAvailable {
670 /// The producer identity it would write under.
671 pub producer_id: ProducerId,
672 /// Its modality.
673 pub kind: MediaKind,
674 /// The model it would run.
675 pub model: String,
676 /// Whether the store already holds records for exactly this identity — so a
677 /// caller can see at a glance that a rebuild would produce *new* records
678 /// because the installed model moved.
679 pub current: bool,
680}
681
682// --- Building --------------------------------------------------------------
683
684/// One modality's generator. The seam exists so the orchestration in
685/// [`build_media`] — incrementality, idempotence, `--force`, per-blob dedup — is
686/// testable without a 3 GB model and without a GPU, which is what CI actually
687/// runs.
688pub trait MediaProducer {
689 /// The identity every record this producer writes is keyed by.
690 fn producer(&self) -> &Producer;
691
692 /// Generate content for one blob, or `None` when the model declines to
693 /// produce anything usable. `path` is passed for logging and for producers
694 /// that need the extension; the identity never includes it.
695 fn generate(&self, path: &str, bytes: &[u8]) -> Option<GeneratedContent>;
696}
697
698/// Which modalities a `media build` should run, whether to redo work, and what
699/// the [pre-generation gate](gate) refuses.
700///
701/// Not `Eq`, because the thresholds are floats. Nothing compares two option sets
702/// for equality; they are read, not matched.
703#[derive(Debug, Clone, Copy, PartialEq)]
704pub struct MediaBuildOptions {
705 /// Generate audio transcripts.
706 pub audio: bool,
707 /// Generate image descriptions.
708 pub vision: bool,
709 /// Regenerate even where a record already exists for the current producer,
710 /// replacing it in place. Without this, `build` is incremental: a second run
711 /// with the same producer does no work.
712 ///
713 /// **Also overrides the gate.** Asking explicitly for a silent clip to be
714 /// transcribed is a legitimate request — to see what the model says, or
715 /// because the operator disagrees with a threshold — and a flag named
716 /// `--force` that quietly declined would be worse than no flag at all.
717 pub force: bool,
718 /// What the gate refuses. [`GateThresholds::disabled`] turns it off.
719 pub thresholds: GateThresholds,
720}
721
722impl Default for MediaBuildOptions {
723 /// Both modalities, incremental, gate on at its conservative defaults.
724 fn default() -> Self {
725 Self {
726 audio: true,
727 vision: true,
728 force: false,
729 thresholds: GateThresholds::default(),
730 }
731 }
732}
733
734impl MediaBuildOptions {
735 /// Whether `kind` is requested.
736 #[must_use]
737 pub fn wants(self, kind: MediaKind) -> bool {
738 match kind {
739 MediaKind::Audio => self.audio,
740 MediaKind::Vision => self.vision,
741 }
742 }
743}
744
745/// What one `media build` did.
746#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
747pub struct MediaBuildReport {
748 /// Stable schema tag ([`MEDIA_SCHEMA`]).
749 #[serde(default = "media_schema")]
750 pub schema: &'static str,
751 /// Distinct `(blob, producer)` pairs considered.
752 pub candidates: usize,
753 /// Records written (new, or replaced under `--force`).
754 pub generated: usize,
755 /// Pairs skipped because a record for that exact producer already existed —
756 /// the number that makes incrementality visible.
757 pub skipped_existing: usize,
758 /// Pairs the [pre-generation gate](gate) refused **before loading a model**,
759 /// each of which wrote a skip record naming its measured value.
760 #[serde(default)]
761 pub gated: usize,
762 /// Pairs where the model was invoked and returned nothing usable.
763 ///
764 /// Distinct from `gated`: this one *did* load a model and run it. The two
765 /// counts are what make the projector saving legible in a build report.
766 pub empty: usize,
767 /// Producer ids that ran, ordered.
768 pub producers: Vec<ProducerId>,
769}
770
771/// Default for [`MediaBuildReport::schema`] when deserialising.
772fn media_schema() -> &'static str {
773 MEDIA_SCHEMA
774}
775
776/// One candidate blob: a media file in the tree, within its modality's cap.
777#[derive(Debug, Clone, PartialEq, Eq)]
778pub struct MediaBlob {
779 /// Git blob id.
780 pub blob_id: String,
781 /// Repository path.
782 pub path: String,
783 /// Which modality can read it.
784 pub kind: MediaKind,
785}
786
787/// Every media blob in `repo`'s `HEAD` tree that some modality can read and that
788/// is within that modality's byte cap, de-duplicated by `(blob id, kind)` and
789/// ordered by `(kind, blob id)` so a build is deterministic.
790///
791/// The same blob committed at two paths is **one** candidate: the record is keyed
792/// by blob, so describing it twice would be work for one row. The lexically
793/// first path is the one recorded.
794///
795/// # Errors
796/// Returns [`crate::GitError`] if the tree cannot be walked or a blob cannot be
797/// read.
798pub fn media_blobs(repo: &crate::Repo) -> Result<Vec<MediaBlob>, crate::GitError> {
799 let mut blobs = repo.walk_blobs()?;
800 // Sort by path so "the lexically first path wins" is a fact, not an accident
801 // of the walk order.
802 blobs.sort_by(|a, b| a.path.cmp(&b.path));
803 let mut seen: std::collections::BTreeSet<(MediaKind, String)> =
804 std::collections::BTreeSet::new();
805 let mut out = Vec::new();
806 for blob in blobs {
807 for kind in [MediaKind::Audio, MediaKind::Vision] {
808 if !kind.accepts_path(&blob.path) {
809 continue;
810 }
811 if seen.contains(&(kind, blob.oid.clone())) {
812 continue;
813 }
814 // The size cap is applied here, before any model is loaded: an
815 // oversized clip is refused rather than partially transcribed.
816 let bytes = repo.read_blob(&blob.oid)?;
817 if bytes.len() > kind.max_bytes() {
818 continue;
819 }
820 seen.insert((kind, blob.oid.clone()));
821 out.push(MediaBlob {
822 blob_id: blob.oid.clone(),
823 path: blob.path.clone(),
824 kind,
825 });
826 }
827 }
828 out.sort_by(|a, b| (a.kind, &a.blob_id).cmp(&(b.kind, &b.blob_id)));
829 Ok(out)
830}
831
832/// Generate content for every candidate blob that has no record for the current
833/// producer, writing one record per `(blob, producer)`.
834///
835/// **Incremental by default and idempotent**: a second run with the same
836/// producers does no work at all — every pair lands in
837/// [`MediaBuildReport::skipped_existing`] and no model is invoked. A producer
838/// whose identity changed (a new model, a different quantisation, an edited
839/// prompt) has a different [`Producer::id`], so its pairs are *not* skipped and
840/// its output is a **new record beside the old one**, never an overwrite. Only
841/// [`MediaBuildOptions::force`] replaces, and only for the identical producer.
842///
843/// Before a producer is asked for anything, the [pre-generation gate](gate)
844/// measures the blob. A refusal writes a **skip record** — the reason and the
845/// measured value — and `generate` is never called, which is what keeps a
846/// repository of silent or blank assets from loading a model at all. `--force`
847/// overrides it.
848///
849/// `read` supplies a blob's bytes — [`crate::Repo::read_blob`] in production, a
850/// closure in tests.
851///
852/// Nothing here writes a node or an edge.
853///
854/// # Errors
855/// Returns [`StoreError`] on a store failure; a producer that fails to generate
856/// contributes to [`MediaBuildReport::empty`] rather than aborting the build, so
857/// one bad blob cannot lose the whole run's work.
858pub fn build_media<F>(
859 store: &mut crate::Store,
860 blobs: &[MediaBlob],
861 producers: &[&dyn MediaProducer],
862 opts: MediaBuildOptions,
863 mut read: F,
864) -> Result<MediaBuildReport, StoreError>
865where
866 F: FnMut(&MediaBlob) -> Option<Vec<u8>>,
867{
868 let tool_version = env!("CARGO_PKG_VERSION");
869 let mut report = MediaBuildReport {
870 schema: MEDIA_SCHEMA,
871 ..MediaBuildReport::default()
872 };
873 let mut ids: Vec<ProducerId> = producers.iter().map(|p| p.producer().id()).collect();
874 ids.sort();
875 ids.dedup();
876 report.producers = ids;
877
878 for producer in producers {
879 let identity = producer.producer();
880 let id = identity.id();
881 for blob in blobs {
882 if blob.kind != identity.kind || !opts.wants(blob.kind) {
883 continue;
884 }
885 report.candidates += 1;
886 // The incrementality decision, made *before* the bytes are read and
887 // long before a model is loaded.
888 if !opts.force && store.has_media_record(&blob.blob_id, id.as_str())? {
889 report.skipped_existing += 1;
890 continue;
891 }
892 let Some(bytes) = read(blob) else {
893 report.empty += 1;
894 continue;
895 };
896 // **The pre-generation gate**, evaluated here and not inside the
897 // producer. This is the whole point of its position: `generate` is
898 // never called, and the llama.cpp engines are built lazily *inside*
899 // `generate` (see `producers`), so a repository of silent or blank
900 // assets loads no model at all — no 715 MB projector, no backend
901 // init, nothing (ADR-0015; issue #301).
902 //
903 // `--force` skips the check outright rather than recording and then
904 // overriding: an operator who asked for the model to run wants the
905 // model to run.
906 let gated = if opts.force {
907 None
908 } else {
909 gate::evaluate(blob.kind, &bytes, opts.thresholds)
910 };
911 if let Some(skip) = gated {
912 // Recorded, not silent: the blob gets a record stating why and
913 // what was measured, so `media status` can tell an operator
914 // "skipped: below silence threshold (rms=0)" instead of leaving
915 // an indistinguishable hole.
916 store.record_media_content(&MediaWrite {
917 blob_id: &blob.blob_id,
918 path: &blob.path,
919 producer: identity,
920 tool_version,
921 outcome: &MediaOutcome::Skipped(skip),
922 replace: false,
923 })?;
924 report.gated += 1;
925 continue;
926 }
927 let Some(content) = producer.generate(&blob.path, &bytes) else {
928 report.empty += 1;
929 continue;
930 };
931 if content.text.trim().is_empty() {
932 report.empty += 1;
933 continue;
934 }
935 let written = store.record_media_content(&MediaWrite {
936 blob_id: &blob.blob_id,
937 path: &blob.path,
938 producer: identity,
939 tool_version,
940 outcome: &MediaOutcome::Generated(content),
941 replace: opts.force,
942 })?;
943 if written {
944 report.generated += 1;
945 } else {
946 report.skipped_existing += 1;
947 }
948 }
949 }
950 Ok(report)
951}
952
953/// Assemble the `media status` report: what is stored, by which producer, and
954/// how it compares with the media blobs actually in the tree.
955///
956/// `blobs` is the current candidate set (see [`media_blobs`]); pass an empty
957/// slice to report on the store alone.
958///
959/// # Errors
960/// Returns [`StoreError`] on a store failure.
961pub fn status(store: &crate::Store, blobs: &[MediaBlob]) -> Result<MediaStatus, StoreError> {
962 let mut candidates = Vec::new();
963 for kind in [MediaKind::Audio, MediaKind::Vision] {
964 let described_ids = store.described_media_blobs(kind)?;
965 let gated_ids = store.gated_media_blobs(kind)?;
966 let in_tree: std::collections::BTreeSet<&str> = blobs
967 .iter()
968 .filter(|b| b.kind == kind)
969 .map(|b| b.blob_id.as_str())
970 .collect();
971 let described = in_tree
972 .iter()
973 .filter(|id| described_ids.contains(**id))
974 .count();
975 // A blob one producer refused and another described counts as described,
976 // not skipped: it has content, so the operator has what they came for.
977 let skipped = in_tree
978 .iter()
979 .filter(|id| gated_ids.contains(**id) && !described_ids.contains(**id))
980 .count();
981 candidates.push(CandidateCount {
982 kind,
983 blobs: u64::try_from(in_tree.len()).unwrap_or(u64::MAX),
984 described: u64::try_from(described).unwrap_or(u64::MAX),
985 skipped: u64::try_from(skipped).unwrap_or(u64::MAX),
986 });
987 }
988 let stored = store.media_producer_summaries()?;
989 let mut available_producers: Vec<ProducerSummaryAvailable> = producers::available()
990 .into_iter()
991 .map(|p| {
992 let producer_id = p.id();
993 ProducerSummaryAvailable {
994 current: stored.iter().any(|s| s.producer_id == producer_id),
995 producer_id,
996 kind: p.kind,
997 model: p.model,
998 }
999 })
1000 .collect();
1001 available_producers.sort_by(|a, b| a.producer_id.cmp(&b.producer_id));
1002 // Read from the records themselves rather than from a counter, so a skip
1003 // reported here is one that is actually stored.
1004 let skipped = store
1005 .media_records(&MediaFilter::default())?
1006 .into_iter()
1007 .filter_map(|record| {
1008 record.outcome.skip().map(|skip| SkipEntry {
1009 blob_id: record.blob_id,
1010 path: record.path,
1011 kind: record.producer.kind,
1012 producer_id: record.producer_id,
1013 skip,
1014 })
1015 })
1016 .collect();
1017 Ok(MediaStatus {
1018 schema: MEDIA_SCHEMA,
1019 records: store.media_content_count()?,
1020 producers: stored,
1021 candidates,
1022 available_producers,
1023 skipped,
1024 })
1025}
1026
1027// --- Persistence. Free helpers over a `Connection` (a `Transaction` derefs to
1028// one), mirroring the findings store. Every statement here touches
1029// `media_content` and nothing else: nothing in this module reads or writes
1030// `nodes` or `edges`. ---
1031
1032/// Columns of `media_content`, in the order [`record_from_row`] decodes them.
1033const RECORD_COLS: &str = "m.blob_id, m.path, m.kind, m.producer, m.model, m.model_digest, \
1034 m.quantisation, m.mmproj_digest, m.prompt, m.temperature, m.max_tokens, \
1035 m.tool_version, m.generation, m.produced_at, m.text, m.confidence, \
1036 m.skip_reason, m.skip_value, m.skip_threshold";
1037
1038/// Write one record, returning whether a row was written. See [`MediaWrite`].
1039pub(crate) fn record(conn: &Connection, write: &MediaWrite<'_>) -> Result<bool, StoreError> {
1040 let id = write.producer.id();
1041 let existing: Option<i64> = conn
1042 .query_row(
1043 "SELECT id FROM media_content WHERE blob_id = ?1 AND producer = ?2",
1044 params![write.blob_id, id.as_str()],
1045 |r| r.get(0),
1046 )
1047 .optional()?;
1048 // The generation counter is per *blob*, not per producer: it answers "has
1049 // this blob been described before, by anyone?".
1050 //
1051 // Read **before** the `--force` delete below, and from `MAX(generation)`
1052 // rather than `COUNT(*)`. Both details are load-bearing, and getting either
1053 // wrong makes the counter silently non-monotonic:
1054 //
1055 // * *Before the delete*, because a forced rebuild removes the row it is
1056 // about to replace. Counting afterwards made `--force` on a blob with one
1057 // producer write `generation = 1` for ever, so the field could not
1058 // distinguish a first description from a fifth.
1059 // * *`MAX`, not `COUNT`*, because rows are deletable — `media clear
1060 // --producer X` removes some of a blob's records — and a count would then
1061 // hand a later write a number it had already used, putting two different
1062 // descriptions at the same generation.
1063 //
1064 // The counter is therefore derived from the blob's **surviving** records. It
1065 // rises for as long as records accumulate — which is the property that was
1066 // broken — and it falls back when they are discarded, so `media clear` does
1067 // reset it, completely if every record for that blob goes.
1068 //
1069 // That is a deliberate limit, not an oversight. Making the counter survive
1070 // deletion means persisting a per-blob high-water mark that nothing ever
1071 // clears: a second table, maintained on every write, kept for blobs that have
1072 // since left the tree — real complexity, and an unbounded one, for a field
1073 // that exists so `media status` can say "described twice". Deriving it from
1074 // the rows keeps it honest about what is actually stored, which is the more
1075 // defensible thing for a display counter to be.
1076 let previous: i64 = conn.query_row(
1077 "SELECT COALESCE(MAX(generation), 0) FROM media_content WHERE blob_id = ?1",
1078 [write.blob_id],
1079 |r| r.get(0),
1080 )?;
1081
1082 match (existing, write.replace) {
1083 // Already described by this exact producer, and not forced: leave it.
1084 // This is what makes a second `media build` free.
1085 (Some(_), false) => return Ok(false),
1086 (Some(row), true) => {
1087 conn.execute("DELETE FROM media_content WHERE id = ?1", [row])?;
1088 }
1089 (None, _) => {}
1090 }
1091 let generation = u32::try_from(previous + 1).unwrap_or(u32::MAX);
1092 // A generated record carries text and no measurement; a gated skip carries a
1093 // measurement and — literally — no text. The table's `CHECK` refuses any
1094 // other combination, so this is the only shape that can be written.
1095 let (text, confidence, skip) = match write.outcome {
1096 MediaOutcome::Generated(content) => {
1097 (content.text.as_str(), content.confidence, None::<MediaSkip>)
1098 }
1099 MediaOutcome::Skipped(skip) => ("", None, Some(*skip)),
1100 };
1101 conn.execute(
1102 "INSERT INTO media_content (
1103 blob_id, path, kind, producer, model, model_digest, quantisation, mmproj_digest,
1104 prompt, temperature, max_tokens, tool_version, generation, text, confidence,
1105 skip_reason, skip_value, skip_threshold
1106 ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18)",
1107 params![
1108 write.blob_id,
1109 write.path,
1110 write.producer.kind.as_str(),
1111 id.as_str(),
1112 write.producer.model,
1113 write.producer.model_digest,
1114 write.producer.quantisation,
1115 write.producer.mmproj_digest,
1116 write.producer.prompt,
1117 write.producer.temperature,
1118 write.producer.max_tokens,
1119 write.tool_version,
1120 generation,
1121 text,
1122 confidence,
1123 // All three from the SAME `Option`, and `MediaSkip`'s fields are not
1124 // themselves optional — so the trio is all-present or all-absent by
1125 // construction, matching the table's outcome `CHECK`. Sourcing any
1126 // one of them from a different `Option` would make a row with a
1127 // measurement but no reason writable, and such a row reads back as
1128 // *generated content that happens to be empty*. The schema is the
1129 // backstop; this is the guard.
1130 skip.map(|s| s.reason.as_str()),
1131 skip.map(|s| s.value),
1132 skip.map(|s| s.threshold),
1133 ],
1134 )?;
1135 Ok(true)
1136}
1137
1138/// Whether a record exists for exactly this `(blob, producer)`.
1139pub(crate) fn exists(conn: &Connection, blob_id: &str, producer: &str) -> Result<bool, StoreError> {
1140 let n: i64 = conn.query_row(
1141 "SELECT COUNT(*) FROM media_content WHERE blob_id = ?1 AND producer = ?2",
1142 params![blob_id, producer],
1143 |r| r.get(0),
1144 )?;
1145 Ok(n > 0)
1146}
1147
1148/// Records matching `filter`, ordered by `(producer, blob_id)` so output is
1149/// deterministic.
1150pub(crate) fn records(
1151 conn: &Connection,
1152 filter: &MediaFilter<'_>,
1153) -> Result<Vec<MediaRecord>, StoreError> {
1154 let mut where_parts: Vec<&str> = Vec::new();
1155 let mut bound: Vec<String> = Vec::new();
1156 if let Some(producer) = filter.producer {
1157 where_parts.push("m.producer = ?");
1158 bound.push(producer.to_owned());
1159 }
1160 if let Some(kind) = filter.kind {
1161 where_parts.push("m.kind = ?");
1162 bound.push(kind.as_str().to_owned());
1163 }
1164 if let Some(blob) = filter.blob_id {
1165 where_parts.push("m.blob_id = ?");
1166 bound.push(blob.to_owned());
1167 }
1168 let clause = if where_parts.is_empty() {
1169 String::new()
1170 } else {
1171 format!(" WHERE {}", where_parts.join(" AND "))
1172 };
1173 let sql =
1174 format!("SELECT {RECORD_COLS} FROM media_content m{clause} ORDER BY m.producer, m.blob_id");
1175 let mut stmt = conn.prepare(&sql)?;
1176 let mut rows = stmt.query(rusqlite::params_from_iter(bound))?;
1177 let mut out = Vec::new();
1178 while let Some(row) = rows.next()? {
1179 out.push(record_from_row(row)?);
1180 }
1181 Ok(out)
1182}
1183
1184/// Delete every record, or only one producer's. Returns how many rows went.
1185pub(crate) fn delete(conn: &Connection, producer: Option<&str>) -> Result<usize, StoreError> {
1186 let removed = match producer {
1187 Some(id) => conn.execute("DELETE FROM media_content WHERE producer = ?1", [id])?,
1188 None => conn.execute("DELETE FROM media_content", [])?,
1189 };
1190 Ok(removed)
1191}
1192
1193/// Total number of stored records.
1194pub(crate) fn count(conn: &Connection) -> Result<u64, StoreError> {
1195 let n: i64 = conn.query_row("SELECT COUNT(*) FROM media_content", [], |r| r.get(0))?;
1196 Ok(u64::try_from(n).unwrap_or(0))
1197}
1198
1199/// One summary row per producer, ordered by producer id.
1200pub(crate) fn producer_summaries(conn: &Connection) -> Result<Vec<ProducerSummary>, StoreError> {
1201 let mut stmt = conn.prepare(
1202 "SELECT producer, kind, model, quantisation, COUNT(*),
1203 SUM(skip_reason IS NOT NULL), MAX(produced_at)
1204 FROM media_content GROUP BY producer, kind, model, quantisation ORDER BY producer",
1205 )?;
1206 let mut rows = stmt.query([])?;
1207 let mut out = Vec::new();
1208 while let Some(row) = rows.next()? {
1209 let kind_token: String = row.get(1)?;
1210 let kind = MediaKind::from_token(&kind_token)
1211 .ok_or_else(|| StoreError::Corrupt(format!("unknown media kind: {kind_token}")))?;
1212 let records: i64 = row.get(4)?;
1213 let skipped: i64 = row.get(5)?;
1214 out.push(ProducerSummary {
1215 producer_id: ProducerId(row.get(0)?),
1216 kind,
1217 model: row.get(2)?,
1218 quantisation: row.get(3)?,
1219 records: u64::try_from(records).unwrap_or(0),
1220 skipped: u64::try_from(skipped).unwrap_or(0),
1221 latest: row.get(6)?,
1222 });
1223 }
1224 Ok(out)
1225}
1226
1227/// The set of blob ids with at least one record carrying **generated text**, for
1228/// a modality. A gated skip is not a description, so it does not appear here.
1229pub(crate) fn described_blobs(
1230 conn: &Connection,
1231 kind: MediaKind,
1232) -> Result<std::collections::BTreeSet<String>, StoreError> {
1233 blob_ids(
1234 conn,
1235 "SELECT DISTINCT blob_id FROM media_content
1236 WHERE kind = ?1 AND skip_reason IS NULL ORDER BY blob_id",
1237 kind,
1238 )
1239}
1240
1241/// The set of blob ids the gate refused, for a modality. The complement of
1242/// [`described_blobs`] over the records that exist.
1243pub(crate) fn gated_blobs(
1244 conn: &Connection,
1245 kind: MediaKind,
1246) -> Result<std::collections::BTreeSet<String>, StoreError> {
1247 blob_ids(
1248 conn,
1249 "SELECT DISTINCT blob_id FROM media_content
1250 WHERE kind = ?1 AND skip_reason IS NOT NULL ORDER BY blob_id",
1251 kind,
1252 )
1253}
1254
1255/// Run a one-column blob-id query for one modality.
1256fn blob_ids(
1257 conn: &Connection,
1258 sql: &str,
1259 kind: MediaKind,
1260) -> Result<std::collections::BTreeSet<String>, StoreError> {
1261 let mut stmt = conn.prepare(sql)?;
1262 let mut rows = stmt.query([kind.as_str()])?;
1263 let mut out = std::collections::BTreeSet::new();
1264 while let Some(row) = rows.next()? {
1265 out.insert(row.get::<_, String>(0)?);
1266 }
1267 Ok(out)
1268}
1269
1270/// Decode a `media_content` row.
1271fn record_from_row(row: &rusqlite::Row<'_>) -> Result<MediaRecord, StoreError> {
1272 let kind_token: String = row.get(2)?;
1273 let kind = MediaKind::from_token(&kind_token)
1274 .ok_or_else(|| StoreError::Corrupt(format!("unknown media kind: {kind_token}")))?;
1275 let generation: i64 = row.get(12)?;
1276 // `skip_reason` is the discriminant; the table's `CHECK` guarantees its two
1277 // companions are present exactly when it is, so a row that disagrees is
1278 // corruption and is reported as such rather than silently read as generated.
1279 let skip_reason: Option<String> = row.get(16)?;
1280 let outcome = match skip_reason {
1281 Some(token) => {
1282 let reason = GateReason::from_token(&token).ok_or_else(|| {
1283 StoreError::Corrupt(format!("unknown media skip reason: {token}"))
1284 })?;
1285 MediaOutcome::Skipped(MediaSkip {
1286 reason,
1287 value: row.get(17)?,
1288 threshold: row.get(18)?,
1289 })
1290 }
1291 None => MediaOutcome::Generated(GeneratedContent {
1292 text: row.get(14)?,
1293 confidence: row.get(15)?,
1294 }),
1295 };
1296 Ok(MediaRecord {
1297 blob_id: row.get(0)?,
1298 path: row.get(1)?,
1299 producer_id: ProducerId(row.get(3)?),
1300 producer: Producer {
1301 kind,
1302 model: row.get(4)?,
1303 model_digest: row.get(5)?,
1304 quantisation: row.get(6)?,
1305 mmproj_digest: row.get(7)?,
1306 prompt: row.get(8)?,
1307 temperature: row.get(9)?,
1308 max_tokens: row.get(10)?,
1309 },
1310 tool_version: row.get(11)?,
1311 generation: u32::try_from(generation).unwrap_or(u32::MAX),
1312 produced_at: row.get(13)?,
1313 outcome,
1314 })
1315}
1316
1317#[cfg(test)]
1318mod tests {
1319 use super::{
1320 GeneratedContent, MAX_MODEL_ID, MAX_PROMPT, MediaError, MediaKind, Producer,
1321 is_valid_model_id,
1322 };
1323
1324 fn producer() -> Producer {
1325 Producer {
1326 kind: MediaKind::Audio,
1327 model: "voxtral-mini-3b".to_owned(),
1328 model_digest: "4705be8e".to_owned(),
1329 quantisation: "Q4_K_M".to_owned(),
1330 mmproj_digest: "4f24c4ef".to_owned(),
1331 prompt: "Transcribe this audio recording.".to_owned(),
1332 temperature: 0.0,
1333 max_tokens: 512,
1334 }
1335 }
1336
1337 #[test]
1338 fn a_producer_id_names_its_modality_and_model() {
1339 let id = producer().id();
1340 assert!(
1341 id.as_str().starts_with("media:audio:voxtral-mini-3b:"),
1342 "got {id}"
1343 );
1344 // Stable across calls — the identity is a pure function of the fields.
1345 assert_eq!(producer().id(), producer().id());
1346 }
1347
1348 /// Every identity field must move the id. This is the property the whole
1349 /// store rests on: if changing the quantisation left the id alone, a
1350 /// re-describe would silently *skip* instead of writing a new record.
1351 #[test]
1352 fn every_identity_field_changes_the_producer_id() {
1353 /// A named single-field edit to a [`Producer`].
1354 type Mutation = (&'static str, fn(&mut Producer));
1355
1356 let base = producer().id();
1357 let mutate: [Mutation; 7] = [
1358 ("kind", |p| p.kind = MediaKind::Vision),
1359 ("model", |p| p.model = "smolvlm-500m-gguf".to_owned()),
1360 ("model_digest", |p| p.model_digest = "deadbeef".to_owned()),
1361 ("quantisation", |p| p.quantisation = "Q8_0".to_owned()),
1362 ("mmproj_digest", |p| p.mmproj_digest = "cafebabe".to_owned()),
1363 ("prompt", |p| p.prompt = "Describe this.".to_owned()),
1364 ("temperature", |p| p.temperature = 0.2),
1365 ];
1366 for (field, apply) in mutate {
1367 let mut p = producer();
1368 apply(&mut p);
1369 assert_ne!(p.id(), base, "changing {field} must change the producer id");
1370 }
1371 // …including `max_tokens`, which the table above cannot express because it
1372 // is not a `String` field.
1373 let mut p = producer();
1374 p.max_tokens = 256;
1375 assert_ne!(
1376 p.id(),
1377 base,
1378 "changing max_tokens must change the producer id"
1379 );
1380 }
1381
1382 /// The canonical rendering is length-prefixed, so no field can borrow a
1383 /// character from its neighbour to impersonate a different identity.
1384 #[test]
1385 fn adjacent_fields_cannot_be_confused() {
1386 let mut a = producer();
1387 a.model_digest = "ab".to_owned();
1388 a.quantisation = "cd".to_owned();
1389 let mut b = producer();
1390 b.model_digest = "abc".to_owned();
1391 b.quantisation = "d".to_owned();
1392 assert_ne!(a.id(), b.id());
1393 }
1394
1395 #[test]
1396 fn model_ids_accept_the_registry_names_and_reject_separators() {
1397 assert!(is_valid_model_id("voxtral-mini-3b"));
1398 assert!(is_valid_model_id("smolvlm-500m-gguf"));
1399 assert!(!is_valid_model_id(""));
1400 // A `:` would make a producer id ambiguous.
1401 assert!(!is_valid_model_id("a:b"));
1402 assert!(!is_valid_model_id("Voxtral"));
1403 assert!(is_valid_model_id(&"a".repeat(MAX_MODEL_ID)));
1404 assert!(!is_valid_model_id(&"a".repeat(MAX_MODEL_ID + 1)));
1405 }
1406
1407 #[test]
1408 fn validation_names_the_field_it_refused() {
1409 assert!(producer().validate().is_ok());
1410
1411 let mut bad = producer();
1412 bad.model = "Voxtral".to_owned();
1413 assert_eq!(
1414 bad.validate(),
1415 Err(MediaError::InvalidModelId("Voxtral".to_owned()))
1416 );
1417
1418 for (field, apply) in [
1419 (
1420 "model_digest",
1421 (|p: &mut Producer| p.model_digest.clear()) as fn(&mut Producer),
1422 ),
1423 ("quantisation", |p: &mut Producer| p.quantisation.clear()),
1424 ("mmproj_digest", |p: &mut Producer| p.mmproj_digest.clear()),
1425 ("prompt", |p: &mut Producer| p.prompt.clear()),
1426 ] {
1427 let mut p = producer();
1428 apply(&mut p);
1429 let err = p.validate().expect_err("empty field must be refused");
1430 assert!(
1431 err.to_string().contains(field),
1432 "the rejection must name {field}: {err}"
1433 );
1434 }
1435
1436 let mut long = producer();
1437 long.prompt = "x".repeat(MAX_PROMPT + 1);
1438 assert!(
1439 long.validate()
1440 .expect_err("over-long prompt")
1441 .to_string()
1442 .contains("over the")
1443 );
1444
1445 let mut nan = producer();
1446 nan.temperature = f64::NAN;
1447 assert!(
1448 nan.validate()
1449 .expect_err("NaN temperature")
1450 .to_string()
1451 .contains("finite")
1452 );
1453 }
1454
1455 #[test]
1456 fn media_kind_tokens_round_trip() {
1457 for kind in [MediaKind::Audio, MediaKind::Vision] {
1458 assert_eq!(MediaKind::from_token(kind.as_str()), Some(kind));
1459 }
1460 // OCR is not a generative modality, so it has no token here — the
1461 // boundary of ADR-0015 expressed as a type.
1462 assert_eq!(MediaKind::from_token("ocr"), None);
1463 assert_eq!(MediaKind::from_token("nope"), None);
1464 }
1465
1466 #[test]
1467 fn modalities_accept_only_their_own_extensions() {
1468 assert!(MediaKind::Audio.accepts_path("a/clip.wav"));
1469 assert!(MediaKind::Audio.accepts_path("a/clip.MP3"));
1470 assert!(MediaKind::Audio.accepts_path("a/clip.flac"));
1471 assert!(!MediaKind::Audio.accepts_path("a/clip.ogg"));
1472 assert!(!MediaKind::Audio.accepts_path("a/clip.wav.bak"));
1473 assert!(MediaKind::Vision.accepts_path("a/x.png"));
1474 assert!(MediaKind::Vision.accepts_path("a/x.jpeg"));
1475 assert!(!MediaKind::Vision.accepts_path("a/x.gif"));
1476 // No modality claims a document.
1477 assert!(!MediaKind::Audio.accepts_path("a/x.md"));
1478 assert!(!MediaKind::Vision.accepts_path("a/x.md"));
1479 }
1480
1481 #[test]
1482 fn generated_content_omits_an_absent_confidence() {
1483 let bare = GeneratedContent {
1484 text: "hello".to_owned(),
1485 confidence: None,
1486 };
1487 assert_eq!(
1488 serde_json::to_string(&bare).expect("serialize"),
1489 r#"{"text":"hello"}"#
1490 );
1491 }
1492}