long_screen/engine/
scan.rs

1//! The scan pass (pass 1): per-frame motion evidence and region learning.
2//!
3//! The host decodes frames in order and, for each one, writes its native RGBA into [`ScanSession::frame_buffer`],
4//! optionally asks [`ScanSession::is_duplicate`] (so it can skip work of its own, such as a WebGPU downscale, on an
5//! exact repeat), then calls [`ScanSession::push`] with the frame's index and timing and, when it already has one, the
6//! analysis gray. The step returns the `scan/<frame>` record, the `scan-features/<frame>` row (not for a duplicate),
7//! and the per-frame diagnostics. After the last frame, [`ScanSession::finish`] builds the run's regions and the
8//! diagnostics about their shape.
9//!
10//! What stays with the host: decoding, the geometry check against the run's frame size, storing rows (in batches),
11//! error classification, progress, pause and stop, and the facts only it knows (the compute backend, how frames were
12//! converted, the frame-reference image).
13
14use crate::engine::diagnostic::{Diagnostic, Severity};
15use crate::engine::layers::{ManualRegion, RegionLearner};
16use crate::kernels::features::{extract_features, match_features, Feature, MAX_FEATURES_DEFAULT};
17use crate::kernels::layers::{Field, FieldMotion};
18use crate::kernels::motion::{self, Gray, MatchPoints, Motion, MotionField, MOTION_CELL};
19use crate::kernels::raster::{downscale_gray, downscaled_size, mean_difference};
20use crate::kernels::regions::{Kind, RegionOut};
21use std::fmt;
22use std::rc::Rc;
23
24/// Frames at least this far behind the current one are tried as a long-baseline reference when the single step was
25/// not informative: sub-analysis-pixel motion per frame never clears the evidence bar, the same motion over several
26/// frames does.
27const LONG_BASELINE_MIN_GAP: u32 = 4;
28/// A long-baseline reference older than this many frames that still yields no evidence is replaced by the current
29/// frame.
30const LONG_BASELINE_MAX_GAP: u32 = 24;
31/// Mean analysis-gray change below which two frames count as paused (no feature matching is done).
32const PAUSED_DIFFERENCE: f64 = 0.12;
33/// A frame shown longer than this (seconds) is reported as temporal undersampling.
34const UNDERSAMPLED_DURATION: f64 = 0.12;
35
36/// Which branch first laid out a motion field. The page persists the field as an object whose key order follows
37/// the branch that built it (a duplicate frame keeps its predecessor's), and the scenario fingerprint hashes key
38/// order, so the host rebuilds the object in this layout.
39#[derive(Clone, Copy, Debug, PartialEq, Eq)]
40pub enum FieldLayout {
41    /// The first frame's stand-in zero-motion field.
42    Initial,
43    /// A field estimated against an earlier frame.
44    Estimated,
45}
46
47/// The `scan/<frame>` row.
48#[derive(Clone, Debug, PartialEq)]
49pub struct ScanRecord {
50    pub index: u32,
51    pub time: f64,
52    pub duration: f64,
53    pub field: MotionField,
54    pub layout: FieldLayout,
55    /// Exact native RGBA equality with the previous frame.
56    pub duplicate: bool,
57}
58
59/// The `scan-features/<frame>` row: a frame's features as three flat arrays (analysis-pixel `x, y` pairs as i16,
60/// scores as f32, eight descriptor words each).
61#[derive(Clone, Debug, PartialEq)]
62pub struct CompactFeatures {
63    pub xy: Vec<i16>,
64    pub score: Vec<f32>,
65    pub descriptors: Vec<u32>,
66}
67
68impl CompactFeatures {
69    pub fn encode(features: &[Feature]) -> CompactFeatures {
70        CompactFeatures {
71            xy: features
72                .iter()
73                .flat_map(|f| [f.x as i16, f.y as i16])
74                .collect(),
75            score: features.iter().map(|f| f.score as f32).collect(),
76            descriptors: features.iter().flat_map(|f| f.descriptor).collect(),
77        }
78    }
79}
80
81/// What one frame produced.
82#[derive(Clone, Debug, PartialEq)]
83pub struct ScanStep {
84    pub record: ScanRecord,
85    /// `None` for a duplicate frame, which reuses its predecessor's features.
86    pub features: Option<CompactFeatures>,
87    /// This was the run's first analysed frame (the host records its own first-frame facts then).
88    pub first: bool,
89    pub diagnostics: Vec<Diagnostic>,
90}
91
92/// The index and timing of one decoded frame.
93#[derive(Clone, Copy, Debug, PartialEq)]
94pub struct FrameMeta {
95    pub index: u32,
96    pub time: f64,
97    pub duration: f64,
98}
99
100#[derive(Clone, Debug, PartialEq, Eq)]
101pub enum ScanError {
102    /// `push` without a frame written into `frame_buffer` first.
103    NoFrame,
104    /// The host's analysis gray does not have the session's analysis size.
105    GrayGeometry { expected: usize, actual: usize },
106    /// Two analysis frames of different sizes were compared.
107    FrameGeometryChanged,
108}
109
110impl fmt::Display for ScanError {
111    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
112        match self {
113            ScanError::NoFrame => write!(
114                f,
115                "CORE_BAD_ARGUMENT: no frame was written before this scan step."
116            ),
117            ScanError::GrayGeometry { expected, actual } => write!(
118                f,
119                "CORE_BAD_ARGUMENT: the analysis gray holds {actual} pixels, not {expected}."
120            ),
121            ScanError::FrameGeometryChanged => write!(f, "FRAME_GEOMETRY_CHANGED"),
122        }
123    }
124}
125
126impl std::error::Error for ScanError {}
127
128/// The run's regions and the diagnostics about their shape.
129pub struct ScanRegions {
130    pub regions: Vec<RegionOut>,
131    pub diagnostics: Vec<Diagnostic>,
132}
133
134/// One analysed frame as later frames compare against it: its native slot, analysis gray and features.
135#[derive(Clone)]
136struct Observed {
137    slot: usize,
138    gray: Rc<Vec<u8>>,
139    features: Rc<Vec<Feature>>,
140    index: u32,
141}
142
143/// The scan pass of one run.
144pub struct ScanSession {
145    width: usize,
146    height: usize,
147    factor: usize,
148    analysis_width: usize,
149    analysis_height: usize,
150    /// Native RGBA frames: at most three are alive (the previous frame, the long-baseline reference and the frame
151    /// being written), so each decoded frame is written once and never copied between them.
152    slots: Vec<Vec<u8>>,
153    staged: Option<usize>,
154    staged_duplicate: Option<bool>,
155    previous: Option<Observed>,
156    baseline: Option<Observed>,
157    last_field: Option<(MotionField, FieldLayout)>,
158    learner: RegionLearner,
159    analysed: u32,
160}
161
162impl ScanSession {
163    /// A scan over `width × height` native frames analysed at the integer `factor`.
164    pub fn new(width: usize, height: usize, factor: usize) -> ScanSession {
165        let factor = factor.max(1);
166        let (analysis_width, analysis_height) = downscaled_size(width, height, factor);
167        ScanSession {
168            width,
169            height,
170            factor,
171            analysis_width,
172            analysis_height,
173            slots: Vec::new(),
174            staged: None,
175            staged_duplicate: None,
176            previous: None,
177            baseline: None,
178            last_field: None,
179            learner: RegionLearner::new(analysis_width, analysis_height),
180            analysed: 0,
181        }
182    }
183
184    pub fn native_size(&self) -> (usize, usize) {
185        (self.width, self.height)
186    }
187
188    pub fn analysis_size(&self) -> (usize, usize) {
189        (self.analysis_width, self.analysis_height)
190    }
191
192    /// The layer-learning state, for hosts and tests that read the accumulators back.
193    pub fn learner(&self) -> &RegionLearner {
194        &self.learner
195    }
196
197    /// The buffer the next frame's native RGBA (`width × height × 4` bytes) goes into. Asking again before `push`
198    /// returns the same buffer.
199    pub fn frame_buffer(&mut self) -> &mut [u8] {
200        let slot = match self.staged {
201            Some(slot) => slot,
202            None => {
203                let held = |s: usize| {
204                    self.previous.as_ref().is_some_and(|p| p.slot == s)
205                        || self.baseline.as_ref().is_some_and(|b| b.slot == s)
206                };
207                let slot = match (0..self.slots.len()).find(|&s| !held(s)) {
208                    Some(slot) => slot,
209                    None => {
210                        self.slots.push(vec![0; self.width * self.height * 4]);
211                        self.slots.len() - 1
212                    }
213                };
214                self.staged = Some(slot);
215                self.staged_duplicate = None;
216                slot
217            }
218        };
219        &mut self.slots[slot]
220    }
221
222    /// Whether the frame written into `frame_buffer` repeats the previous frame exactly (every RGBA byte). A
223    /// duplicate needs no analysis gray.
224    pub fn is_duplicate(&mut self) -> bool {
225        let Some(slot) = self.staged else {
226            return false;
227        };
228        if let Some(known) = self.staged_duplicate {
229            return known;
230        }
231        let duplicate = self
232            .previous
233            .as_ref()
234            .is_some_and(|p| self.slots[p.slot] == self.slots[slot]);
235        self.staged_duplicate = Some(duplicate);
236        duplicate
237    }
238
239    /// Analyses the frame written into `frame_buffer`. `gray` is the host's analysis gray for it (for example from
240    /// WebGPU), which must equal the box-filtered luma at the session's factor; without one the session computes it.
241    pub fn push(&mut self, meta: FrameMeta, gray: Option<&[u8]>) -> Result<ScanStep, ScanError> {
242        let Some(slot) = self.staged else {
243            return Err(ScanError::NoFrame);
244        };
245        let duplicate = self.is_duplicate();
246        let (aw, ah) = (self.analysis_width, self.analysis_height);
247        let (g, features) = match (&self.previous, duplicate) {
248            (Some(previous), true) => (previous.gray.clone(), previous.features.clone()),
249            _ => {
250                let g = match gray {
251                    Some(g) if g.len() != aw * ah => {
252                        return Err(ScanError::GrayGeometry {
253                            expected: aw * ah,
254                            actual: g.len(),
255                        })
256                    }
257                    Some(g) => g.to_vec(),
258                    None => {
259                        let mut out = vec![0; aw * ah];
260                        downscale_gray(
261                            &self.slots[slot],
262                            self.width,
263                            self.height,
264                            self.factor,
265                            &mut out,
266                        );
267                        out
268                    }
269                };
270                let features = features_of(&g, aw, ah);
271                (Rc::new(g), Rc::new(features))
272            }
273        };
274        self.staged = None;
275        self.staged_duplicate = None;
276        self.analysed += 1;
277        let current = Observed {
278            slot,
279            gray: g,
280            features,
281            index: meta.index,
282        };
283        let (field, layout, first) = match &self.last_field {
284            Some((last, layout)) if duplicate => (
285                duplicate_field(last, current.features.len()),
286                *layout,
287                false,
288            ),
289            _ if self.previous.is_some() => {
290                (self.step_motion(&current)?, FieldLayout::Estimated, false)
291            }
292            _ => {
293                self.baseline = Some(current.clone());
294                (
295                    initial_field(aw, ah, current.features.len()),
296                    FieldLayout::Initial,
297                    true,
298                )
299            }
300        };
301        let diagnostics = frame_diagnostics(&meta, current.features.len(), &field);
302        let features = (!duplicate).then(|| CompactFeatures::encode(&current.features));
303        self.last_field = Some((field.clone(), layout));
304        self.previous = Some(current);
305        Ok(ScanStep {
306            record: ScanRecord {
307                index: meta.index,
308                time: meta.time,
309                duration: meta.duration,
310                field,
311                layout,
312                duplicate,
313            },
314            features,
315            first,
316            diagnostics,
317        })
318    }
319
320    /// Estimates motion against the previous frame and feeds an informative field to the learner; when the single
321    /// step is not informative, retries against the long-baseline reference, whose displacement accumulated over
322    /// several frames can be.
323    fn step_motion(&mut self, current: &Observed) -> Result<MotionField, ScanError> {
324        let (aw, ah) = (self.analysis_width, self.analysis_height);
325        let previous = self
326            .previous
327            .clone()
328            .expect("step_motion runs with a previous frame");
329        let field = estimate(&previous, current, aw, ah)?;
330        if informative_field(&field) {
331            self.learn(&field, &previous, current);
332            self.baseline = Some(current.clone());
333        } else if let Some(baseline) = self.baseline.clone() {
334            let gap = current.index - baseline.index;
335            if gap >= LONG_BASELINE_MIN_GAP {
336                let long = estimate(&baseline, current, aw, ah)?;
337                if informative_field(&long) {
338                    self.learn(&long, &baseline, current);
339                    self.baseline = Some(current.clone());
340                } else if gap > LONG_BASELINE_MAX_GAP {
341                    self.baseline = Some(current.clone());
342                }
343            }
344        }
345        Ok(field)
346    }
347
348    fn learn(&mut self, field: &MotionField, from: &Observed, to: &Observed) {
349        let motions: Vec<FieldMotion> = field
350            .motions
351            .iter()
352            .map(|m| FieldMotion {
353                x: m.x,
354                y: m.y,
355                support: m.support,
356                confidence: m.confidence,
357            })
358            .collect();
359        let learner_field = Field {
360            motions: &motions,
361            labels: &field.labels,
362            confidence: &field.confidence,
363            dynamic: &field.dynamic,
364            cols: field.cols,
365            rows: field.rows,
366            difference: field.difference,
367            unknown: field.unknown,
368        };
369        let native = Some((
370            self.slots[from.slot].as_slice(),
371            self.slots[to.slot].as_slice(),
372            self.width,
373            self.height,
374        ));
375        self.learner
376            .add(&learner_field, &from.gray, &to.gray, native);
377    }
378
379    /// Builds the run's regions (none when no frame was analysed) from the learned evidence, or from `manual` when
380    /// the run has hand-placed regions, and the diagnostics about the resulting layout.
381    pub fn finish(&self, manual: &[ManualRegion]) -> ScanRegions {
382        let regions = if self.analysed == 0 {
383            Vec::new()
384        } else {
385            self.learner.finish(
386                self.width as f64,
387                self.height as f64,
388                manual,
389                self.factor as u32,
390            )
391        };
392        let diagnostics = region_diagnostics(&regions, manual.len());
393        ScanRegions {
394            regions,
395            diagnostics,
396        }
397    }
398}
399
400/// Features of one analysis gray, with their scores at the f32 precision every stored or exchanged copy has.
401fn features_of(gray: &[u8], width: usize, height: usize) -> Vec<Feature> {
402    let mut features = extract_features(gray, width, height, MAX_FEATURES_DEFAULT, None);
403    for f in &mut features {
404        f.score = f.score as f32 as f64;
405    }
406    features
407}
408
409fn estimate(
410    a: &Observed,
411    b: &Observed,
412    width: usize,
413    height: usize,
414) -> Result<MotionField, ScanError> {
415    fn gray(data: &[u8], width: usize, height: usize) -> Gray<'_> {
416        Gray {
417            width,
418            height,
419            data,
420        }
421    }
422    estimate_motion(
423        gray(&a.gray, width, height),
424        gray(&b.gray, width, height),
425        Some(&a.features),
426        Some(&b.features),
427    )
428}
429
430/// Motion from analysis frame `a` to `b`, matching the given features (extracted here when absent). A paused pair
431/// (mean change below 0.12) skips feature matching altogether.
432pub fn estimate_motion(
433    a: Gray<'_>,
434    b: Gray<'_>,
435    a_features: Option<&[Feature]>,
436    b_features: Option<&[Feature]>,
437) -> Result<MotionField, ScanError> {
438    if a.width != b.width || a.height != b.height {
439        return Err(ScanError::FrameGeometryChanged);
440    }
441    if mean_difference(a.data, b.data) < PAUSED_DIFFERENCE {
442        return Ok(motion::estimate_motion(a, b, &[], 0));
443    }
444    let extracted_a;
445    let fa = match a_features {
446        Some(f) => f,
447        None => {
448            extracted_a = features_of(a.data, a.width, a.height);
449            &extracted_a
450        }
451    };
452    let extracted_b;
453    let fb = match b_features {
454        Some(f) => f,
455        None => {
456            extracted_b = features_of(b.data, b.width, b.height);
457            &extracted_b
458        }
459    };
460    let points: Vec<MatchPoints> = if fa.is_empty() || fb.is_empty() {
461        Vec::new()
462    } else {
463        match_features(fa, fb, true)
464            .iter()
465            .map(|m| {
466                let (p, q) = (&fa[m.a as usize], &fb[m.b as usize]);
467                MatchPoints {
468                    ax: p.x as f64,
469                    ay: p.y as f64,
470                    bx: q.x as f64,
471                    by: q.y as f64,
472                    unique: m.unique,
473                }
474            })
475            .collect()
476    };
477    Ok(motion::estimate_motion(a, b, &points, fb.len() as u32))
478}
479
480/// A field carries real evidence only once some model moved more than analysis jitter and is reasonably well
481/// matched. Squared length against 1, not a hypot: plain IEEE multiply and add, identical on every engine.
482pub fn informative_field(field: &MotionField) -> bool {
483    !field.unknown
484        && field.difference >= 0.2
485        && field
486            .motions
487            .iter()
488            .any(|m| m.x * m.x + m.y * m.y > 1.0 && m.confidence > 0.3)
489}
490
491fn still(support: usize) -> Motion {
492    Motion {
493        x: 0.0,
494        y: 0.0,
495        support: support as u32,
496        unique: support as u32,
497        confidence: 1.0,
498        error: 0.0,
499        ambiguous: false,
500    }
501}
502
503/// The first frame has nothing to compare against: a zero-motion field over the analysis grid stands in.
504fn initial_field(width: usize, height: usize, features: usize) -> MotionField {
505    let (cols, rows) = (width.div_ceil(MOTION_CELL), height.div_ceil(MOTION_CELL));
506    MotionField {
507        motions: vec![still(features)],
508        labels: vec![0; cols * rows],
509        confidence: vec![255; cols * rows],
510        dynamic: vec![0; cols * rows],
511        cols,
512        rows,
513        cell: MOTION_CELL,
514        difference: 0.0,
515        feature_count: features as u32,
516        unknown: false,
517        zoom: 1.0,
518    }
519}
520
521/// A duplicate frame moved nothing: the previous field's layout, confidence and feature count, no labels or
522/// dynamic cells.
523fn duplicate_field(last: &MotionField, features: usize) -> MotionField {
524    MotionField {
525        motions: vec![still(features)],
526        labels: vec![0; last.labels.len()],
527        dynamic: vec![0; last.labels.len()],
528        difference: 0.0,
529        unknown: false,
530        zoom: 1.0,
531        ..(*last).clone()
532    }
533}
534
535fn frame_diagnostics(meta: &FrameMeta, features: usize, field: &MotionField) -> Vec<Diagnostic> {
536    let mut out = Vec::new();
537    let at = |code, severity| Diagnostic::new(code, severity).at(meta.index, meta.time);
538    if features < 8 && meta.index == 0 {
539        out.push(at("LOW_TEXTURE_UNOBSERVABLE", Severity::Warning));
540    }
541    if field.unknown {
542        out.push(at("UNRESOLVED_MOTION", Severity::Warning));
543    }
544    if field
545        .motions
546        .first()
547        .is_some_and(|m| m.ambiguous && m.support >= 6)
548    {
549        out.push(at("AMBIGUOUS_PATTERN", Severity::Warning));
550    }
551    if meta.duration > UNDERSAMPLED_DURATION {
552        out.push(at("TEMPORAL_UNDERSAMPLING", Severity::Info));
553    }
554    out
555}
556
557fn region_diagnostics(regions: &[RegionOut], manual: usize) -> Vec<Diagnostic> {
558    let mut out = Vec::new();
559    if regions.iter().any(|r| r.unassigned) {
560        out.push(Diagnostic::new("MANUAL_UNASSIGNED", Severity::Warning));
561    }
562    if regions.iter().any(|r| r.kind == Kind::Ignore) {
563        out.push(Diagnostic::new(
564            "EXPLICITLY_EXCLUDED_REGION",
565            Severity::Warning,
566        ));
567    }
568    if manual > 0 {
569        out.push(Diagnostic::new("MANUAL_REGION_PRIORITY", Severity::Info));
570    }
571    let count = |kind| regions.iter().filter(|r| r.kind == kind).count() as f64;
572    let (moving, fixed) = (count(Kind::Moving), count(Kind::Fixed));
573    if manual == 0 {
574        out.push(
575            Diagnostic::new("AUTOMATIC_LAYER_MASK", Severity::Info)
576                .param("moving", moving)
577                .param("fixed", fixed),
578        );
579    }
580    if moving > 1.0 {
581        out.push(Diagnostic::new("MULTIPLE_SCROLL_LAYERS", Severity::Info));
582    }
583    out
584}
585
586#[cfg(test)]
587mod tests;
long_screen/engine/scan.rs

long_screen/engine/
scan.rs
