long_screen/engine/
compositor.rs

1//! Compositing one placed region into the tile cache: the per-tile paint, temporal-conflict resolution against the
2//! per-canvas temporal index, and the diagnostics both raise (docs/ARCHITECTURE.md §七, §十 "合成与瞬态像素").
3//!
4//! One placement is a resumable job. [`Compositor::begin`] plans it; each [`Compositor::step`] runs until it needs
5//! something the host must fetch — a tile that is not resident ([`Step::NeedTile`]: make room with
6//! [`TileCache::victim`] and [`TileCache::evict`], then [`TileCache::insert`] it) or a canvas's stored temporal records
7//! ([`Step::NeedTemporal`]: [`Compositor::load_temporal`]) — and returns [`Step::Done`] with the placement's
8//! statistics and diagnostics once it has finished. The observed frame is passed to every step, never kept.
9//!
10//! Temporal records changed since the last flush come out of [`Compositor::flush`], per canvas in the order the
11//! canvases were first loaded: the ids to delete, then the rows to write.
12
13use crate::engine::tiles::{Tile, TileCache, TileKey, QUALITY_BLOCK};
14use crate::kernels::compositor::{composite_tile, Observation, TileBuffers};
15use crate::kernels::geometry::{js_round, Rect};
16use crate::kernels::temporal::{
17    components, mask_complete, overwrite_tile, DirtyRow, OverwriteTile, TemporalIndex,
18    TemporalRecord,
19};
20
21/// Blocks within ±2^25 of the origin are addressable: a placed rect must stay inside `CANVAS_PIXEL_BOUND` native
22/// pixels on either side (src/pipeline/solve/track.ts derives its pose bound from the same figure).
23pub const CANVAS_PIXEL_BOUND: f64 = (1u64 << 25) as f64 * QUALITY_BLOCK as f64;
24
25#[derive(Clone, Copy, Debug, PartialEq, Eq)]
26pub enum TemporalPolicy {
27    /// Keep the first complete moment of a conflicting area.
28    Stable,
29    /// Replace it with every later complete moment.
30    Latest,
31}
32
33/// One placement of a region on a canvas.
34#[derive(Clone, Debug, PartialEq)]
35pub struct Placement {
36    pub canvas: String,
37    /// The region's rect on screen.
38    pub region: Rect,
39    /// The region's code in the atlas label plane.
40    pub code: u8,
41    /// Pixels the atlas assigns to the region; equal to the rect's area for a rectangular region.
42    pub owned_pixels: f64,
43    /// The region's pose on the canvas (rounded to the raster pose here).
44    pub x: f64,
45    pub y: f64,
46    pub frame: u32,
47    pub time: f64,
48    pub confidence: f64,
49    pub uncertain: bool,
50    pub occlusions: Vec<Rect>,
51}
52
53/// The observed frame, its atlas label plane and its world-consistency mask (1 = consistent; `None` treats every
54/// pixel as consistent). All three are frame-sized.
55#[derive(Clone, Copy, Debug)]
56pub struct Frame<'a> {
57    pub rgba: &'a [u8],
58    pub width: usize,
59    pub height: usize,
60    pub labels: &'a [u8],
61    pub consistent: Option<&'a [u8]>,
62}
63
64#[derive(Clone, Copy, Debug, PartialEq)]
65pub struct CompositeStats {
66    pub added: u64,
67    /// Conflicting pixels: those detected per tile plus every pixel a resolved patch rewrote.
68    pub conflicts: u64,
69    pub uncertain: u64,
70    /// Tiles this placement painted first.
71    pub tiles: u64,
72    /// The placed rect in canvas coordinates.
73    pub bounds: Rect,
74    /// Net change in provisional pixels (set minus healed); can be negative.
75    pub provisional_pixels: i64,
76}
77
78impl Default for CompositeStats {
79    fn default() -> Self {
80        Self {
81            added: 0,
82            conflicts: 0,
83            uncertain: 0,
84            tiles: 0,
85            bounds: Rect {
86                x: 0.0,
87                y: 0.0,
88                width: 0.0,
89                height: 0.0,
90            },
91            provisional_pixels: 0,
92        }
93    }
94}
95
96/// What a placement reports; the host formats the text from the catalogue.
97#[derive(Clone, Debug, PartialEq)]
98pub enum CompositorDiagnostic {
99    /// `INCOMPLETE_TEMPORAL_PATCH` (warning): a conflicting area has no complete consistent moment yet.
100    IncompletePatch {
101        canvas: String,
102        frame: u32,
103        time: f64,
104        region: Rect,
105    },
106    /// `TEMPORAL_OR_ALIGNMENT_CONFLICT` (warning), one per placement with conflicts: `region` is the largest
107    /// conflicting component, `regions` all of them, `conflicts` the pixels detected before patching.
108    Conflict {
109        canvas: String,
110        frame: u32,
111        time: f64,
112        region: Rect,
113        confidence: f64,
114        regions: Vec<Rect>,
115        conflicts: u64,
116    },
117}
118
119#[derive(Clone, Debug, PartialEq)]
120pub struct Outcome {
121    pub stats: CompositeStats,
122    /// In emission order: every incomplete patch, then the conflict summary.
123    pub diagnostics: Vec<CompositorDiagnostic>,
124}
125
126#[derive(Clone, Debug, PartialEq)]
127pub enum Step {
128    /// The tile is not resident: make room, insert it, step again.
129    NeedTile(TileKey),
130    /// The canvas's stored temporal records are needed: load them, step again.
131    NeedTemporal(String),
132    Done(Outcome),
133}
134
135#[derive(Clone, Copy, Debug, PartialEq, Eq)]
136pub enum CompositeError {
137    /// The frame does not have the atlas's size.
138    FrameSize,
139    /// The placed rect leaves the addressable canvas.
140    OutsideCanvas,
141    /// `step` without a placement in progress, or `begin` while one is.
142    NoPlacement,
143    Busy,
144    /// A tile's planes do not have the cache's tile size.
145    BadTile,
146    /// The canvas's temporal records were already disposed.
147    Disposed,
148    /// `load_temporal` for a canvas the placement did not ask for.
149    Unexpected,
150}
151
152/// Temporal state of one canvas, in first-loaded order. `stash` holds what `dispose` drained for the next flush.
153struct CanvasTemporal {
154    canvas: String,
155    index: Option<TemporalIndex>,
156    stash: Option<(Vec<String>, Vec<DirtyRow>)>,
157}
158
159/// Rows a flush writes for one canvas: deletions first, then the changed records with their blocks in persisted
160/// order.
161#[derive(Debug, PartialEq)]
162pub struct TemporalFlush {
163    pub canvas: String,
164    pub deleted: Vec<String>,
165    pub dirty: Vec<DirtyRow>,
166}
167
168#[derive(Clone, Copy, Debug, PartialEq, Eq)]
169enum Phase {
170    Tiles(usize),
171    Component(usize),
172    Patch(usize, usize),
173    Finish(usize),
174    Done,
175}
176
177struct Component {
178    bounds: Rect,
179    blocks: Vec<(i32, i32)>,
180}
181
182/// One tile of a resolved patch: its tile coordinates and the absolute blocks to write on it.
183type PatchTile = ((i32, i32), Vec<(i64, i64)>);
184
185/// One placement in progress.
186struct Job {
187    p: Placement,
188    ox: i64,
189    oy: i64,
190    world: Rect,
191    rectangular: bool,
192    order: Vec<(i32, i32)>,
193    stats: CompositeStats,
194    conflict_blocks: Vec<(i32, i32)>,
195    components: Vec<Component>,
196    /// Tile groups of the patch being written, in first-block order, and whether to report it incomplete.
197    patch: Vec<PatchTile>,
198    incomplete: Option<Rect>,
199    patched: CompositeStats,
200    diagnostics: Vec<CompositorDiagnostic>,
201    phase: Phase,
202}
203
204pub struct Compositor {
205    policy: TemporalPolicy,
206    atlas_width: usize,
207    atlas_height: usize,
208    /// Fresh temporal-record ids, shared by every canvas and never reset.
209    sequence: u64,
210    canvases: Vec<CanvasTemporal>,
211    job: Option<Job>,
212}
213
214impl Compositor {
215    pub fn new(policy: TemporalPolicy, atlas_width: usize, atlas_height: usize) -> Self {
216        Self {
217            policy,
218            atlas_width,
219            atlas_height,
220            sequence: 0,
221            canvases: Vec::new(),
222            job: None,
223        }
224    }
225
226    /// Plans one placement: its world rect and tile order (resident tiles first, so cold loads only evict the
227    /// oldest residents). Tiles stay untouched until the first [`Compositor::step`].
228    pub fn begin(
229        &mut self,
230        cache: &TileCache,
231        p: Placement,
232        frame_width: usize,
233        frame_height: usize,
234    ) -> Result<(), CompositeError> {
235        if self.job.is_some() {
236            return Err(CompositeError::Busy);
237        }
238        if frame_width != self.atlas_width || frame_height != self.atlas_height {
239            return Err(CompositeError::FrameSize);
240        }
241        if !p.x.is_finite() || !p.y.is_finite() {
242            return Err(CompositeError::OutsideCanvas);
243        }
244        let (ox, oy) = (js_round(p.x) as i64, js_round(p.y) as i64);
245        let world = Rect {
246            x: p.region.x + ox as f64,
247            y: p.region.y + oy as f64,
248            width: p.region.width,
249            height: p.region.height,
250        };
251        let bound = CANVAS_PIXEL_BOUND;
252        if !world.x.is_finite()
253            || !world.y.is_finite()
254            || world.x.abs() >= bound
255            || world.y.abs() >= bound
256            || (world.x + world.width).abs() >= bound
257            || (world.y + world.height).abs() >= bound
258        {
259            return Err(CompositeError::OutsideCanvas);
260        }
261        let r = p.region;
262        let rectangular = [r.x, r.y, r.width, r.height]
263            .iter()
264            .all(|v| v.fract() == 0.0)
265            && p.owned_pixels == r.width * r.height;
266        let size = cache.size() as f64;
267        let (x0, x1) = (
268            (world.x / size).floor() as i32,
269            ((world.x + world.width - 1.0) / size).floor() as i32,
270        );
271        let (y0, y1) = (
272            (world.y / size).floor() as i32,
273            ((world.y + world.height - 1.0) / size).floor() as i32,
274        );
275        let mut resident = Vec::new();
276        let mut cold = Vec::new();
277        for ty in y0..=y1 {
278            for tx in x0..=x1 {
279                if cache.is_resident(&TileKey::new(&p.canvas, 0, tx, ty)) {
280                    resident.push((tx, ty));
281                } else {
282                    cold.push((tx, ty));
283                }
284            }
285        }
286        resident.extend(cold);
287        self.job = Some(Job {
288            ox,
289            oy,
290            world,
291            rectangular,
292            order: resident,
293            stats: CompositeStats {
294                bounds: world,
295                ..CompositeStats::default()
296            },
297            conflict_blocks: Vec::new(),
298            components: Vec::new(),
299            patch: Vec::new(),
300            incomplete: None,
301            patched: CompositeStats::default(),
302            diagnostics: Vec::new(),
303            phase: Phase::Tiles(0),
304            p,
305        });
306        Ok(())
307    }
308
309    /// Abandons the placement in progress (the host stopped or failed between steps).
310    pub fn cancel(&mut self) {
311        self.job = None;
312    }
313
314    /// Runs the placement until it needs a tile or temporal records, or is done. `now` is the host's clock, stamped
315    /// on every tile this step writes.
316    pub fn step(
317        &mut self,
318        cache: &mut TileCache,
319        frame: &Frame,
320        now: f64,
321    ) -> Result<Step, CompositeError> {
322        let mut job = self.job.take().ok_or(CompositeError::NoPlacement)?;
323        let result = self.run(&mut job, cache, frame, now);
324        match result {
325            Ok(Step::Done(_)) | Err(_) => {}
326            Ok(_) => self.job = Some(job),
327        }
328        result
329    }
330
331    fn run(
332        &mut self,
333        job: &mut Job,
334        cache: &mut TileCache,
335        frame: &Frame,
336        now: f64,
337    ) -> Result<Step, CompositeError> {
338        if frame.width != self.atlas_width || frame.height != self.atlas_height {
339            return Err(CompositeError::FrameSize);
340        }
341        let size = cache.size() as i32;
342        let per_tile = size / QUALITY_BLOCK as i32;
343        loop {
344            match job.phase {
345                Phase::Tiles(i) if i < job.order.len() => {
346                    let (tx, ty) = job.order[i];
347                    let key = TileKey::new(&job.p.canvas, 0, tx, ty);
348                    let Some(id) = cache.touch(&key) else {
349                        return Ok(Step::NeedTile(key));
350                    };
351                    let tile = cache.get_mut(id).unwrap();
352                    let was_new = !tile.existed && tile.coverage.iter().all(|&b| b == 0);
353                    let obs = Observation {
354                        rgba: frame.rgba,
355                        width: frame.width,
356                        height: frame.height,
357                        labels: (!job.rectangular).then_some((frame.labels, job.p.code)),
358                        occlusions: &job.p.occlusions,
359                        consistent: frame.consistent,
360                        confidence: job.p.confidence,
361                        uncertain: job.p.uncertain,
362                        frame: job.p.frame,
363                    };
364                    let stats = {
365                        let mut buffers = tile_buffers(tile, cache_size(size))?;
366                        composite_tile(
367                            &mut buffers,
368                            &obs,
369                            job.world,
370                            job.ox,
371                            job.oy,
372                            tx as i64,
373                            ty as i64,
374                        )
375                    };
376                    job.stats.added += stats.added as u64;
377                    job.stats.conflicts += stats.conflicts as u64;
378                    job.stats.uncertain += stats.uncertain as u64;
379                    job.stats.provisional_pixels += stats.provisional_delta as i64;
380                    for (bx, by) in stats.conflict_blocks {
381                        job.conflict_blocks
382                            .push((tx * per_tile + bx as i32, ty * per_tile + by as i32));
383                    }
384                    if stats.changed {
385                        cache.mark_dirty(id, now);
386                        if was_new {
387                            job.stats.tiles += 1;
388                        }
389                    }
390                    job.phase = Phase::Tiles(i + 1);
391                }
392                Phase::Tiles(_) => {
393                    if job.conflict_blocks.is_empty() {
394                        job.phase = Phase::Done;
395                        continue;
396                    }
397                    // Seed order is tile-then-local-block raster order, whatever order the tiles were painted in
398                    // (resident first): component order must not depend on the cache.
399                    let mut cells = std::mem::take(&mut job.conflict_blocks);
400                    cells.sort_by_key(|&(bx, by)| {
401                        let (tx, ty) = (bx.div_euclid(per_tile), by.div_euclid(per_tile));
402                        (ty, tx, by - ty * per_tile, bx - tx * per_tile)
403                    });
404                    cells.dedup();
405                    job.components = components(&cells, QUALITY_BLOCK as i32)
406                        .into_iter()
407                        .map(|c| Component {
408                            bounds: Rect {
409                                x: c.x0 as f64,
410                                y: c.y0 as f64,
411                                width: c.width as f64,
412                                height: c.height as f64,
413                            },
414                            blocks: c.blocks,
415                        })
416                        .collect();
417                    job.phase = Phase::Component(0);
418                }
419                Phase::Component(c) if c < job.components.len() => {
420                    let canvas = &job.p.canvas;
421                    let Some(entry) = self.canvases.iter_mut().find(|e| &e.canvas == canvas) else {
422                        return Ok(Step::NeedTemporal(canvas.clone()));
423                    };
424                    let index = entry.index.as_mut().ok_or(CompositeError::Disposed)?;
425                    let comp = &job.components[c];
426                    let (count, next) = index.decide(
427                        comp.bounds,
428                        &comp.blocks,
429                        job.p.frame,
430                        job.p.time,
431                        job.world,
432                        self.policy == TemporalPolicy::Latest,
433                        self.sequence,
434                    );
435                    self.sequence = next;
436                    let write_blocks = index.pending_write_blocks().to_vec();
437                    let mask_ok = count > 0
438                        && mask_complete(
439                            &write_blocks,
440                            frame.labels,
441                            self.atlas_width as i64,
442                            self.atlas_height as i64,
443                            job.p.code,
444                            &job.p.occlusions,
445                            frame.consistent,
446                            job.ox,
447                            job.oy,
448                        );
449                    let commit = index.commit(mask_ok).unwrap();
450                    job.patch.clear();
451                    if commit.chosen {
452                        for &(bx, by) in &write_blocks {
453                            let b = QUALITY_BLOCK as i32;
454                            let t = ((bx * b).div_euclid(size), (by * b).div_euclid(size));
455                            match job.patch.iter_mut().find(|(k, _)| *k == t) {
456                                Some((_, v)) => v.push((bx as i64, by as i64)),
457                                None => job.patch.push((t, vec![(bx as i64, by as i64)])),
458                            }
459                        }
460                    }
461                    job.incomplete = commit.emit_incomplete.then_some(commit.rect);
462                    job.phase = Phase::Patch(c, 0);
463                }
464                Phase::Component(_) => {
465                    let mut regions: Vec<Rect> = job.components.iter().map(|c| c.bounds).collect();
466                    let region = regions
467                        .iter()
468                        .copied()
469                        .reduce(|a, b| {
470                            if b.width * b.height > a.width * a.height {
471                                b
472                            } else {
473                                a
474                            }
475                        })
476                        .unwrap();
477                    job.diagnostics.push(CompositorDiagnostic::Conflict {
478                        canvas: job.p.canvas.clone(),
479                        frame: job.p.frame,
480                        time: job.p.time,
481                        region,
482                        confidence: job.p.confidence,
483                        regions: std::mem::take(&mut regions),
484                        conflicts: job.stats.conflicts,
485                    });
486                    job.stats.added += job.patched.added;
487                    job.stats.tiles += job.patched.tiles;
488                    job.stats.conflicts += job.patched.conflicts;
489                    job.stats.provisional_pixels += job.patched.provisional_pixels;
490                    job.phase = Phase::Done;
491                }
492                Phase::Patch(c, g) if g < job.patch.len() => {
493                    let ((tx, ty), ref blocks) = job.patch[g];
494                    let key = TileKey::new(&job.p.canvas, 0, tx, ty);
495                    let Some(id) = cache.touch(&key) else {
496                        return Ok(Step::NeedTile(key));
497                    };
498                    let tile = cache.get_mut(id).unwrap();
499                    check_tile(tile, cache_size(size))?;
500                    let stats = overwrite_tile(
501                        &mut OverwriteTile {
502                            size: size as usize,
503                            pixels: &mut tile.pixels,
504                            coverage: &mut tile.coverage,
505                            provisional: &mut tile.provisional,
506                            quality: &mut tile.quality,
507                            conflicts: &mut tile.conflicts,
508                            owner: &mut tile.owner,
509                            frozen: &mut tile.frozen,
510                        },
511                        frame.rgba,
512                        frame.width as i64,
513                        blocks,
514                        job.ox,
515                        job.oy,
516                        tx as i64,
517                        ty as i64,
518                        job.p.frame,
519                        job.p.confidence,
520                        self.policy == TemporalPolicy::Stable,
521                    );
522                    job.patched.added += stats.added as u64;
523                    job.patched.conflicts += stats.conflict_pixels as u64;
524                    job.patched.provisional_pixels += stats.provisional_delta as i64;
525                    if stats.changed {
526                        cache.mark_dirty(id, now);
527                    }
528                    job.phase = Phase::Patch(c, g + 1);
529                }
530                Phase::Patch(c, _) => job.phase = Phase::Finish(c),
531                Phase::Finish(c) => {
532                    if let Some(region) = job.incomplete.take() {
533                        job.diagnostics.push(CompositorDiagnostic::IncompletePatch {
534                            canvas: job.p.canvas.clone(),
535                            frame: job.p.frame,
536                            time: job.p.time,
537                            region,
538                        });
539                    }
540                    job.phase = Phase::Component(c + 1);
541                }
542                Phase::Done => {
543                    return Ok(Step::Done(Outcome {
544                        stats: job.stats,
545                        diagnostics: std::mem::take(&mut job.diagnostics),
546                    }));
547                }
548            }
549        }
550    }
551
552    /// Seeds `canvas`'s temporal index from its stored records (ascending id order) the first time a placement
553    /// asks for it; afterwards the index is authoritative.
554    pub fn load_temporal(
555        &mut self,
556        canvas: &str,
557        records: Vec<TemporalRecord>,
558    ) -> Result<(), CompositeError> {
559        if self.canvases.iter().any(|e| e.canvas == canvas) {
560            return Err(if self.disposed(canvas) {
561                CompositeError::Disposed
562            } else {
563                CompositeError::Unexpected
564            });
565        }
566        let mut index = TemporalIndex::new();
567        for record in records {
568            index.load(record);
569        }
570        self.canvases.push(CanvasTemporal {
571            canvas: canvas.to_string(),
572            index: Some(index),
573            stash: None,
574        });
575        Ok(())
576    }
577
578    fn disposed(&self, canvas: &str) -> bool {
579        self.canvases
580            .iter()
581            .any(|e| e.canvas == canvas && e.index.is_none())
582    }
583
584    /// Temporal records changed since the last flush, per canvas in first-loaded order.
585    pub fn flush(&mut self) -> Vec<TemporalFlush> {
586        self.canvases
587            .iter_mut()
588            .map(|e| {
589                let (deleted, dirty) = match e.index.as_mut() {
590                    Some(index) => index.flush_take(),
591                    None => e.stash.take().unwrap_or_default(),
592                };
593                TemporalFlush {
594                    canvas: e.canvas.clone(),
595                    deleted,
596                    dirty,
597                }
598            })
599            .collect()
600    }
601
602    /// Releases every canvas's index, keeping what the next [`Compositor::flush`] still owes storage. A placement
603    /// on a disposed canvas fails with [`CompositeError::Disposed`].
604    pub fn dispose(&mut self) {
605        for e in &mut self.canvases {
606            if let Some(mut index) = e.index.take() {
607                e.stash = Some(index.flush_take());
608            }
609        }
610    }
611}
612
613/// `size` as the tile side in pixels.
614fn cache_size(size: i32) -> usize {
615    size as usize
616}
617
618fn check_tile(tile: &Tile, size: usize) -> Result<(), CompositeError> {
619    let n = size * size;
620    let blocks = (size / QUALITY_BLOCK) * (size / QUALITY_BLOCK);
621    let ok = tile.pixels.len() == n * 4
622        && tile.coverage.len() == n.div_ceil(8)
623        && tile.provisional.len() == n.div_ceil(8)
624        && [
625            tile.quality.len(),
626            tile.conflicts.len(),
627            tile.owner.len(),
628            tile.score.len(),
629            tile.frozen.len(),
630        ]
631        .iter()
632        .all(|&l| l == blocks);
633    if ok {
634        Ok(())
635    } else {
636        Err(CompositeError::BadTile)
637    }
638}
639
640fn tile_buffers(tile: &mut Tile, size: usize) -> Result<TileBuffers<'_>, CompositeError> {
641    check_tile(tile, size)?;
642    Ok(TileBuffers {
643        size,
644        pixels: &mut tile.pixels,
645        coverage: &mut tile.coverage,
646        provisional: &mut tile.provisional,
647        quality: &mut tile.quality,
648        conflicts: &mut tile.conflicts,
649        owner: &mut tile.owner,
650        score: &mut tile.score,
651        frozen: &tile.frozen,
652    })
653}
long_screen/engine/compositor.rs

long_screen/engine/
compositor.rs
