monocr_onnx/segmenter.rs
1//! Line Segmentation
2//!
3//! This module handles the segmentation of document images into individual text lines
4//! using horizontal projection profile analysis.
5
6use anyhow::Result;
7use image::{imageops::crop_imm, GrayImage, ImageBuffer};
8use std::ops::Range;
9use std::path::Path;
10
11/// Where a tile may be cut, as a fraction of the tile width, searching backwards
12/// from the ideal boundary. 0.12 of a 1024px window is ~123px, roughly two Mon
13/// glyphs — wide enough to find a gap, narrow enough that tiles stay near full
14/// width.
15pub const CUT_SEARCH_FRACTION: f64 = 0.12;
16
17/// A column counts as carrying ink below this grayscale value.
18pub const CUT_INK_THRESHOLD: u8 = 250;
19
20/// Where to end a tile that starts at `x0` and may not pass `ideal`.
21///
22/// Cutting at exactly `ideal` lands wherever the arithmetic falls, which is
23/// usually the middle of a glyph. Both halves keep their pixels, so a coverage
24/// check still passes, but the model reads each half as a whole character and one
25/// glyph becomes two. Measured upstream on 120 drawn lines this showed up as
26/// `ဗော်` read back as `ဗေဗိာ်`.
27///
28/// So search backwards from `ideal` for a column of white. A tile may only get
29/// narrower, never wider, or it stops fitting the model window. Returns `ideal`
30/// unchanged when there is no gap to cut at, which is the honest outcome for a
31/// continuous script: a known-bad seam beats an overflowing tile.
32///
33/// Ported from `monocr_onnx.segmenter.cut_column`; the constants and the
34/// tie-breaking are the same, so the two produce the same cuts on the same
35/// input. The shared fixture in `monocr-monorepo/shared/segmentation-fixtures`
36/// is what holds them together.
37pub fn cut_column(crop: &GrayImage, x0: u32, ideal: u32, crop_w: u32) -> u32 {
38 if ideal >= crop_w {
39 return crop_w;
40 }
41
42 // `as u32` truncates toward zero, which is what Python's `int()` does, so
43 // the two ports pick the same window on the same input.
44 let window = (((ideal - x0) as f64 * CUT_SEARCH_FRACTION) as u32).max(1);
45 // Python computes `ideal - window` in unbounded integers and can go
46 // negative; `max(x0 + 1, ...)` then discards it. Saturating at 0 reaches the
47 // same answer because x0 + 1 is always the larger value there.
48 let lo = (x0 + 1).max(ideal.saturating_sub(window));
49 if lo >= ideal {
50 return ideal;
51 }
52
53 let height = crop.height();
54 let mut rightmost_blank: Option<u32> = None;
55 let mut lightest_offset = 0u32;
56 let mut lightest_ink = u32::MAX;
57
58 for x in lo..ideal {
59 let mut ink = 0u32;
60 for y in 0..height {
61 if crop.get_pixel(x, y)[0] < CUT_INK_THRESHOLD {
62 ink += 1;
63 }
64 }
65
66 let offset = x - lo;
67 if ink == 0 {
68 rightmost_blank = Some(offset);
69 }
70 // Strict `<` keeps the leftmost of equally light columns, which is what
71 // numpy's argmin returns. The fixture pins this: on solid ink every
72 // column ties and the cut must land on `lo`.
73 if ink < lightest_ink {
74 lightest_ink = ink;
75 lightest_offset = offset;
76 }
77 }
78
79 // Prefer a truly empty column, and the rightmost one, so tiles stay as wide
80 // as the window allows. Fall back to the lightest column present.
81 lo + rightmost_blank.unwrap_or(lightest_offset)
82}
83
84/// Split one line crop into pieces that each fit the model window.
85///
86/// Returns the crop unchanged when the line already fits after being scaled to
87/// `target_h`. Otherwise cuts at whitespace columns and returns the pieces left
88/// to right, to be read separately and joined with no separator.
89///
90/// `target_h` and `target_w` come from the model contract and must both be
91/// positive. A zero `target_w` would ask for one-pixel tiles, which is garbage
92/// in, garbage out rather than an error worth a result type.
93pub fn tile_line(crop: &GrayImage, target_h: u32, target_w: u32) -> Vec<GrayImage> {
94 let (crop_w, crop_h) = crop.dimensions();
95 if crop_h == 0 || crop_w == 0 {
96 return vec![crop.clone()];
97 }
98
99 let scale = target_h as f64 / crop_h as f64;
100 // Truncation again matches Python's `int()`. It matters at the boundary: a
101 // line that scales to exactly target_w is left alone, one pixel over is
102 // tiled.
103 if (crop_w as f64 * scale) as u32 <= target_w {
104 return vec![crop.clone()];
105 }
106
107 // Must stay an f64 division in this order. Integer arithmetic on
108 // target_w * crop_h / target_h would round differently, and the fixture's
109 // 1.6 scale is a case where the difference is a whole pixel per tile.
110 let tile_w_src = ((target_w as f64 / scale) as u32).max(1);
111 let mut tiles = Vec::new();
112 let mut x0 = 0u32;
113 while x0 < crop_w {
114 let ideal = x0.saturating_add(tile_w_src).min(crop_w);
115 // Structural guard, not a tuning knob: cut_column can only return a
116 // value in (x0, ideal], but if it ever returned x0 this loop would spin
117 // forever on a page. One pixel of forced progress bounds it.
118 let x1 = cut_column(crop, x0, ideal, crop_w).max(x0 + 1);
119 tiles.push(crop_imm(crop, x0, 0, x1 - x0, crop_h).to_image());
120 x0 = x1;
121 }
122 tiles
123}
124
125/// Bounding box for a line segment
126///
127/// Represents a rectangular region in the image with pixel coordinates.
128#[derive(Debug, Clone, Copy)]
129pub struct BBox {
130 /// X coordinate of the top-left corner
131 pub x: u32,
132 /// Y coordinate of the top-left corner
133 pub y: u32,
134 /// Width of the bounding box
135 pub w: u32,
136 /// Height of the bounding box
137 pub h: u32,
138}
139
140/// Result of line segmentation
141///
142/// Contains the cropped image of a single text line and its bounding box
143/// in the original image.
144#[derive(Debug, Clone)]
145pub struct LineSegment {
146 /// Cropped grayscale image containing only this text line
147 pub img: GrayImage,
148 /// Bounding box of this line in the original image
149 pub bbox: BBox,
150}
151
152/// A page and the text mask derived from it, kept together.
153///
154/// The two are only meaningful as a pair: `binary` is a flat row-major slice
155/// that can only be indexed with the image's own width as the stride. Passing
156/// them as separate arguments made it possible to hand one function a mask and a
157/// width that did not agree; here the stride comes from `gray` so it cannot
158/// disagree.
159struct BinarizedPage<'a> {
160 /// Grayscale source that line crops are taken from
161 gray: &'a GrayImage,
162 /// Text mask, 1 = text and 0 = background, row-major over `gray`
163 binary: &'a [u8],
164}
165
166/// A printed rule -- a page border, a table rule, an underline -- spans at least
167/// this fraction of the page in one direction.
168///
169/// Deliberately coarse: no Mon, Burmese or Latin glyph holds an unbroken stroke
170/// half a page long, so the false-positive risk against text is structural
171/// rather than merely small. Lowering it toward a glyph's width is what would
172/// make rule suppression dangerous.
173const RULE_SPAN: f64 = 0.5;
174
175/// A rule must span at least this many pixels whatever [`RULE_SPAN`] works out
176/// to. On a 20px-wide crop `width * RULE_SPAN` is 10px, which a single character
177/// can reach; the floor is what keeps the span out of glyph range on small
178/// crops.
179const RULE_MIN_SPAN_PX: usize = 15;
180
181/// Suppression that would remove more than this share of the page's ink has
182/// found text, not rules, and is abandoned.
183///
184/// [`RULE_SPAN`] is a fraction of the page, so on a SHORT page a tall block of
185/// text can exceed it vertically and be deleted wholesale. Upstream that was not
186/// hypothetical: without this guard an existing test -- six 30px bands touching
187/// on a 200px page, so each glyph column is 180px of unbroken ink -- lost 98.7%
188/// of its ink and returned zero lines.
189///
190/// The threshold sits in a measured gap rather than being a round number: real
191/// framed pages classify 21.5%-58.8% of their ink as rules, every page carrying
192/// no rules 0.00%, and that false positive 98.7%. 1.36x above the worst
193/// legitimate case and 1.23x below the true positive.
194const RULE_MAX_INK_SHARE: f64 = 0.80;
195
196/// Two runs separated by at most this many rows are one text line, provided the
197/// raw profile never reaches zero inside the gap.
198///
199/// WHY THIS EXISTS, measured 2026-08-28 on a 300 DPI render of a real Mon page.
200/// Detecting boundaries on the raw profile splits a single line wherever one row
201/// dips below the gap threshold, and in Mon that happens between the upper
202/// diacritic zone and the consonant bodies. On the measured page the line spanned
203/// rows 260-324, the threshold was `0.05 * 139.9 = 7.0`, and **row 280 carried 6
204/// ink pixels** — one pixel under, one row wide. That split every line on the page
205/// into a 28px strip of glyph tops, which decoded to `0069...` because a row of
206/// circle-tops IS digits, and a decapitated 52px body, which decoded missing its
207/// asats because the asat went with the strip.
208///
209/// A 1-row gap holding ink is not a line boundary at any resolution. This is the
210/// reference's rule (`mon_OCR` `_MIN_GAP_MERGE`, `segmenter.py` step 8), ported
211/// with its value, and it is the half of the dual histogram the ports left behind:
212/// raw detection needs a merge to be safe, and every port took the first without
213/// the second.
214///
215/// The two clauses do different jobs. The size bound refuses to merge real
216/// inter-line spacing even when overlapping diacritics hold the raw profile above
217/// zero across it — upstream that unmerged case collapsed 3 PDF lines into 1. The
218/// zero test refuses to merge across a genuine clean break, which always has at
219/// least one empty row.
220const MIN_GAP_MERGE: u32 = 10;
221
222/// Fuse runs that a single sub-threshold row split apart.
223///
224/// Merges `runs[i]` into `runs[i-1]` when the gap between them is at most
225/// `max_gap` rows AND every row in the gap carries ink. See [`MIN_GAP_MERGE`] for
226/// why, and for the measurement.
227///
228/// A free function taking the profile rather than a method, so the arithmetic is
229/// testable without a page, a mask or a model.
230fn merge_runs(runs: &[(u32, u32)], hist: &[f32], max_gap: u32, min_line: u32) -> Vec<(u32, u32)> {
231 if runs.is_empty() {
232 return Vec::new();
233 }
234
235 // The page's own typical line height, from the runs as detected. Both tests
236 // below are relative to this rather than to the neighbouring run, and that is
237 // a correction rather than a preference: judging a fragment against its
238 // neighbour CASCADES. The merge mutates the accumulated run, so every merge
239 // makes it taller, and a taller run makes the next line look more like a
240 // fragment. Measured 2026-08-28 on page 47 of a 56-page book: 36 bands
241 // collapsed to 10, with single bands of 534, 632 and 732 rows holding a dozen
242 // text lines each, and the page lost 92% of its readable characters.
243 // Median over runs that could BE a line, not over every run.
244 //
245 // The merge deliberately runs before the height filter, so `runs` still holds
246 // every speckle the profile picked up. Medianing over all of them lets noise
247 // decide what a typical line is, and on a heavily speckled scan the noise
248 // wins: measured on a sibling port, 30% of collected runs were under the
249 // minimum, and on 8 of 55 pages that drove `typical` below 10 — one page
250 // reached `typical` 2 and a ceiling of 4, against a real line height of 35. The
251 // ceiling then refuses every merge, so the pass switches itself off on exactly
252 // the pages that need it most.
253 //
254 // Falling back to the unfiltered median when nothing clears the minimum is
255 // safe rather than principled: on such a page the height filter discards
256 // everything anyway, so no crop depends on the value.
257 let mut heights: Vec<u32> = runs
258 .iter()
259 .map(|&(a, b)| b - a)
260 .filter(|&h| h >= min_line)
261 .collect();
262 if heights.is_empty() {
263 heights = runs.iter().map(|&(a, b)| b - a).collect();
264 }
265 heights.sort_unstable();
266 let typical = heights[heights.len() / 2].max(1);
267
268 // No merge may produce a band more than twice a typical line. This is the
269 // backstop for the cascade above: the fragment test alone cannot bound the
270 // result, and one runaway band costs a whole page. Twice rather than tighter
271 // because a legitimate merge of two halves lands at about one typical line and
272 // must not be refused; the value is the loosest one that still stopped page
273 // 47, checked by re-measuring all 56 pages rather than by argument.
274 let ceiling = typical * 2;
275
276 let mut merged: Vec<(u32, u32)> = Vec::with_capacity(runs.len());
277 for &(r0, r1) in runs {
278 if let Some(last) = merged.last_mut() {
279 let gap_start = last.1;
280 let gap_size = r0.saturating_sub(gap_start);
281 // An empty gap cannot occur from the run collector, but a caller can
282 // hand us touching runs; treat those as already one line.
283 let gap_has_ink =
284 (gap_start..r0).all(|y| hist.get(y as usize).is_some_and(|&v| v > 0.0));
285
286 // A run at most half a typical line is a fragment of a line, not a
287 // line. This is the clause that crosses a gap of genuinely ZERO ink,
288 // which `gap_has_ink` refuses and which a floating Mon diacritic
289 // produces: measured, runs `341-360` and `362-404` are the upper marks
290 // and the body of one line separated by two empty rows. Two REAL lines
291 // two rows apart are each a full line by this test, so they stay apart.
292 // A fragment attaches to a LINE, never to another fragment. Without the
293 // second half of this, a run of speckle merges with itself: measured on
294 // a 12-speck fixture, twelve 2-row specks fused into one 46-row band,
295 // which then CLEARS the height filter and is sent to the recogniser as
296 // a line. Two pieces that are both too short to be a line do not become
297 // one by being adjacent.
298 let (ha, hb) = (last.1 - last.0, r1 - r0);
299 let fragment = 2 * ha.min(hb) <= typical && ha.max(hb) >= min_line;
300
301 if gap_size <= max_gap && (gap_has_ink || fragment) && r1 - last.0 <= ceiling {
302 last.1 = r1;
303 continue;
304 }
305 }
306 merged.push((r0, r1));
307 }
308 merged
309}
310
311/// Zero out printed rules in `mask` (1 = ink), in place, and report whether
312/// anything was removed.
313///
314/// A printed page border adds a constant ink floor to every row it spans, and
315/// once that floor clears the gap threshold no in-frame row reads as a gap: the
316/// page comes back as one band and is squeezed into the model window. Nothing
317/// downstream can recover from that, because the line was never found.
318///
319/// MEASURED WITH THIS PARAMETER SET (global threshold 128, no smear, smoothing
320/// 3, ratio 0.05 of the mean) over twelve real MNEC page-ones: nine collapse to
321/// three bands or fewer, and the twelve together go from 118 bands to 215. Pages
322/// carrying no rules come back byte-identical.
323///
324/// A run-length scan rather than a generic erode-then-dilate: opening with a 1xL
325/// line kernel keeps exactly those ink runs at least L long, which one sweep per
326/// axis computes directly. That is the form `js/src/segmenter.js` and
327/// `go/pkg/segmenter/segmenter.go` use; the reference
328/// (`mon_OCR/src/monocr/segmenter.py` `_suppress_page_rules`) reaches the same
329/// answer with `cv2.morphologyEx`, and the shared fixture
330/// `monocr-monorepo/shared/segmentation-fixtures/rule-cases.json` is what holds
331/// the four together.
332///
333/// There is deliberately NO thickness test. "A rule is long AND thin" was
334/// written, measured and deleted upstream: across twelve real pages the rule
335/// pixels found with a thickness limit and with none were identical to the
336/// pixel.
337fn suppress_page_rules(mask: &mut [u8], width: u32, height: u32) -> bool {
338 let (w, h) = (width as usize, height as usize);
339 // A mask shorter than its own stated dimensions cannot be indexed safely,
340 // and guessing at the real shape would corrupt a page rather than skip it.
341 if w == 0 || h == 0 || mask.len() < w * h {
342 return false;
343 }
344
345 // `as usize` truncates toward zero, matching Python's `int()` and Go's
346 // `int()`, so all four ports pick the same span on an odd page width. The
347 // fixture's "odd width, run at the truncated span" case is what pins it.
348 let min_h = ((width as f64 * RULE_SPAN) as usize).max(RULE_MIN_SPAN_PX);
349 let min_v = ((height as f64 * RULE_SPAN) as usize).max(RULE_MIN_SPAN_PX);
350
351 // Rules are collected into a separate plane and only subtracted at the end.
352 // Clearing them as they are found would let the horizontal sweep break a
353 // vertical rule before the vertical sweep ever sees it, which is
354 // order-dependent and silently loses one axis.
355 let mut rules = vec![0u8; w * h];
356
357 for y in 0..h {
358 let row = y * w;
359 let mut start = 0usize;
360 // The extra step past the end closes a run that reaches the edge; a
361 // border is exactly that run, so stopping at `w` would miss every
362 // full-width rule.
363 for x in 0..=w {
364 if x < w && mask[row + x] != 0 {
365 continue;
366 }
367 if x - start >= min_h {
368 rules[row + start..row + x].fill(1);
369 }
370 start = x + 1;
371 }
372 }
373 for x in 0..w {
374 let mut start = 0usize;
375 for y in 0..=h {
376 if y < h && mask[y * w + x] != 0 {
377 continue;
378 }
379 if y - start >= min_v {
380 for i in start..y {
381 rules[i * w + x] = 1;
382 }
383 }
384 start = y + 1;
385 }
386 }
387
388 let ink = mask[..w * h].iter().filter(|&&v| v != 0).count();
389 let rule_ink = rules.iter().filter(|&&v| v != 0).count();
390 if ink == 0 || rule_ink == 0 || rule_ink as f64 > ink as f64 * RULE_MAX_INK_SHARE {
391 // Found the text. Leaving the page alone is strictly better than
392 // emptying it, and the caller is no worse off than before this step
393 // existed.
394 return false;
395 }
396
397 for (cell, &rule) in mask.iter_mut().zip(rules.iter()) {
398 if rule != 0 {
399 *cell = 0;
400 }
401 }
402 true
403}
404
405/// Line segmenter using horizontal projection profile
406///
407/// This segmenter detects text lines in a document image by analyzing the
408/// horizontal projection profile - the sum of dark pixels in each row.
409///
410/// # Algorithm
411///
412/// 1. Convert image to grayscale and binarize (threshold at 128)
413/// 2. Suppress printed rules — page borders, table rules, underlines — so their
414/// ink floor cannot hide every gap; see `suppress_page_rules`
415/// 3. Compute horizontal projection profile (sum of dark pixels per row)
416/// 4. Apply smoothing to reduce noise
417/// 5. Find gaps between text regions (where projection is near zero)
418/// 6. Extract each text region as a separate line
419///
420/// # Parameters
421///
422/// - `min_line_height`: Minimum height to consider as a valid text line
423/// - `smooth_window`: Window size for smoothing the projection profile
424/// - `density_threshold_ratio`: Fraction of mean row density that still counts
425/// as a gap
426pub struct LineSegmenter {
427 /// Minimum height for a valid text line (in pixels)
428 min_line_height: u32,
429 /// Window size for histogram smoothing
430 smooth_window: u32,
431 /// Fraction of the mean non-empty row density below which a row counts as a
432 /// gap between lines
433 density_threshold_ratio: f32,
434}
435
436/// The gap threshold this segmenter has always used, kept as the default so
437/// existing callers segment identically.
438///
439/// Every port of this pipeline picked a different number (canonical mon_OCR
440/// 0.12, the Python binding 0.02 of max, web and Android 0.03, iOS 0.03), which
441/// is the sign that it belongs to the input class rather than to the algorithm.
442pub const DEFAULT_DENSITY_THRESHOLD_RATIO: f32 = 0.05;
443
444impl LineSegmenter {
445 /// Create a new line segmenter with specified parameters
446 ///
447 /// # Arguments
448 ///
449 /// * `min_line_height` - Minimum height in pixels to consider as a valid text line
450 /// * `smooth_window` - Window size for smoothing the projection profile (1 = no smoothing)
451 ///
452 /// # Returns
453 ///
454 /// A new `LineSegmenter` instance
455 ///
456 /// # Example
457 ///
458 /// ```ignore
459 /// use monocr_onnx::segmenter::LineSegmenter;
460 ///
461 /// // Create segmenter with default parameters
462 /// let segmenter = LineSegmenter::new(10, 3);
463 /// ```
464 pub fn new(min_line_height: u32, smooth_window: u32) -> Self {
465 Self::with_density_ratio(
466 min_line_height,
467 smooth_window,
468 DEFAULT_DENSITY_THRESHOLD_RATIO,
469 )
470 }
471
472 /// Create a segmenter with an explicit gap threshold ratio.
473 ///
474 /// See [`crate::MonOcrBuilder::density_threshold_ratio`] for what the ratio
475 /// does and why it is worth setting per input class. The caller is
476 /// responsible for passing a finite, positive ratio; the builder validates
477 /// it.
478 pub fn with_density_ratio(
479 min_line_height: u32,
480 smooth_window: u32,
481 density_threshold_ratio: f32,
482 ) -> Self {
483 Self {
484 min_line_height,
485 smooth_window,
486 density_threshold_ratio,
487 }
488 }
489
490 /// Segment an image into text lines
491 ///
492 /// This is the main method that performs line segmentation on a document image.
493 /// It uses horizontal projection profile analysis to detect text lines.
494 ///
495 /// # Arguments
496 ///
497 /// * `image_path` - Path to the image file
498 ///
499 /// # Returns
500 ///
501 /// * `Ok(Vec<LineSegment>)` - Vector of segmented lines with images and bounding boxes
502 /// * `Err(anyhow::Error)` - If the image cannot be opened or processed
503 ///
504 /// # Algorithm Details
505 ///
506 /// 1. **Binarization**: Convert to grayscale and threshold at 128 (pixels < 128 are text)
507 /// 2. **Rule suppression**: Remove printed rules, so a page border cannot
508 /// fuse the whole page into one band (`suppress_page_rules`)
509 /// 3. **Projection**: Compute horizontal projection profile (sum of text pixels per row)
510 /// 4. **Smoothing**: Apply moving average filter if smooth_window > 1
511 /// 5. **Gap Detection**: Find gaps where the RAW projection is below
512 /// `density_threshold_ratio` of the SMOOTHED profile's mean non-empty row
513 /// density (default 5%). The two profiles are deliberately different: the
514 /// smoothed mean is the steadier calibration, and the raw profile is the
515 /// only one that still reaches zero between tightly set lines
516 /// 6. **Line Extraction**: Extract each region between gaps as a separate line
517 /// 7. **Padding**: Add 4-pixel padding around each line for edge character capture
518 ///
519 /// # Polarity
520 ///
521 /// The threshold treats dark as ink, so a light-on-dark page must be
522 /// inverted before it reaches here or the BACKGROUND is what gets segmented.
523 /// [`crate::normalize_polarity`] is that step and `MonOcr::predict_page`
524 /// runs it. This method does not, because it is also the entry point for a
525 /// caller who has already corrected polarity.
526 pub fn segment(&self, image_path: impl AsRef<Path>) -> Result<Vec<LineSegment>> {
527 let img = image::open(image_path.as_ref())?;
528 self.segment_image(&img.to_luma8())
529 }
530
531 /// Segment an image that is already decoded and grayscale.
532 ///
533 /// The path-taking [`Self::segment`] is a thin wrapper over this. The split
534 /// exists because polarity has to be corrected BEFORE segmentation — the
535 /// threshold below treats dark as ink, so a light-on-dark page segments the
536 /// BACKGROUND and returns the gaps between lines — and the caller doing that
537 /// correction is holding an image, not a path. `go/monocr.go`'s
538 /// `predictImage` and `js/src/monocr.js`'s `normalizePageForSegmentation`
539 /// are the same arrangement.
540 pub fn segment_image(&self, gray_img: &GrayImage) -> Result<Vec<LineSegment>> {
541 let (width, height) = gray_img.dimensions();
542
543 // 1. Get grayscale data and apply threshold
544 //
545 // The mask is materialised before the profile, and the profile is
546 // counted from the mask afterwards rather than in this loop, because
547 // rule suppression needs the 2-D shape of the ink: a per-row count
548 // cannot express "is there an unbroken run this long". Folding the two
549 // back together is what would silently compute the profile from the
550 // unsuppressed page.
551 let mut binary = vec![0u8; (width * height) as usize];
552
553 for y in 0..height {
554 for x in 0..width {
555 let idx = (y * width + x) as usize;
556 let pixel = gray_img.get_pixel(x, y);
557 // Threshold: 128, inverted so text is high (1)
558 if pixel[0] < 128 {
559 binary[idx] = 1;
560 }
561 }
562 }
563
564 // 1.5 Printed-rule suppression, before the profile.
565 //
566 // See `suppress_page_rules` for what a page border costs: its ink floor
567 // clears the gap threshold on every row it spans, and at THIS parameter
568 // set the twelve measured MNEC pages went from 118 bands to 215. It also
569 // runs before `extract_line` reads the mask for column extents, so
570 // removing rules here keeps the border out of the crops too.
571 //
572 // The character-count figure quoted in the Python binding (3,846 to
573 // 5,924) belongs to the reference's adaptive threshold and smear, not to
574 // this one, so it is not repeated here.
575 suppress_page_rules(&mut binary, width, height);
576
577 let mut hist = vec![0f32; height as usize];
578 for y in 0..height {
579 let row = (y * width) as usize;
580 for x in 0..width as usize {
581 if binary[row + x] != 0 {
582 hist[y as usize] += 1.0;
583 }
584 }
585 }
586
587 // 2. Smooth projection profile
588 //
589 // `hist` is kept alive because the two profiles have different jobs: the
590 // threshold below is calibrated on the smoothed one, the boundaries are
591 // detected on the raw one. See step 4 for why.
592 let smoothed_hist = if self.smooth_window > 1 {
593 self.smooth_histogram(&hist)
594 } else {
595 hist.clone()
596 };
597
598 // 3. Gap detection
599 let non_zero_vals: Vec<f32> = smoothed_hist
600 .iter()
601 .filter(|&&v| v > 0.0)
602 .copied()
603 .collect();
604
605 if non_zero_vals.is_empty() {
606 return Ok(Vec::new());
607 }
608
609 let mean_density: f32 = non_zero_vals.iter().sum::<f32>() / non_zero_vals.len() as f32;
610 let gap_threshold = mean_density * self.density_threshold_ratio;
611
612 // 4. Find line regions
613 let page = BinarizedPage {
614 gray: gray_img,
615 binary: &binary,
616 };
617 let mut results = Vec::new();
618 let mut runs: Vec<(u32, u32)> = Vec::new();
619 let mut start: Option<u32> = None;
620
621 for y in 0..height {
622 // Boundaries come off the RAW profile, not the smoothed one.
623 //
624 // The threshold above stays calibrated on the smoothed profile,
625 // because its non-zero mean is steadier. But the smoother averages
626 // several rows together, so a gap narrower than its span never
627 // reaches zero in the smoothed profile: the ink either side bleeds
628 // into it, the bled rows clear the threshold, and the two lines
629 // fuse. The raw profile needs one clean row.
630 //
631 // Measured HERE, at this port's own parameters (min_line_height 10,
632 // density_threshold_ratio 0.05) on 29 drawn bands, driving the
633 // pre-fix form that read boundaries off the smoothed profile. First
634 // gap that returned all 29 bands, by `smooth_window` 1 to 12:
635 //
636 // 1 3 3 5 5 7 7 9 9 11 11 13
637 //
638 // So the break point is `smooth_histogram`'s SPAN,
639 // 2 * (smooth_window / 2) + 1, and NOT the requested window: at
640 // `smooth_window` 4 a gap of exactly 4px still fused. Python's table
641 // is 1,2,...,12 because its kernel is a true window-tap box.
642 // `smooth_window` is a constructor argument, so a caller who raises
643 // it widens the failure with it — at 15 the smoothed profile lost
644 // every page whose lines sat closer than 15px while the raw profile
645 // kept all 29.
646 //
647 // Rust's break point is far tighter than the monorepo apps', which
648 // fused at 5px to 8px, because those ports dilate the mask
649 // vertically before the profile and this one does not. Their
650 // measurements do not transfer; these are this port's.
651 let is_text = hist[y as usize] > gap_threshold;
652
653 if is_text && start.is_none() {
654 start = Some(y);
655 } else if !is_text && start.is_some() {
656 runs.push((start.unwrap(), y));
657 start = None;
658 }
659 }
660
661 // Handle last line if image ends with text
662 if let Some(s) = start {
663 runs.push((s, height));
664 }
665
666 // 4.5 Fuse runs a single sub-threshold row split apart, BEFORE the height
667 // filter. The order is the reference's and it matters: a diacritic strip
668 // can be shorter than `min_line_height`, and filtering first would discard
669 // the strip and leave the decapitated body behind as a whole line.
670 let runs = merge_runs(&runs, &hist, MIN_GAP_MERGE, self.min_line_height);
671
672 for (r0, r1) in runs {
673 if r1 - r0 >= self.min_line_height {
674 self.extract_line(&page, r0..r1, &mut results)?;
675 }
676 }
677
678 Ok(results)
679 }
680
681 /// Smooth histogram using moving average
682 ///
683 /// Applies a moving average filter to the projection histogram to reduce
684 /// noise and smooth out variations. This helps identify text regions more accurately.
685 ///
686 /// # Arguments
687 ///
688 /// * `hist` - Input projection histogram (one value per row)
689 ///
690 /// # Returns
691 ///
692 /// Smoothed histogram with the same length as input
693 ///
694 /// # Algorithm, and two measured divergences from the Python binding
695 ///
696 /// For each position, the mean of `[i - half, i + half]` with
697 /// `half = smooth_window / 2`, over the rows actually in range. Neither
698 /// divergence below is reconciled here: the formula is published behaviour
699 /// for anyone reading the profile, so changing it changes this port's output
700 /// and that is an owner decision.
701 ///
702 /// 1. **Span is `2 * (smooth_window / 2) + 1`, not `smooth_window`.** An EVEN
703 /// window therefore spans one row MORE than asked and is bit-identical to
704 /// the odd window ABOVE it — bit-identical here and in JS because both
705 /// divide by what they summed, but only TAP-identical in Go, which divides
706 /// by the window it was asked for. Python convolves a true `window`-tap
707 /// kernel and spans exactly what it was given.
708 /// Measured on 29 drawn glyph-blob bands at `min_line_height` 10, driving
709 /// the pre-fix form that read boundaries off this profile: the first gap
710 /// returning all 29 bands, for windows 1 to 12, was
711 /// 1,3,3,5,5,7,7,9,9,11,11,13, against Python's 1,2,...,12. So at
712 /// `smooth_window` 4 a gap of exactly 4px still fused. JS and Go measure
713 /// the same table as this port.
714 /// 2. **The divisor is the rows visited, not the window.** Near the top and
715 /// bottom edges fewer rows are in range, and dividing by that count reports
716 /// the true local mean. numpy's `mode='same'` zero-pads and divides by the
717 /// window, attenuating those rows to `(window / 2 + 1) / window` of the
718 /// true mean — two thirds at window 3, 8/15 at window 15. Go divides by the
719 /// window too and so matches numpy, but only at ODD windows: at an even
720 /// window Go sums `window + 1` rows and still divides by `window`, matching
721 /// neither numpy nor this port.
722 ///
723 /// Measured cost, now that the smoothed profile only sets the threshold
724 /// LEVEL: the two formulas disagree only on rows `0..half-1` and their
725 /// mirror at the bottom, and the windows of those rows together cover rows
726 /// `0..2*half-1`, so the blank margin that hides the divergence is
727 /// `2 * half` rows and NOT `half`. Measured on an 8-band page: a 1-row
728 /// margin still left window 3 disagreeing (17.1607 here against Go's
729 /// 17.1429) and a 2-row margin made them agree; window 15 needed 14. Every
730 /// fixture in this repo uses a 30px margin, so all of them sit on the
731 /// agreeing side. On a page cropped flush to the ink the threshold moved
732 /// 0.21% at window 3 and 1.17% at window 15, and no band count changed.
733 ///
734 /// Dividing by the rows visited also means this port cannot produce Go's
735 /// even-window defect, where `window + 1` terms are divided by `window` and the
736 /// smoothed peak clears the raw one by up to 1.5x.
737 fn smooth_histogram(&self, hist: &[f32]) -> Vec<f32> {
738 let height = hist.len();
739 let mut smoothed = vec![0f32; height];
740 let half = (self.smooth_window / 2) as i32;
741
742 // The window reads `hist` while the position writes `smoothed`, two
743 // separate allocations, so iterating the write target does not disturb
744 // the values being averaged.
745 for (i, out) in smoothed.iter_mut().enumerate() {
746 let mut sum = 0f32;
747 let mut count = 0u32;
748
749 for j in (i as i32 - half)..=(i as i32 + half) {
750 if j >= 0 && j < height as i32 {
751 sum += hist[j as usize];
752 count += 1;
753 }
754 }
755
756 *out = if count > 0 { sum / count as f32 } else { 0.0 };
757 }
758
759 smoothed
760 }
761
762 /// Extract a single line from the image and add to results
763 ///
764 /// This method extracts a rectangular region from the grayscale image
765 /// corresponding to a detected text line.
766 ///
767 /// # Process
768 ///
769 /// 1. Find horizontal bounds (x_min, x_max) of text pixels in the region
770 /// 2. Add 4-pixel padding around the detected text
771 /// 3. Crop the region from the original image
772 /// 4. Create a LineSegment with the cropped image and bounding box
773 ///
774 /// # Arguments
775 ///
776 /// * `page` - Source grayscale image and its matching text mask
777 /// * `rows` - Half-open row range (y coordinates) of the line region
778 /// * `results` - Vector to append the extracted line to
779 fn extract_line(
780 &self,
781 page: &BinarizedPage,
782 rows: Range<u32>,
783 results: &mut Vec<LineSegment>,
784 ) -> Result<()> {
785 let gray_img = page.gray;
786 let binary = page.binary;
787 let (width, height) = gray_img.dimensions();
788 let (r_start, r_end) = (rows.start, rows.end);
789
790 // Find horizontal bounds
791 let mut x_min = width;
792 let mut x_max = 0u32;
793 let mut has_pixels = false;
794
795 for y in r_start..r_end {
796 for x in 0..width {
797 let idx = (y * width + x) as usize;
798 if binary[idx] == 1 {
799 if x < x_min {
800 x_min = x;
801 }
802 if x > x_max {
803 x_max = x;
804 }
805 has_pixels = true;
806 }
807 }
808 }
809
810 if !has_pixels {
811 return Ok(());
812 }
813
814 // Add padding around detected text regions to capture edge characters
815 // 4 pixels provides enough margin without including excessive background
816 let pad = 4;
817 let y1 = r_start.saturating_sub(pad);
818 let y2 = (r_end + pad).min(height);
819 let x1 = x_min.saturating_sub(pad);
820 let x2 = (x_max + pad).min(width);
821
822 let w = x2 - x1;
823 let h = y2 - y1;
824
825 // Extract the region
826 let mut line_img = ImageBuffer::new(w, h);
827 for y in 0..h {
828 for x in 0..w {
829 let src_x = x1 + x;
830 let src_y = y1 + y;
831 let pixel = gray_img.get_pixel(src_x, src_y);
832 line_img.put_pixel(x, y, *pixel);
833 }
834 }
835
836 results.push(LineSegment {
837 img: line_img,
838 bbox: BBox { x: x1, y: y1, w, h },
839 });
840
841 Ok(())
842 }
843}
844
845#[cfg(test)]
846mod tests {
847 use super::*;
848 use image::Luma;
849 use serde_json::Value;
850 use std::path::PathBuf;
851
852 /// Override for the shared fixture, for checkouts that do not sit next to
853 /// the monorepo.
854 const FIXTURE_ENV: &str = "MONOCR_TILING_FIXTURE";
855
856 /// The fixture is shared with the web, Android and iOS ports on purpose: one
857 /// file, generated from the Python implementation, so a port that drifts
858 /// fails here instead of in production. Transcribing the numbers into Rust
859 /// would defeat that, so the tests read the JSON.
860 fn fixture_path() -> PathBuf {
861 if let Some(path) = std::env::var_os(FIXTURE_ENV) {
862 return PathBuf::from(path);
863 }
864 PathBuf::from(env!("CARGO_MANIFEST_DIR"))
865 .join("../../monocr-monorepo/shared/segmentation-fixtures/tiling-cases.json")
866 }
867
868 /// A missing fixture fails loudly. Skipping would report a green run for a
869 /// port nothing checked, which is the exact failure this fixture exists to
870 /// prevent.
871 fn load_fixture() -> Value {
872 let path = fixture_path();
873 let raw = std::fs::read_to_string(&path).unwrap_or_else(|e| {
874 panic!(
875 "cannot read the shared tiling fixture at {}: {e}\n\
876 set {FIXTURE_ENV} to point at \
877 monocr-monorepo/shared/segmentation-fixtures/tiling-cases.json",
878 path.display()
879 )
880 });
881 serde_json::from_str(&raw)
882 .unwrap_or_else(|e| panic!("{} is not valid JSON: {e}", path.display()))
883 }
884
885 fn u32_field(value: &Value, key: &str) -> u32 {
886 value
887 .get(key)
888 .and_then(Value::as_u64)
889 .unwrap_or_else(|| panic!("fixture entry is missing an integer '{key}': {value}"))
890 as u32
891 }
892
893 /// Ink is grey 0, background grey 255, per the fixture contract.
894 fn build_image(width: u32, height: u32, ink: &Value) -> GrayImage {
895 let kind = ink
896 .get("kind")
897 .and_then(Value::as_str)
898 .unwrap_or_else(|| panic!("ink rule has no 'kind': {ink}"));
899 let modulus = u32_field(ink, "modulus");
900
901 let mut img = GrayImage::from_pixel(width, height, Luma([255u8]));
902 for x in 0..width {
903 let is_ink = match kind {
904 "mod_eq" => x % modulus == 0,
905 "mod_ne" => x % modulus != 0,
906 "solid" => true,
907 "blank" => false,
908 other => panic!("unknown ink rule '{other}' in the fixture"),
909 };
910 if is_ink {
911 for y in 0..height {
912 img.put_pixel(x, y, Luma([0u8]));
913 }
914 }
915 }
916 img
917 }
918
919 struct Fixture {
920 target_height: u32,
921 target_width: u32,
922 root: Value,
923 }
924
925 fn fixture() -> Fixture {
926 let root = load_fixture();
927 let target_height = u32_field(&root, "target_height");
928 let target_width = u32_field(&root, "target_width");
929
930 // The constants live in two places, so pin them to each other here.
931 assert_eq!(
932 root.get("cut_search_fraction").and_then(Value::as_f64),
933 Some(CUT_SEARCH_FRACTION),
934 "fixture and port disagree on the cut search fraction"
935 );
936 assert_eq!(
937 root.get("cut_ink_threshold").and_then(Value::as_u64),
938 Some(CUT_INK_THRESHOLD as u64),
939 "fixture and port disagree on the ink threshold"
940 );
941
942 Fixture {
943 target_height,
944 target_width,
945 root,
946 }
947 }
948
949 /// An empty array would make every assertion below vacuous, so an empty one
950 /// is a fixture failure rather than a pass.
951 // Floors on the shared fixtures, read off the files on 2026-08-30. Raise one
952 // when a fixture grows and you want the growth pinned; never lower one to make
953 // a red test pass, which is the whole failure this replaces.
954 const TILING_CASES_MIN: usize = 14;
955 const TILING_PROBES_MIN: usize = 3;
956 const RULE_CASES_MIN: usize = 23;
957 const MERGE_CASES_MIN: usize = 18;
958
959 /// `at_least` is a FLOOR ON THE COUNT, and the emptiness check it replaces was
960 /// not enough. A fixture regenerated with three of its eighteen cases is not
961 /// empty, so every port went on passing while testing a sixth of what it
962 /// advertised. The floor catches that; it deliberately does not pin equality,
963 /// because a fixture GAINING a case is the normal and wanted direction and
964 /// should not need an edit in four languages to land.
965 ///
966 /// What a floor cannot catch is a swap: a case removed and another added keeps
967 /// the count. Nothing here notices that, and equality would not either.
968 fn cases(root: &Value, key: &str, at_least: usize) -> Vec<Value> {
969 let cases = root
970 .get(key)
971 .and_then(Value::as_array)
972 .unwrap_or_else(|| panic!("fixture has no '{key}' array"))
973 .clone();
974 assert!(
975 cases.len() >= at_least,
976 "fixture '{key}' carries {} cases, expected at least {at_least} -- \
977 a fixture that shrank is a fixture that stopped testing what it claims",
978 cases.len()
979 );
980 cases
981 }
982
983 fn case_image(case: &Value) -> (GrayImage, String) {
984 let name = case
985 .get("name")
986 .and_then(Value::as_str)
987 .unwrap_or("<unnamed>")
988 .to_string();
989 let ink = case
990 .get("ink")
991 .unwrap_or_else(|| panic!("case '{name}' has no ink rule"));
992 let img = build_image(u32_field(case, "width"), u32_field(case, "height"), ink);
993 (img, name)
994 }
995
996 #[test]
997 fn tile_widths_match_the_shared_fixture() {
998 let f = fixture();
999 for case in cases(&f.root, "cases", TILING_CASES_MIN) {
1000 let (img, name) = case_image(&case);
1001 let expected: Vec<u32> = case
1002 .get("expected_tile_widths")
1003 .and_then(Value::as_array)
1004 .unwrap_or_else(|| panic!("case '{name}' has no expected_tile_widths"))
1005 .iter()
1006 .map(|v| {
1007 v.as_u64()
1008 .unwrap_or_else(|| panic!("case '{name}' has a non-integer tile width"))
1009 as u32
1010 })
1011 .collect();
1012
1013 let widths: Vec<u32> = tile_line(&img, f.target_height, f.target_width)
1014 .iter()
1015 .map(|t| t.width())
1016 .collect();
1017
1018 assert_eq!(widths, expected, "case '{name}'");
1019 }
1020 }
1021
1022 /// Tiles must cover the line exactly once. Concatenating them back is the
1023 /// direct proof: a gap or an overlap changes the pixels, not just the count.
1024 #[test]
1025 fn tiles_partition_the_line() {
1026 let f = fixture();
1027 for case in cases(&f.root, "cases", TILING_CASES_MIN) {
1028 let (img, name) = case_image(&case);
1029 let tiles = tile_line(&img, f.target_height, f.target_width);
1030
1031 let total: u32 = tiles.iter().map(|t| t.width()).sum();
1032 assert_eq!(
1033 total,
1034 img.width(),
1035 "case '{name}': tile widths must sum to the line width"
1036 );
1037
1038 let mut rebuilt = GrayImage::new(img.width(), img.height());
1039 let mut x_off = 0u32;
1040 for tile in &tiles {
1041 assert_eq!(
1042 tile.height(),
1043 img.height(),
1044 "case '{name}': a tile must keep the full line height"
1045 );
1046 assert!(tile.width() > 0, "case '{name}': empty tile");
1047 for x in 0..tile.width() {
1048 for y in 0..tile.height() {
1049 rebuilt.put_pixel(x_off + x, y, *tile.get_pixel(x, y));
1050 }
1051 }
1052 x_off += tile.width();
1053 }
1054 assert!(
1055 rebuilt.as_raw() == img.as_raw(),
1056 "case '{name}': tiles do not reassemble into the source line"
1057 );
1058 }
1059 }
1060
1061 /// The single-line path (`MonOcr::predict_single_line`) runs a caller's
1062 /// already-cropped line through the same `tile_line`, so a wide crop must
1063 /// come back as several tiles covering the full width. If it ever returned
1064 /// one tile the crop would be squeezed into the model window. Measured cost of
1065 /// that on this binding: nothing at 3 tiles, 4.1x the error at 4, and 23x at 8
1066 /// (`examples/tiling_ab.rs`, `mon_OCR/eval/tiling-ab-2026-08-22.md`).
1067 #[test]
1068 fn a_wide_crop_is_tiled_not_squeezed() {
1069 let f = fixture();
1070 let mut checked = 0;
1071
1072 for case in cases(&f.root, "cases", TILING_CASES_MIN) {
1073 let expected_count = case
1074 .get("expected_tile_widths")
1075 .and_then(Value::as_array)
1076 .map(|a| a.len())
1077 .unwrap_or(0);
1078 if expected_count < 2 {
1079 continue;
1080 }
1081
1082 let (img, name) = case_image(&case);
1083 let tiles = tile_line(&img, f.target_height, f.target_width);
1084 assert!(
1085 tiles.len() > 1,
1086 "case '{name}': a crop this wide must be tiled, got {} tile(s)",
1087 tiles.len()
1088 );
1089 assert_eq!(
1090 tiles.iter().map(|t| t.width()).sum::<u32>(),
1091 img.width(),
1092 "case '{name}': tiles must cover the whole crop"
1093 );
1094 for tile in &tiles {
1095 assert!(
1096 tile.width() <= img.width(),
1097 "case '{name}': a tile cannot be wider than the crop"
1098 );
1099 }
1100 checked += 1;
1101 }
1102
1103 assert!(checked > 0, "the fixture has no multi-tile case to check");
1104 }
1105
1106 /// Override for the shared printed-rule fixture, for checkouts that do not
1107 /// sit next to the monorepo.
1108 const RULE_FIXTURE_ENV: &str = "MONOCR_RULE_FIXTURE";
1109
1110 /// The printed-rule fixture is the oracle four ports share, generated from
1111 /// the reference implementation by
1112 /// `monocr-monorepo/shared/segmentation-fixtures/generate-rule-cases.py`. It
1113 /// describes each mask as a PRNG seed plus a list of rules, so a port builds
1114 /// the same 23 masks without shipping any pixels, and checks the result
1115 /// against an ink count and a position-weighted checksum.
1116 ///
1117 /// The checksum is the part that matters: a bare ink count would not notice
1118 /// suppression that removed the right NUMBER of pixels in the wrong places,
1119 /// which is exactly what an off-by-one in a run-length scan produces.
1120 fn rule_fixture_path() -> PathBuf {
1121 if let Some(path) = std::env::var_os(RULE_FIXTURE_ENV) {
1122 return PathBuf::from(path);
1123 }
1124 PathBuf::from(env!("CARGO_MANIFEST_DIR"))
1125 .join("../../monocr-monorepo/shared/segmentation-fixtures/rule-cases.json")
1126 }
1127
1128 /// A missing fixture fails loudly, for the same reason `load_fixture` does:
1129 /// skipping would report a green run for a port nothing checked.
1130 fn load_rule_fixture() -> Value {
1131 let path = rule_fixture_path();
1132 let raw = std::fs::read_to_string(&path).unwrap_or_else(|e| {
1133 panic!(
1134 "cannot read the shared printed-rule fixture at {}: {e}\n\
1135 set {RULE_FIXTURE_ENV} to point at \
1136 monocr-monorepo/shared/segmentation-fixtures/rule-cases.json",
1137 path.display()
1138 )
1139 });
1140 serde_json::from_str(&raw)
1141 .unwrap_or_else(|e| panic!("{} is not valid JSON: {e}", path.display()))
1142 }
1143
1144 fn i64_field(value: &Value, key: &str) -> i64 {
1145 value
1146 .get(key)
1147 .and_then(Value::as_i64)
1148 .unwrap_or_else(|| panic!("fixture entry is missing an integer '{key}': {value}"))
1149 }
1150
1151 /// Rebuild one fixture mask: xorshift32 noise, then the rules drawn over it.
1152 ///
1153 /// The PRNG is transcribed from the fixture's own `prng` field rather than
1154 /// invented here: `x ^= x<<13; x ^= x>>17; x ^= x<<5`, seeded 2463534242,
1155 /// pixel `i` ink where `x % 100 < density` with `x` taken after the i-th
1156 /// step. Rust's `<<` on `u32` discards the high bits, which is the `&
1157 /// 0xFFFFFFFF` the generator writes explicitly.
1158 fn rule_mask(case: &Value) -> (Vec<u8>, u32, u32) {
1159 let width = u32_field(case, "width");
1160 let height = u32_field(case, "height");
1161 let (w, h) = (width as usize, height as usize);
1162 let density = u32_field(case, "density");
1163
1164 let mut x: u32 = 2_463_534_242;
1165 let mut mask = vec![0u8; w * h];
1166 for cell in mask.iter_mut() {
1167 x ^= x << 13;
1168 x ^= x >> 17;
1169 x ^= x << 5;
1170 if x % 100 < density {
1171 *cell = 1;
1172 }
1173 }
1174
1175 let run_length = i64_field(case, "run_length");
1176 let run_start = i64_field(case, "run_start") as usize;
1177 for row in cases_array(case, "rule_rows") {
1178 let row = row as usize;
1179 let (len, start) = if run_length < 0 {
1180 (w, 0)
1181 } else {
1182 (run_length as usize, run_start)
1183 };
1184 for cell in mask[row * w + start..row * w + w.min(start + len)].iter_mut() {
1185 *cell = 1;
1186 }
1187 }
1188
1189 let col_length = i64_field(case, "col_length");
1190 let col_start = i64_field(case, "col_start") as usize;
1191 for col in cases_array(case, "rule_cols") {
1192 let col = col as usize;
1193 let (len, start) = if col_length < 0 {
1194 (h, 0)
1195 } else {
1196 (col_length as usize, col_start)
1197 };
1198 for y in start..h.min(start + len) {
1199 mask[y * w + col] = 1;
1200 }
1201 }
1202
1203 (mask, width, height)
1204 }
1205
1206 fn cases_array(case: &Value, key: &str) -> Vec<u64> {
1207 case.get(key)
1208 .and_then(Value::as_array)
1209 .unwrap_or_else(|| panic!("fixture case has no '{key}' array: {case}"))
1210 .iter()
1211 .map(|v| {
1212 v.as_u64()
1213 .unwrap_or_else(|| panic!("non-integer entry in '{key}'"))
1214 })
1215 .collect()
1216 }
1217
1218 /// Ink count and position-weighted checksum of a mask, flattened row-major,
1219 /// exactly as the fixture generator's `signature` computes them.
1220 fn rule_signature(mask: &[u8], modulus: u64) -> (u64, u64) {
1221 let mut ink = 0u64;
1222 let mut sum = 0u64;
1223 for (i, &v) in mask.iter().enumerate() {
1224 if v != 0 {
1225 ink += 1;
1226 sum += i as u64 + 1;
1227 }
1228 }
1229 (ink, sum % modulus)
1230 }
1231
1232 /// The whole printed-rule contract, against the oracle the other ports use.
1233 ///
1234 /// 23 cases, including the pair that pins `>=` on each axis (a run of exactly
1235 /// the span and one pixel short), the 15px floor on a narrow crop, the
1236 /// truncated span on an odd width, and the ink-share ceiling both firing and
1237 /// exactly at the boundary.
1238 #[test]
1239 fn page_rules_match_the_shared_fixture() {
1240 let root = load_rule_fixture();
1241
1242 // The constants live in two places, so pin them to each other here, the
1243 // same way `fixture()` does for the tiling constants.
1244 assert_eq!(
1245 root.get("rule_span").and_then(Value::as_f64),
1246 Some(RULE_SPAN),
1247 "fixture and port disagree on the rule span"
1248 );
1249 assert_eq!(
1250 root.get("rule_max_ink_share").and_then(Value::as_f64),
1251 Some(RULE_MAX_INK_SHARE),
1252 "fixture and port disagree on the ink-share ceiling"
1253 );
1254 let modulus = root
1255 .get("checksum_modulus")
1256 .and_then(Value::as_u64)
1257 .expect("fixture has no checksum_modulus");
1258
1259 for case in cases(&root, "cases", RULE_CASES_MIN) {
1260 let name = case
1261 .get("name")
1262 .and_then(Value::as_str)
1263 .unwrap_or("<unnamed>")
1264 .to_string();
1265 let (mut mask, width, height) = rule_mask(&case);
1266
1267 let changed = suppress_page_rules(&mut mask, width, height);
1268 assert_eq!(
1269 changed,
1270 case.get("expected_changed")
1271 .and_then(Value::as_bool)
1272 .unwrap_or_else(|| panic!("case '{name}' has no expected_changed")),
1273 "case '{name}': wrong answer on whether anything was suppressed"
1274 );
1275
1276 let (ink, checksum) = rule_signature(&mask, modulus);
1277 assert_eq!(
1278 ink,
1279 case.get("expected_ink")
1280 .and_then(Value::as_u64)
1281 .unwrap_or_else(|| panic!("case '{name}' has no expected_ink")),
1282 "case '{name}': wrong ink count after suppression"
1283 );
1284 assert_eq!(
1285 checksum,
1286 case.get("expected_checksum")
1287 .and_then(Value::as_u64)
1288 .unwrap_or_else(|| panic!("case '{name}' has no expected_checksum")),
1289 "case '{name}': right ink count, wrong pixels — an off-by-one in \
1290 one of the run-length scans"
1291 );
1292 }
1293 }
1294
1295 // A realistic page, in the shape `go/pkg/segmenter/page_rules_test.go` uses:
1296 // glyph blobs rather than solid bars, because a solid bar the width of a text
1297 // column IS a rule by any definition and would prove nothing.
1298 const T_WIDTH: u32 = 800;
1299 const T_BAND: u32 = 40;
1300 const T_MARGIN: u32 = 30;
1301 const T_GLYPH_W: u32 = 12;
1302 const T_PITCH: u32 = 20;
1303 const T_RULE_W: u32 = 4;
1304
1305 /// Build a page as a grayscale image: ink 0, background 255.
1306 fn drawn_page(bands: u32, gap: u32, glyphs: u32, framed: bool) -> GrayImage {
1307 let height = T_MARGIN * 2 + T_BAND * bands + gap * (bands - 1);
1308 let mut img = GrayImage::from_pixel(T_WIDTH, height, Luma([255u8]));
1309 let mut y = T_MARGIN;
1310 for _ in 0..bands {
1311 for yy in y..y + T_BAND {
1312 for k in 0..glyphs {
1313 let x0 = 100 + k * T_PITCH;
1314 for i in 0..T_GLYPH_W {
1315 if x0 + i < T_WIDTH {
1316 img.put_pixel(x0 + i, yy, Luma([0u8]));
1317 }
1318 }
1319 }
1320 }
1321 y += T_BAND + gap;
1322 }
1323 if framed {
1324 for yy in 0..height {
1325 for i in 0..T_RULE_W {
1326 img.put_pixel(10 + i, yy, Luma([0u8]));
1327 img.put_pixel(T_WIDTH - 10 - T_RULE_W + i, yy, Luma([0u8]));
1328 }
1329 }
1330 for i in 0..T_RULE_W {
1331 for x in 0..T_WIDTH {
1332 img.put_pixel(x, 10 + i, Luma([0u8]));
1333 img.put_pixel(x, height - 10 - T_RULE_W + i, Luma([0u8]));
1334 }
1335 }
1336 }
1337 img
1338 }
1339
1340 /// THE PROPERTY THAT MAKES THIS SAFE UNCONDITIONALLY. Every page gets the
1341 /// step whether it carries rules or not, so "does nothing" has to be exact
1342 /// rather than approximate.
1343 #[test]
1344 fn a_page_with_no_rules_is_untouched_to_the_pixel() {
1345 let img = drawn_page(4, 40, 30, false);
1346 let (w, h) = img.dimensions();
1347 let mut mask = vec![0u8; (w * h) as usize];
1348 for y in 0..h {
1349 for x in 0..w {
1350 if img.get_pixel(x, y)[0] < 128 {
1351 mask[(y * w + x) as usize] = 1;
1352 }
1353 }
1354 }
1355 let before = mask.clone();
1356
1357 assert!(
1358 !suppress_page_rules(&mut mask, w, h),
1359 "suppression reported a change on a page with no rules"
1360 );
1361 assert_eq!(
1362 mask, before,
1363 "glyph-sized ink was classified as a rule and removed"
1364 );
1365 }
1366
1367 /// The behavioural test, and the fixture took finding.
1368 ///
1369 /// A DENSE framed page does not fuse at this parameter set — 30 glyphs per
1370 /// line segments the same with or without suppression, which is why a
1371 /// structural check on the mask alone cannot catch a profile computed from
1372 /// the wrong buffer. SPARSE text reproduces the real mechanism: with 8 glyphs
1373 /// per line the profile mean drops far enough that the frame's ink floor
1374 /// clears the 0.05 threshold on every row, and the page comes back as one
1375 /// band. Ported from `go/pkg/segmenter/page_rules_test.go`
1376 /// `TestSegmentRecoversAFramedPage`.
1377 #[test]
1378 fn segmenting_recovers_a_framed_page() {
1379 let seg = LineSegmenter::new(10, 3);
1380 let clean = seg.segment_image(&drawn_page(4, 40, 8, false)).unwrap();
1381 let framed = seg.segment_image(&drawn_page(4, 40, 8, true)).unwrap();
1382
1383 assert_eq!(
1384 clean.len(),
1385 4,
1386 "the unframed control must segment into 4 lines, or the comparison \
1387 below proves nothing"
1388 );
1389 assert_eq!(
1390 framed.len(),
1391 clean.len(),
1392 "a framed page came back as {} line(s) where the same page unframed \
1393 gave {} — the page border is fusing the profile",
1394 framed.len(),
1395 clean.len()
1396 );
1397 }
1398
1399 /// Degenerate shapes reach this from real callers: a 1px crop, and a mask
1400 /// whose length disagrees with its stated dimensions. Indexing is what would
1401 /// panic, so the guards are worth a test even though they assert nothing but
1402 /// survival.
1403 #[test]
1404 fn degenerate_masks_do_not_panic() {
1405 assert!(!suppress_page_rules(&mut [], 0, 0));
1406 assert!(!suppress_page_rules(&mut [], 10, 10));
1407 assert!(!suppress_page_rules(&mut vec![0u8; 50 * 50], 50, 50));
1408 assert!(!suppress_page_rules(&mut vec![1u8; 50 * 50], 50, 50));
1409 assert!(!suppress_page_rules(&mut [1u8; 1], 1, 1));
1410 }
1411
1412 #[test]
1413 fn cut_column_matches_the_shared_fixture() {
1414 let f = fixture();
1415 for probe in cases(&f.root, "cut_column_probes", TILING_PROBES_MIN) {
1416 let (img, name) = case_image(&probe);
1417 let got = cut_column(
1418 &img,
1419 u32_field(&probe, "x0"),
1420 u32_field(&probe, "ideal"),
1421 img.width(),
1422 );
1423 assert_eq!(got, u32_field(&probe, "expected_cut"), "probe '{name}'");
1424 }
1425 }
1426
1427 /// A page of `bands` dense bands plus one faint band carrying exactly
1428 /// `faint_ink` ink pixels per row, used to probe the threshold LEVEL rather
1429 /// than the profile the boundaries come from.
1430 fn page_with_a_faint_band(
1431 bands: u32,
1432 gap: u32,
1433 glyphs: u32,
1434 faint_ink: u32,
1435 faint_h: u32,
1436 ) -> GrayImage {
1437 let height = T_MARGIN * 2 + T_BAND * bands + gap * bands + faint_h;
1438 let mut img = GrayImage::from_pixel(T_WIDTH, height, Luma([255u8]));
1439 let mut y = T_MARGIN;
1440 for _ in 0..bands {
1441 for yy in y..y + T_BAND {
1442 for k in 0..glyphs {
1443 let x0 = 100 + k * T_PITCH;
1444 for i in 0..T_GLYPH_W {
1445 if x0 + i < T_WIDTH {
1446 img.put_pixel(x0 + i, yy, Luma([0u8]));
1447 }
1448 }
1449 }
1450 }
1451 y += T_BAND + gap;
1452 }
1453 for yy in y..y + faint_h {
1454 for i in 0..faint_ink {
1455 img.put_pixel(100 + i, yy, Luma([0u8]));
1456 }
1457 }
1458 img
1459 }
1460
1461 /// THE CASE THE DUAL HISTOGRAM EXISTS FOR, measured at this port's own
1462 /// parameters rather than borrowed from another port.
1463 ///
1464 /// With the default `smooth_window` of 3 the smoother averages three rows,
1465 /// so a gap of 1px or 2px never reaches zero in the smoothed profile — the
1466 /// ink either side bleeds into it and clears the threshold. Reading
1467 /// boundaries there returned 1 band against 29 drawn. 3px is the first gap
1468 /// the smoothed profile survives, which is why it is the control here and
1469 /// not the interesting case.
1470 #[test]
1471 fn lines_two_pixels_apart_are_not_fused() {
1472 let seg = LineSegmenter::new(10, 3);
1473 for gap in [1u32, 2] {
1474 let got = seg.segment_image(&drawn_page(29, gap, 30, false)).unwrap();
1475 assert_eq!(
1476 got.len(),
1477 29,
1478 "29 bands {gap}px apart came back as {} — boundaries are being \
1479 read off the smoothed profile again",
1480 got.len()
1481 );
1482 }
1483 let control = seg.segment_image(&drawn_page(29, 3, 30, false)).unwrap();
1484 assert_eq!(
1485 control.len(),
1486 29,
1487 "the 3px control failed, so the regression is not the profile choice"
1488 );
1489 }
1490
1491 /// Speckle must not decide what a typical line is.
1492 ///
1493 /// The merge runs before the height filter, so `runs` holds every speck the
1494 /// profile picked up. Twelve 2-row specks against five 50-row lines: a median
1495 /// over ALL runs is 2 and the ceiling 4, which refuses every merge and switches
1496 /// the pass off on the pages that need it most. Filtering to runs that could be
1497 /// a line gives 50 and a ceiling of 100.
1498 #[test]
1499 fn speckle_does_not_set_the_typical_line_height() {
1500 let mut hist = vec![0f32; 700];
1501 let mut runs: Vec<(u32, u32)> = Vec::new();
1502 // Specks first, so they dominate the count.
1503 for i in 0..12u32 {
1504 let y = i * 4;
1505 hist[y as usize..(y + 2) as usize].fill(20.0);
1506 runs.push((y, y + 2));
1507 }
1508 // Then a split line whose halves ARE halves: 24 rows, a 2-row inked dip, 24
1509 // rows, summing to the 50 an ordinary line measures here. An earlier version
1510 // of this fixture used 50 + 50, which is two whole lines by its own page's
1511 // standard, and the ceiling refused the merge for the right reason.
1512 hist[100..124].fill(300.0);
1513 hist[124..126].fill(5.0);
1514 hist[126..150].fill(300.0);
1515 runs.push((100, 124));
1516 runs.push((126, 150));
1517 // And three ordinary lines, so a real median exists.
1518 for i in 0..3u32 {
1519 let y = 200 + i * 60;
1520 hist[y as usize..(y + 50) as usize].fill(300.0);
1521 runs.push((y, y + 50));
1522 }
1523
1524 let merged = merge_runs(&runs, &hist, MIN_GAP_MERGE, 10);
1525 assert!(
1526 merged.contains(&(100, 150)),
1527 "the split pair did not merge, so speckle set the ceiling: got {merged:?}"
1528 );
1529 // And the specks must not have fused into something the height filter will
1530 // pass. Twelve 2-row specks chaining into one 46-row band is a line the
1531 // recogniser is handed and asked to read.
1532 let speckle_band = merged
1533 .iter()
1534 .filter(|&&(a, b)| a < 100 && b - a >= 10)
1535 .count();
1536 assert_eq!(
1537 speckle_band, 0,
1538 "speckle fused into {speckle_band} band(s) tall enough to clear the \
1539 height filter: got {merged:?}"
1540 );
1541 }
1542
1543 /// The ceiling, isolated: every other clause says merge and only the height
1544 /// cap refuses.
1545 ///
1546 /// Two 60-row runs two rows apart with ink in the gap, on a page whose typical
1547 /// run is 60. `gap_size` is inside the bound, `gap_has_ink` is true, so without
1548 /// the cap this merges. The merged span would be 122 against a ceiling of 120.
1549 #[test]
1550 fn no_merge_may_exceed_twice_a_typical_line() {
1551 let mut hist = vec![0f32; 400];
1552 hist[20..80].fill(300.0);
1553 hist[80..82].fill(5.0); // ink in the gap: the ink clause would merge
1554 hist[82..142].fill(300.0);
1555 hist[200..260].fill(300.0);
1556 hist[300..360].fill(300.0);
1557 let runs = [
1558 (20u32, 80u32),
1559 (82u32, 142u32),
1560 (200u32, 260u32),
1561 (300u32, 360u32),
1562 ];
1563 assert_eq!(
1564 merge_runs(&runs, &hist, MIN_GAP_MERGE, 10),
1565 vec![(20, 80), (82, 142), (200, 260), (300, 360)],
1566 "a merge produced a band taller than twice a typical line"
1567 );
1568 }
1569
1570 /// The cascade the page median exists to prevent.
1571 ///
1572 /// Judging a fragment against the NEIGHBOUR's height snowballs: the merge
1573 /// mutates the accumulated run, and a taller accumulation makes the next line
1574 /// look more like a fragment. Measured on real input before this was fixed —
1575 /// one page went from 36 bands to 10, with single bands of 534, 632 and 732
1576 /// rows, and lost 92% of its readable characters.
1577 ///
1578 /// Here a chain of runs each two rows from the next, with ink throughout, must
1579 /// not collapse into one band. The assertion is on the property rather than an
1580 /// exact list, because what matters is that nothing runs away.
1581 #[test]
1582 fn merging_does_not_cascade_down_a_page() {
1583 // One run is 100 rows and the rest 50, so the page MEDIAN is 50 while its
1584 // MAX is 100. That difference is the test: with the median, `typical` is 50
1585 // and the ceiling 100, so the first merge would reach 102 and is refused.
1586 // Read `typical` off the max instead and the ceiling doubles to 200, the
1587 // fragment clause starts firing on every 50-row line, and the chain
1588 // collapses. A fixture of equal-height runs cannot see that at all — the
1589 // median and the max are the same number — which is how this survived a
1590 // battery once.
1591 let mut hist = vec![0f32; 700];
1592 let mut runs = Vec::new();
1593 let mut y = 20u32;
1594 for i in 0..8 {
1595 let h = if i == 3 { 100 } else { 50 };
1596 hist[y as usize..(y + h) as usize].fill(300.0);
1597 hist[(y + h) as usize..(y + h + 2) as usize].fill(5.0); // ink in every gap
1598 runs.push((y, y + h));
1599 y += h + 2;
1600 }
1601 let merged = merge_runs(&runs, &hist, MIN_GAP_MERGE, 10);
1602 let tallest = merged.iter().map(|&(a, b)| b - a).max().unwrap();
1603 assert!(
1604 tallest <= 100,
1605 "a chain of 50-row runs collapsed into a band {tallest} rows tall, so \
1606 the merge is cascading"
1607 );
1608 assert!(
1609 merged.len() >= 4,
1610 "8 runs became {} bands, so the merge is cascading",
1611 merged.len()
1612 );
1613 }
1614
1615 /// The merge must be reached THROUGH `segment_image`, not only unit-tested.
1616 ///
1617 /// Added after a mutation that deleted the `merge_runs` call from the pipeline
1618 /// SURVIVED all four unit tests below — they call the helper directly, so the
1619 /// call site was unguarded. That is the gap `se-brain`
1620 /// `rules/standards/testing.md` names: a tested helper does not make its call
1621 /// site safe.
1622 ///
1623 /// Geometry is the measured one: a 20-row strip of upper marks, two empty
1624 /// rows, then a 44-row body. One line, and it must come back as one band.
1625 #[test]
1626 fn a_diacritic_strip_is_returned_joined_to_its_line() {
1627 let (w, h) = (T_WIDTH, 200u32);
1628 let mut img = GrayImage::from_pixel(w, h, Luma([255u8]));
1629 let ink = |img: &mut GrayImage, y0: u32, y1: u32, every: u32| {
1630 for yy in y0..y1 {
1631 for k in 0..30u32 {
1632 let x0 = 100 + k * T_PITCH;
1633 for i in 0..every {
1634 if x0 + i < w {
1635 img.put_pixel(x0 + i, yy, Luma([0u8]));
1636 }
1637 }
1638 }
1639 }
1640 };
1641 // Sparse marks above, solid body below, two blank rows between.
1642 ink(&mut img, 60, 80, 2);
1643 ink(&mut img, 82, 126, T_GLYPH_W);
1644
1645 let got = LineSegmenter::new(10, 3).segment_image(&img).unwrap();
1646 assert_eq!(
1647 got.len(),
1648 1,
1649 "the strip and its body came back as {} bands — the merge is not \
1650 reached from segment_image",
1651 got.len()
1652 );
1653 assert!(
1654 got[0].bbox.h >= 60,
1655 "the returned band is {}px tall, so it holds the body without the \
1656 marks above it",
1657 got[0].bbox.h
1658 );
1659 }
1660
1661 /// The ink clause on its own, which the four cases below do not isolate: in
1662 /// the measured dip case the fragment clause ALSO fires, so dropping
1663 /// `gap_has_ink` survived. Here the runs are the same height, so `fragment`
1664 /// is false and only the ink test can merge them.
1665 #[test]
1666 fn a_dip_between_equal_halves_merges_on_ink_alone() {
1667 // The fixture carries two ORDINARY lines as well as the split pair, and
1668 // that is load-bearing rather than decoration. `merge_runs` judges a
1669 // fragment against the page's typical line height, so a page consisting of
1670 // nothing but two halves is degenerate — there is no evidence in it that
1671 // they are halves rather than two short lines, and an earlier version of
1672 // this test asserted a merge the code had no grounds to make.
1673 // The companion lines are 60 rows, not 82, and the arithmetic is the whole
1674 // point of the test. Median run height is 60, so `2 * 40 > 60` and the
1675 // FRAGMENT clause is false — only `gap_has_ink` can merge this pair, and
1676 // the merged 82 rows still fit the 120-row ceiling.
1677 //
1678 // At 82-row companions, which is what this fixture held until 2026-08-28,
1679 // the median rose to 82, `2 * 40 <= 82` fired, and dropping the ink clause
1680 // left the test passing. It was written to isolate one clause and silently
1681 // stopped doing so when the fixture changed to satisfy the ceiling, and the
1682 // mutation battery was not re-run afterwards. Found by a sibling port.
1683 let mut hist = vec![0f32; 400];
1684 hist[20..60].fill(300.0);
1685 hist[60..62].fill(5.0); // two rows of ink: below any threshold, above zero
1686 hist[62..102].fill(300.0);
1687 hist[150..210].fill(300.0);
1688 hist[260..320].fill(300.0);
1689 let runs = [
1690 (20u32, 60u32),
1691 (62u32, 102u32),
1692 (150u32, 210u32),
1693 (260u32, 320u32),
1694 ];
1695 assert_eq!(
1696 merge_runs(&runs, &hist, MIN_GAP_MERGE, 10),
1697 vec![(20, 102), (150, 210), (260, 320)],
1698 "an ink-holding 2-row dip between two halves of a typical line did \
1699 not merge"
1700 );
1701 }
1702
1703 /// `merge_runs`, both clauses, on the numbers that were measured rather than
1704 /// on invented ones. See `MIN_GAP_MERGE`.
1705 #[test]
1706 fn a_sub_threshold_dip_does_not_end_a_line() {
1707 // The measured case: one line, rows 260-324, split by row 280 carrying 6
1708 // ink pixels against a threshold of 7.0.
1709 let mut hist = vec![0f32; 400];
1710 hist[260..325].fill(200.0);
1711 hist[280] = 6.0; // above zero, below the gap threshold
1712 let runs = [(260u32, 280u32), (281u32, 325u32)];
1713 assert_eq!(
1714 merge_runs(&runs, &hist, MIN_GAP_MERGE, 10),
1715 vec![(260, 325)],
1716 "a 1-row dip holding ink split one line in two"
1717 );
1718 }
1719
1720 #[test]
1721 fn a_zero_gap_still_merges_a_fragment_into_its_line() {
1722 // The other measured case: rows 341-360 are the upper marks and 362-404
1723 // the body of one line, separated by TWO rows of genuinely zero ink. The
1724 // ink clause cannot cross that; the height ratio is what does.
1725 let mut hist = vec![0f32; 500];
1726 hist[341..360].fill(40.0);
1727 hist[362..404].fill(300.0);
1728 let runs = [(341u32, 360u32), (362u32, 404u32)];
1729 assert_eq!(
1730 merge_runs(&runs, &hist, MIN_GAP_MERGE, 10),
1731 vec![(341, 404)],
1732 "a 19-row fragment two empty rows from a 42-row line stayed separate"
1733 );
1734 }
1735
1736 #[test]
1737 fn two_real_lines_two_rows_apart_stay_separate() {
1738 // The case the fragment clause must NOT swallow, and the reason it is a
1739 // ratio: same gap, same emptiness, but both runs are full height.
1740 // 60-row companions so the FRAGMENT test is the only thing refusing this.
1741 // Median 60, ceiling 120, merged span 82 — inside the ceiling. The gap holds
1742 // no ink, so `gap_has_ink` is false. `2 * 40 > 60`, so `fragment` is false
1743 // and the pair stays apart for that reason alone.
1744 //
1745 // Without the companions the ceiling refused it independently (median 40,
1746 // ceiling 80, merged 82), so loosening the fragment ratio from 2x to 1x
1747 // left this test passing.
1748 let mut hist = vec![0f32; 400];
1749 hist[20..60].fill(300.0);
1750 hist[62..102].fill(300.0);
1751 hist[180..240].fill(300.0);
1752 hist[280..340].fill(300.0);
1753 let runs = [
1754 (20u32, 60u32),
1755 (62u32, 102u32),
1756 (180u32, 240u32),
1757 (280u32, 340u32),
1758 ];
1759 assert_eq!(
1760 merge_runs(&runs, &hist, MIN_GAP_MERGE, 10),
1761 vec![(20, 60), (62, 102), (180, 240), (280, 340)],
1762 "two 40-row lines were fused, which is what SMEAR_Y would have done"
1763 );
1764 }
1765
1766 #[test]
1767 fn a_wide_gap_is_a_line_boundary_however_much_ink_it_holds() {
1768 // The size bound on its own. Overlapping diacritics can hold the raw
1769 // profile above zero right across real inter-line spacing; upstream that
1770 // collapsed 3 PDF lines into 1.
1771 // Companions at 60 rows so the SIZE BOUND is the only thing refusing this
1772 // merge. Median 60, ceiling 120, and the merged span would be 95 — inside
1773 // the ceiling. `2 * 40 > 60`, so the fragment clause is false. The gap
1774 // holds ink throughout, so `gap_has_ink` is true and WOULD merge. Only
1775 // `gap_size <= max_gap` stands in the way.
1776 //
1777 // Without the companions the ceiling refused it independently (median 40,
1778 // ceiling 80, merged 95), so removing the size bound left this test
1779 // passing and its coverage came incidentally from an unrelated test.
1780 let mut hist = vec![0f32; 400];
1781 hist[20..60].fill(300.0);
1782 hist[60..75].fill(5.0); // 15 rows of ink between two lines
1783 hist[75..115].fill(300.0);
1784 hist[180..240].fill(300.0);
1785 hist[280..340].fill(300.0);
1786 let runs = [
1787 (20u32, 60u32),
1788 (75u32, 115u32),
1789 (180u32, 240u32),
1790 (280u32, 340u32),
1791 ];
1792 assert_eq!(
1793 merge_runs(&runs, &hist, MIN_GAP_MERGE, 10),
1794 vec![(20, 60), (75, 115), (180, 240), (280, 340)],
1795 "a 15-row gap merged, so the size bound is not being applied"
1796 );
1797 }
1798
1799 /// The opposite failure, and the reason this needs its own test: the raw
1800 /// profile is the more sensitive of the two, so the risk of reading it is
1801 /// splitting where no gap exists. Bands that touch share ink on every row,
1802 /// there is no clean row anywhere, and one band is the honest answer.
1803 #[test]
1804 fn touching_bands_stay_one_line() {
1805 let seg = LineSegmenter::new(10, 3);
1806 let got = seg.segment_image(&drawn_page(29, 0, 30, false)).unwrap();
1807 assert_eq!(got.len(), 1, "touching bands were split into {}", got.len());
1808 }
1809
1810 /// `smooth_window` is a constructor argument, and on the smoothed profile
1811 /// raising it widened the damage: the break point is the smoother's SPAN,
1812 /// 2 * (smooth_window / 2) + 1 and not the requested window, so at 15 every
1813 /// page whose lines sat closer than 15px collapsed to one band. Measured at
1814 /// 5px and 12px, both of which the old form lost. 15 is odd, so span and
1815 /// window coincide here; the even-window case is pinned against
1816 /// `smooth_histogram` directly, below.
1817 #[test]
1818 fn a_wide_smoother_does_not_fuse_the_page() {
1819 let seg = LineSegmenter::new(10, 15);
1820 for gap in [5u32, 12] {
1821 let got = seg.segment_image(&drawn_page(29, gap, 30, false)).unwrap();
1822 assert_eq!(
1823 got.len(),
1824 29,
1825 "at smooth_window 15, 29 bands {gap}px apart came back as {}",
1826 got.len()
1827 );
1828 }
1829 }
1830
1831 /// The other half of the dual histogram: the LEVEL still comes off the
1832 /// smoothed profile.
1833 ///
1834 /// Calibrating on the raw profile instead raises the threshold, because
1835 /// smoothing spreads ink into the rows either side of every band and those
1836 /// partial rows pull the non-zero mean down. A band faint enough to sit
1837 /// between the two thresholds is then dropped, and dropping a line is the
1838 /// failure this pipeline is built to avoid.
1839 ///
1840 /// The fixture is tuned, and the tuning is the finding: at the default ratio
1841 /// of 0.05 the two thresholds sit 0.88px apart on this page (16.5642 smoothed
1842 /// against 17.4412 raw), and no whole
1843 /// number of ink pixels lands between them, so no test at the default can
1844 /// tell the two calibrations apart. At 0.5 — the ratio the reference
1845 /// recommends for wide-spaced layouts, and a constructor argument rather
1846 /// than a default — they are 165.6 (smoothed) and 174.4 (raw). Measured: a
1847 /// faint band of 166 to 174 ink pixels per row is found by the smoothed
1848 /// calibration and missed by the raw one. 170 is the middle of that window.
1849 #[test]
1850 fn the_gap_threshold_is_calibrated_on_the_smoothed_profile() {
1851 let seg = LineSegmenter::with_density_ratio(10, 3, 0.5);
1852 let img = page_with_a_faint_band(8, 12, 30, 170, 20);
1853 let got = seg.segment_image(&img).unwrap();
1854 assert_eq!(
1855 got.len(),
1856 9,
1857 "expected 8 dense bands plus the faint one, got {} — the threshold \
1858 is being calibrated on the raw profile",
1859 got.len()
1860 );
1861 }
1862
1863 // The smoother's own arithmetic, pinned separately from the segmenter that
1864 // reads it. Three of the four bindings diverge from Python here and nothing
1865 // caught it, because no test used an even window.
1866
1867 /// `lead` rows of `ink`, then `gap` zero rows, then `lead` rows of `ink`.
1868 fn banded_profile(lead: usize, gap: usize, ink: f32) -> Vec<f32> {
1869 let mut out = vec![0f32; lead * 2 + gap];
1870 out[..lead].fill(ink);
1871 out[lead + gap..].fill(ink);
1872 out
1873 }
1874
1875 /// THE DIVERGENCE AN ODD-WINDOW-ONLY TEST CANNOT SEE, and the reason it
1876 /// survived four ports: at an odd window this formula and Python's agree.
1877 ///
1878 /// The loop is `[i - half, i + half]` with `half = smooth_window / 2`, so an
1879 /// even window spans one row MORE than asked and is bit-identical to the odd
1880 /// window ABOVE it. A gap of exactly `smooth_window` zero rows therefore
1881 /// still reaches zero at odd windows and does NOT at even ones — which is why
1882 /// the measured break-point table reads 1,3,3,5,5,7,7,9,9,11,11,13 here
1883 /// against Python's 1,2,...,12.
1884 #[test]
1885 fn the_box_spans_one_more_row_than_an_even_window_asks() {
1886 for window in 2u32..=12 {
1887 let span = (2 * (window / 2) + 1) as usize;
1888 let at_span =
1889 LineSegmenter::new(10, window).smooth_histogram(&banded_profile(20, span, 9.0));
1890 let min_in_gap = at_span[20..20 + span]
1891 .iter()
1892 .cloned()
1893 .fold(f32::MAX, f32::min);
1894 assert_eq!(
1895 min_in_gap, 0.0,
1896 "window {window} left no zero row across a gap of {span} rows \
1897 (min {min_in_gap}) — its span is no longer 2 * (window / 2) + 1"
1898 );
1899
1900 let profile = banded_profile(20, span - 1, 9.0);
1901 let under = LineSegmenter::new(10, window).smooth_histogram(&profile);
1902 let min_under = under[20..20 + span - 1]
1903 .iter()
1904 .cloned()
1905 .fold(f32::MAX, f32::min);
1906 assert!(
1907 min_under > 0.0,
1908 "window {window} reached zero across a gap of only {} rows, so the \
1909 box is narrower than measured",
1910 span - 1
1911 );
1912
1913 if window % 2 == 0 {
1914 let odd = LineSegmenter::new(10, window + 1).smooth_histogram(&profile);
1915 assert_eq!(
1916 under,
1917 odd,
1918 "window {window} no longer matches window {} — the even-window \
1919 rounding changed",
1920 window + 1
1921 );
1922 }
1923 }
1924 }
1925
1926 /// Edge handling, and the formula difference against Python and Go.
1927 ///
1928 /// numpy's `mode='same'` zero-pads and divides by the window, so row 0 comes
1929 /// back at `(window / 2 + 1) / window` of the true local mean — 200 not 300 on
1930 /// a flat profile at window 3, 160 not 300 at window 15. Go does the same at
1931 /// odd windows. This port divides by the rows it actually visited and reports
1932 /// 300. Recorded, not reconciled: see `smooth_histogram`'s docs.
1933 ///
1934 /// The first fixture is NOT flat, deliberately. On a flat 300 the answer is 300
1935 /// under this formula AND under no smoothing at all, so a flat profile cannot
1936 /// tell the two apart. Zeroing row 0 gives three different answers: 150 here
1937 /// (the mean of rows 0 and 1), 100 under a window divisor, and 0 under a no-op.
1938 #[test]
1939 fn the_divisor_is_the_rows_visited_so_edge_rows_keep_their_true_mean() {
1940 let mut dip = vec![300f32; 60];
1941 dip[0] = 0.0;
1942 dip[59] = 0.0;
1943 let smoothed = LineSegmenter::new(10, 3).smooth_histogram(&dip);
1944 assert_eq!(
1945 smoothed[0], 150.0,
1946 "row 0 is no longer the mean of the rows actually in range"
1947 );
1948 assert_eq!(smoothed[59], 150.0, "the last row lost the same property");
1949 assert_eq!(
1950 LineSegmenter::new(10, 5).smooth_histogram(&dip)[0],
1951 200.0,
1952 "window 5 row 0 should be 600 over the 3 rows in range, not 900 over 5"
1953 );
1954
1955 let flat = vec![300f32; 60];
1956 for window in [3u32, 5, 15] {
1957 let smoothed = LineSegmenter::new(10, window).smooth_histogram(&flat);
1958 assert_eq!(
1959 smoothed[0], 300.0,
1960 "window {window} attenuated row 0 to {} — the divisor became the \
1961 window rather than the rows visited",
1962 smoothed[0]
1963 );
1964 assert_eq!(
1965 smoothed[59], 300.0,
1966 "window {window} attenuated the last row"
1967 );
1968 }
1969 }
1970
1971 /// Go's even-window defect, asserted absent here.
1972 ///
1973 /// Go sums `2 * (window / 2) + 1` terms and divides by the requested `window`,
1974 /// so at an even window every interior row is inflated by
1975 /// `(window + 1) / window` and the smoothed peak clears the raw one — 1.5x at
1976 /// window 2, 1.25x at window 4. Dividing by what you summed cannot do that,
1977 /// and this pins that it does not.
1978 #[test]
1979 fn smoothing_never_lifts_the_profile_above_its_raw_peak() {
1980 // 20 zero rows, 20 rows of ink, 20 zero rows, so the band is wider than the
1981 // widest span tested and its middle keeps the full 300.
1982 let mut profile = vec![0f32; 60];
1983 profile[20..40].fill(300.0);
1984 for window in 2u32..=12 {
1985 let smoothed = LineSegmenter::new(10, window).smooth_histogram(&profile);
1986 let peak = smoothed.iter().cloned().fold(f32::MIN, f32::max);
1987 // Two-sided, not `<=`. A one-sided bound also passes for a smoother that
1988 // attenuates everything, and for one that does not smooth at all.
1989 assert_eq!(
1990 peak, 300.0,
1991 "window {window} peaked at {peak}, not the raw 300 — the divisor no \
1992 longer equals the row count"
1993 );
1994 // And smoothing really ran: the band's own edge rows are pulled down,
1995 // which a no-op smoother would leave at 300.
1996 assert!(
1997 smoothed[20] < 300.0,
1998 "window {window} left the band's first row at 300 — nothing was \
1999 smoothed"
2000 );
2001 }
2002 }
2003
2004 /// Override for the shared line-merge fixture, for checkouts that do not sit
2005 /// next to the monorepo.
2006 const MERGE_FIXTURE_ENV: &str = "MONOCR_MERGE_FIXTURE";
2007
2008 fn merge_fixture_path() -> PathBuf {
2009 if let Some(path) = std::env::var_os(MERGE_FIXTURE_ENV) {
2010 return PathBuf::from(path);
2011 }
2012 PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2013 .join("../../monocr-monorepo/shared/segmentation-fixtures/merge-cases.json")
2014 }
2015
2016 /// A missing fixture fails loudly, for the same reason `load_rule_fixture`
2017 /// does: skipping would report a green run for a port nothing checked.
2018 fn load_merge_fixture() -> Value {
2019 let path = merge_fixture_path();
2020 let raw = std::fs::read_to_string(&path).unwrap_or_else(|e| {
2021 panic!(
2022 "cannot read the shared line-merge fixture at {}: {e}\n\
2023 set {MERGE_FIXTURE_ENV} to point at \
2024 monocr-monorepo/shared/segmentation-fixtures/merge-cases.json",
2025 path.display()
2026 )
2027 });
2028 serde_json::from_str(&raw)
2029 .unwrap_or_else(|e| panic!("{} is not valid JSON: {e}", path.display()))
2030 }
2031
2032 /// A list of `[start, end]` pairs from a fixture case.
2033 fn merge_pairs(case: &Value, key: &str) -> Vec<(u32, u32)> {
2034 case.get(key)
2035 .and_then(Value::as_array)
2036 .unwrap_or_else(|| panic!("fixture case has no '{key}' array: {case}"))
2037 .iter()
2038 .map(|pair| {
2039 let pair = pair
2040 .as_array()
2041 .unwrap_or_else(|| panic!("'{key}' entry is not a pair: {pair}"));
2042 assert_eq!(pair.len(), 2, "'{key}' entry is not a pair: {pair:?}");
2043 let value = |i: usize| {
2044 pair[i]
2045 .as_u64()
2046 .unwrap_or_else(|| panic!("non-integer in '{key}': {pair:?}"))
2047 as u32
2048 };
2049 (value(0), value(1))
2050 })
2051 .collect()
2052 }
2053
2054 /// The row profile a port must build from the same case description.
2055 ///
2056 /// Fills are applied IN ORDER and overwrite, which is how a one-row
2057 /// sub-threshold dip is written over the band it sits inside. Applying them in
2058 /// any other order gives a different profile and the fixture would not match.
2059 fn merge_profile(case: &Value) -> Vec<f32> {
2060 let length = u32_field(case, "profile_length") as usize;
2061 let mut hist = vec![0f32; length];
2062 for fill in case
2063 .get("profile_fills")
2064 .and_then(Value::as_array)
2065 .unwrap_or_else(|| panic!("fixture case has no 'profile_fills': {case}"))
2066 {
2067 let fill = fill
2068 .as_array()
2069 .unwrap_or_else(|| panic!("'profile_fills' entry is not a triple: {fill}"));
2070 assert_eq!(
2071 fill.len(),
2072 3,
2073 "'profile_fills' entry is not a triple: {fill:?}"
2074 );
2075 let number = |i: usize| {
2076 fill[i]
2077 .as_f64()
2078 .unwrap_or_else(|| panic!("non-number in 'profile_fills': {fill:?}"))
2079 };
2080 let (a, b, value) = (number(0) as usize, number(1) as usize, number(2) as f32);
2081 hist[a..b].fill(value);
2082 }
2083 hist
2084 }
2085
2086 /// The whole `merge_runs` contract, against the oracle the other three ports use.
2087 ///
2088 /// This crate is where the merge was designed, and it is the version the other
2089 /// nine were ported from — which is exactly why it must be checked against an
2090 /// oracle that is NOT itself. The expectations in
2091 /// `shared/segmentation-fixtures/merge-cases.json` are generated by
2092 /// `generate-merge-cases.py`, which reimplements the four decisions from their
2093 /// statement, refuses to write a case that no single-decision mutation kills,
2094 /// and refuses to write anything at all unless the greedy fold agrees with an
2095 /// independent brute-force enumeration of every way to cut the run list into
2096 /// consecutive groups.
2097 ///
2098 /// The four unit tests above are kept as well, not replaced. They are the
2099 /// measured geometry, in comments that record what was measured; the fixture is
2100 /// what stops the ten implementations drifting apart.
2101 #[test]
2102 fn merge_runs_matches_the_shared_fixture() {
2103 let root = load_merge_fixture();
2104
2105 // `MIN_GAP_MERGE` is a constant here and in the other three ports, so pin
2106 // it. `min_line_height` is NOT a constant in this crate — it is a
2107 // constructor argument, see `LineSegmenter::new` — so the fixture's value
2108 // is used per case rather than asserted against a constant that does not
2109 // exist. 10 is what the other three ports compile in and what every test
2110 // in this module builds a segmenter with.
2111 assert_eq!(
2112 root.get("min_gap_merge").and_then(Value::as_u64),
2113 Some(u64::from(MIN_GAP_MERGE)),
2114 "fixture and port disagree on the maximum mergeable gap"
2115 );
2116 assert!(
2117 root.get("mutations")
2118 .and_then(Value::as_object)
2119 .is_some_and(|m| !m.is_empty()),
2120 "the fixture carries no mutation battery, so nothing proves its cases \
2121 discriminate anything"
2122 );
2123
2124 for case in cases(&root, "cases", MERGE_CASES_MIN) {
2125 let name = case
2126 .get("name")
2127 .and_then(Value::as_str)
2128 .unwrap_or("<unnamed>")
2129 .to_string();
2130 let note = case.get("note").and_then(Value::as_str).unwrap_or("");
2131
2132 let hist = merge_profile(&case);
2133 let runs = merge_pairs(&case, "runs");
2134 let expected = merge_pairs(&case, "expected");
2135 let max_gap = u32_field(&case, "max_gap");
2136 let min_line = u32_field(&case, "min_line");
2137
2138 // Exact equality, not a property. Half these cases assert that a merge
2139 // does NOT happen — a speckle chain that must not fuse, two real lines
2140 // that must stay apart — and asserting only the positive is what let
2141 // the speckle-chain mutation survive this crate's own battery once.
2142 assert_eq!(
2143 merge_runs(&runs, &hist, max_gap, min_line),
2144 expected,
2145 "case '{name}': {note}"
2146 );
2147
2148 // A regenerated fixture cannot quietly bring in padding: the generator
2149 // refuses to write a case no mutation kills, and this is the
2150 // consumer-side half of that guard, for a fixture edited by hand.
2151 assert!(
2152 case.get("discriminates")
2153 .and_then(Value::as_array)
2154 .is_some_and(|d| !d.is_empty()),
2155 "case '{name}' discriminates nothing, so it is padding"
2156 );
2157 }
2158 }
2159}