microscope_memory/

build.rs

//! Extracts text from RAW TEXT layer files, constructs a 9-depth block hierarchy
//! (identity → layers → clusters → items → sentences → tokens → syllables → chars → bytes),
//! and writes the binary output files (microscope.bin, data.bin, meta.bin, merkle.bin, embeddings.bin).

use crate::config::Config;
use crate::reader::{read_append_log, FileLock, MicroscopeReader};
use crate::types::BlockHeader;
use crate::{
    content_coords_blended, crc16_ccitt, hex_str, layer_to_id, merkle, safe_truncate, to_block,
    BLOCK_DATA_SIZE, DEPTH_ENTRY_SIZE, HEADER_SIZE, META_HEADER_SIZE,
};

use crate::parallel::*;
use colored::Colorize;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::fs;
use std::io::{BufWriter, Seek, Write};
use std::path::Path;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RebuildOutcome {
    pub rebuilt: bool,
    pub pending_entries: usize,
}

/// Rebuild the main index while holding the same process lock used by stores.
/// The append log is removed only after a successful build.
///
/// `heavy` controls regenerating the expensive deterministic sidecars
/// (embedding vectors + structural fingerprint links). The auto-rebuild path
/// passes `false`: it consolidates the append log fast and drops stale
/// positional sidecars instead of re-serving them after a content shift.
pub fn rebuild_pending(
    config: &Config,
    force_when_empty: bool,
    heavy: bool,
) -> Result<RebuildOutcome, String> {
    let _lock = FileLock::acquire(config)?;
    let output_dir = Path::new(&config.paths.output_dir);
    let append_path = output_dir.join("append.bin");
    let pending_entries = read_append_log(&append_path).len();

    if pending_entries == 0 && !force_when_empty {
        return Ok(RebuildOutcome {
            rebuilt: false,
            pending_entries: 0,
        });
    }

    let previous_blocks = read_meta_block_count(output_dir);

    build(config, true, heavy)?;

    let rebuilt_blocks = read_meta_block_count(output_dir);
    if pending_entries > 0 && previous_blocks > 0 && rebuilt_blocks < previous_blocks {
        return Err(format!(
            "refusing rebuild shrink: {} -> {} blocks",
            previous_blocks, rebuilt_blocks
        ));
    }
    if append_path.exists() {
        fs::remove_file(&append_path).map_err(|e| format!("clear append log: {}", e))?;
        // The append-log vectors are keyed by position in that log, so they die
        // with it: the entries are now main blocks and were embedded by the
        // build above.
        let sidecar = Path::new(&config.paths.output_dir)
            .join(crate::embedding_index::APPEND_EMBEDDINGS_FILE);
        let _ = fs::remove_file(sidecar);
    }

    Ok(RebuildOutcome {
        rebuilt: true,
        pending_entries,
    })
}

/// Consolidate the append log automatically when the configured threshold is
/// reached, or immediately when no main index exists yet.
pub fn maybe_auto_rebuild(config: &Config) -> Result<Option<usize>, String> {
    if !config.index.auto_rebuild {
        return Ok(None);
    }

    let output_dir = Path::new(&config.paths.output_dir);
    let append_path = output_dir.join("append.bin");
    let pending_entries = read_append_log(&append_path).len();
    if pending_entries == 0 {
        return Ok(None);
    }

    let threshold = config.index.auto_rebuild_entries.max(1);
    let index_missing = !output_dir.join("meta.bin").exists();
    if !index_missing && pending_entries < threshold {
        return Ok(None);
    }

    let outcome = rebuild_pending(config, false, false)?;
    Ok(outcome.rebuilt.then_some(outcome.pending_entries))
}

fn read_meta_block_count(output_dir: &Path) -> usize {
    let meta = match fs::read(output_dir.join("meta.bin")) {
        Ok(meta) => meta,
        Err(_) => return 0,
    };
    if meta.len() < 12 {
        return 0;
    }
    u32::from_le_bytes(meta[8..12].try_into().unwrap()) as usize
}

// ─── Internal block for building ─────────────────────
struct RawBlock {
    data: Vec<u8>,
    depth: u8,
    x: f32,
    y: f32,
    z: f32,
    layer_id: u8,
    parent_idx: u32,
    child_count: u16,
    importance: u8,
}

// ─── Extract text values from RAW files ───────────────────
// Zero JSON dependency. Standard UTF-8 text files.
/// Split one layer entry that is longer than BLOCK_DATA_SIZE into pieces that
/// each fit, without losing any bytes.
///
/// `to_block` truncates at BLOCK_DATA_SIZE, so a 3,000-character abstract used
/// to be stored as 1,021 characters plus "...". Everything past that point was
/// unreachable: not retrievable, not embedded, and not in the printed text the
/// evaluation scores. Cutting on sentence boundaries and, failing that, on
/// whitespace, means every character survives and each piece still starts and
/// ends at a natural boundary.
///
/// The last piece may be short. That is fine and intended: merging it into the
/// previous one is exactly the truncation this replaces.
fn split_oversized_entry(line: &str) -> Vec<String> {
    let limit = BLOCK_DATA_SIZE;
    let mut pieces: Vec<String> = Vec::new();
    let mut current = String::new();

    for sentence in split_sentences(line) {
        let sentence = sentence.trim();
        if sentence.is_empty() {
            continue;
        }
        if sentence.len() > limit {
            // One sentence longer than the whole block. Flush what we have and
            // hard-split this one on whitespace.
            if !current.is_empty() {
                pieces.push(std::mem::take(&mut current));
            }
            for word in sentence.split_whitespace() {
                if !current.is_empty() && current.len() + word.len() + 1 > limit {
                    pieces.push(std::mem::take(&mut current));
                }
                if !current.is_empty() {
                    current.push(' ');
                }
                current.push_str(word);
            }
            continue;
        }
        if !current.is_empty() && current.len() + sentence.len() + 2 > limit {
            pieces.push(std::mem::take(&mut current));
        }
        if !current.is_empty() {
            current.push(' ');
        }
        current.push_str(sentence);
    }
    if !current.is_empty() {
        pieces.push(current);
    }
    if pieces.is_empty() {
        pieces.push(line.to_string());
    }
    pieces
}

// Files are read and split into blocks by default.

fn extract_texts_from_file(path: &Path) -> Vec<(String, u8)> {
    let mut texts: Vec<(String, u8)> = Vec::new();
    let raw = match fs::read_to_string(path) {
        Ok(s) => s,
        Err(_) => return texts,
    };

    // Split by double newline or chunking
    for chunk in raw.split("\n\n") {
        let trimmed = chunk.trim();
        if trimmed.len() > 3 {
            let (text, importance) = crate::reader::parse_imp_marker(trimmed);
            texts.push((text.to_string(), importance));
        }
    }

    // Fallback if no doubles: one entry per line.
    //
    // The original bug this replaces: the whole file was cut at an arbitrary
    // byte, so the 60-fact evaluation corpus became a single 1,024-byte block
    // ending "...ends with .hu.\r\nThe user does not hav", and the fragment
    // holding the answer lost the tail the evaluation looks for.
    //
    // The second attempt fixed that by cutting on line boundaries, but it
    // *packed* consecutive lines into one block until the limit was reached.
    // That was invisible at BLOCK_DATA_SIZE = 1024, where a typical entry
    // overflowed the limit on its own, so one line became one block. Raising the
    // limit to 16 KiB exposed it immediately: the same code turned 5,183 SciFact
    // abstracts into 504 blocks of roughly ten documents each, and the
    // evaluation would have measured retrieval over merged documents.
    //
    // A line in a layer file is an entry. Entries are not merged.
    if texts.len() < 2 {
        texts.clear();
        for line in raw.lines() {
            let line = line.trim();
            if line.is_empty() {
                continue;
            }
            if line.len() > BLOCK_DATA_SIZE {
                // One entry larger than a whole block. `to_block` would cut it,
                // losing the tail, so split it on sentence boundaries instead.
                // On BEIR SciFact this used to hit 4,300 of 5,183 abstracts, and
                // the resulting disagreement with the reference embeddings is
                // what the earlier "long stored blocks" investigation chased.
                for piece in split_oversized_entry(line) {
                    let (text, importance) = crate::reader::parse_imp_marker(&piece);
                    if text.trim().len() > 3 {
                        texts.push((text.to_string(), importance));
                    }
                }
            } else {
                let (text, importance) = crate::reader::parse_imp_marker(line);
                if text.trim().len() > 3 {
                    texts.push((text.to_string(), importance));
                }
            }
        }
    }

    texts
}

// ─── Split text into sentences ───────────────────────
fn split_sentences(text: &str) -> Vec<String> {
    let mut sentences = Vec::new();
    let mut current = String::new();
    let chars: Vec<char> = text.chars().collect();
    for (i, &ch) in chars.iter().enumerate() {
        current.push(ch);
        if !matches!(ch, '.' | '!' | '?' | '\n') || current.trim().len() <= 10 {
            continue;
        }
        // A period inside a token is not a sentence end. "ends with .hu." and
        // "3.5" and "e.g." were being cut there, and the fragment that survived
        // lost the very token a search would look for: the 60-fact corpus
        // stored "The user's email address ends with ." with ".hu" split off
        // into its own block. A terminator only counts when what follows is
        // the end of the text, whitespace, or a capital -- which keeps
        // sentence-initial capitals working without breaking abbreviations.
        let next = chars.get(i + 1).copied();
        let ends_here = match next {
            None => true,
            Some(c) => c.is_whitespace() || c.is_uppercase(),
        };
        if ends_here {
            sentences.push(current.trim().to_string());
            current = String::new();
        }
    }
    if current.trim().len() > 5 {
        sentences.push(current.trim().to_string());
    }
    sentences
}

// ─── Compute deterministic SHA-256 hash of all layer source files ────
pub fn compute_layers_hash(config: &Config) -> [u8; 32] {
    let layers_dir = Path::new(&config.paths.layers_dir);
    let layer_files = &config.memory_layers.layers;
    let mut sorted_names: Vec<&String> = layer_files.iter().collect();
    sorted_names.sort();
    let mut hasher = Sha256::new();
    for name in &sorted_names {
        let path = layers_dir.join(format!("{}.txt", name));
        if let Ok(contents) = fs::read(&path) {
            hasher.update(&contents);
        }
    }
    let result = hasher.finalize();
    let mut hash = [0u8; 32];
    hash.copy_from_slice(&result);
    hash
}

// ─── BUILD: layers/ → binary ─────────────────────────
/// Rebuild the binary index from the raw layer files.
///
/// `heavy` regenerates the expensive deterministic sidecars (embeddings +
/// fingerprint links). Auto-rebuild passes `false` and drops stale copies so
/// positional vectors/links are never served with wrong block indices.
pub fn build(config: &Config, force: bool, heavy: bool) -> Result<(), String> {
    let layers_hash = compute_layers_hash(config);

    // Incremental build check — skip if layers unchanged
    if !force {
        let output_dir = Path::new(&config.paths.output_dir);
        let meta_path = output_dir.join("meta.bin");
        if let Ok(meta) = fs::read(&meta_path) {
            if meta.len() >= 152 && (&meta[0..4] == b"MSC3" || &meta[0..4] == b"MSC4") {
                let stored_hash = &meta[120..152];
                if stored_hash == &layers_hash[..] {
                    println!("{}", "Layers unchanged — skipping rebuild".green().bold());
                    return Ok(());
                }
            }
        }
    }

    println!(
        "{}",
        "Building microscope from raw layers (zero JSON)..."
            .cyan()
            .bold()
    );

    let layers_dir = Path::new(&config.paths.layers_dir);
    let output_dir = Path::new(&config.paths.output_dir);

    // Load evidence ledger for flags stamping (epistemic class).
    let ledger = crate::epistemic::EvidenceLedger::load_or_init(output_dir);

    // Capture the pre-rebuild index for content-based state remapping. Rebuild
    // re-derives block positions from layer files, so positional sidecars
    // (Hebbian, spaced repetition, archetypes, patterns, eureka) must be
    // remapped by matching old block content to the new layout — not by the
    // depth sort order, which is a no-op because blocks are already
    // depth-ordered.
    let old_snapshot = capture_old_snapshot(config);
    let layers_changed = read_stored_layers_hash(output_dir)
        .map(|stored| stored != layers_hash)
        .unwrap_or(true);

    if !output_dir.exists() {
        fs::create_dir_all(output_dir).map_err(|e| format!("create output dir: {}", e))?;
    }

    let layer_files = &config.memory_layers.layers;

    // Collect all raw texts per layer
    let mut layer_texts: Vec<(String, Vec<(String, u8)>)> = Vec::new();
    for name in layer_files {
        let path = layers_dir.join(format!("{}.txt", name));
        let texts = extract_texts_from_file(&path);
        println!("  {} {}: {} items", ">".green(), name, texts.len());
        layer_texts.push((name.clone(), texts));
    }

    let mut blocks: Vec<RawBlock> = Vec::new();

    // ═══ DEPTH 0: Identity ═══
    let identity = "Microscope Memory: 9-depth hierarchical cognitive engine. Binary mmap, sub-microsecond spatial search, Hebbian learning, Merkle integrity.";
    blocks.push(RawBlock {
        data: to_block(identity),
        depth: 0,
        x: 0.25,
        y: 0.25,
        z: 0.25,
        layer_id: 0,
        parent_idx: u32::MAX,
        child_count: layer_files.len() as u16,
        importance: 8,
    });

    // ═══ DEPTH 1: Layer summaries ═══
    let sw = config.search.semantic_weight;
    let depth1_start = blocks.len();
    for (name, texts) in &layer_texts {
        let preview: Vec<String> = texts
            .iter()
            .take(3)
            .map(|(t, _)| safe_truncate(t, 40))
            .collect();
        let summary = format!("[{}] {} elem. {}", name, texts.len(), preview.join(" | "));
        let (x, y, z) = content_coords_blended(name, name, sw);
        blocks.push(RawBlock {
            data: to_block(&summary),
            depth: 1,
            x,
            y,
            z,
            layer_id: layer_to_id(name),
            parent_idx: 0,
            child_count: texts.len().div_ceil(5) as u16, // cluster count
            importance: 5,
        });
    }

    // ═══ DEPTH 2: Clusters (5 items each) ═══
    let _depth2_start = blocks.len();
    let mut depth2_layer_offsets: Vec<(usize, usize)> = Vec::new(); // (start_in_blocks, count)
    for (li, (name, texts)) in layer_texts.iter().enumerate() {
        let cluster_start = blocks.len();
        for ci in (0..texts.len()).step_by(5) {
            let chunk: Vec<String> = texts[ci..texts.len().min(ci + 5)]
                .iter()
                .map(|(t, _)| safe_truncate(t, 40))
                .collect();
            let summary = format!("[{} #{}] {}", name, ci / 5, chunk.join(" | "));
            let (x, y, z) = content_coords_blended(&summary, name, sw);
            blocks.push(RawBlock {
                data: to_block(&summary),
                depth: 2,
                x,
                y,
                z,
                layer_id: layer_to_id(name),
                parent_idx: (depth1_start + li) as u32,
                child_count: chunk.len() as u16,
                importance: 5,
            });
        }
        depth2_layer_offsets.push((cluster_start, blocks.len() - cluster_start));
    }

    // ═══ DEPTH 3: Individual items ═══
    let depth3_start = blocks.len();
    let mut depth3_positions: Vec<(f32, f32, f32)> = Vec::new();
    for (li, (name, texts)) in layer_texts.iter().enumerate() {
        for (ti, (text, item_importance)) in texts.iter().enumerate() {
            let (x, y, z) = content_coords_blended(text, name, sw);
            let cluster_idx = ti / 5;
            let (d2_start, d2_count) = depth2_layer_offsets[li];
            let parent = if cluster_idx < d2_count {
                (d2_start + cluster_idx) as u32
            } else {
                u32::MAX
            };

            blocks.push(RawBlock {
                data: to_block(text),
                depth: 3,
                x,
                y,
                z,
                layer_id: layer_to_id(name),
                parent_idx: parent,
                child_count: 0, // will update
                importance: *item_importance,
            });
            depth3_positions.push((x, y, z));
        }
    }

    // ═══ DEPTH 4: Sentences ═══
    let _depth4_start = blocks.len();
    let mut depth4_parents: Vec<usize> = Vec::new();

    let d4_results: Vec<Vec<RawBlock>> = (depth3_start..(depth3_start + depth3_positions.len()))
        .into_par_iter()
        .map(|d3i| {
            let text = std::str::from_utf8(&blocks[d3i].data).unwrap_or("");
            let sentences = split_sentences(text);
            let mut local_blocks = Vec::new();
            for sent in &sentences {
                if sent.len() < 10 {
                    continue;
                }
                let (px, py, pz) = depth3_positions[d3i - depth3_start];
                let h = sent
                    .as_bytes()
                    .iter()
                    .fold(0u64, |a, &b| a.wrapping_mul(31).wrapping_add(b as u64));
                let ox = ((h & 0xFF) as f32 - 128.0) / 25500.0;
                let oy = (((h >> 8) & 0xFF) as f32 - 128.0) / 25500.0;
                let oz = (((h >> 16) & 0xFF) as f32 - 128.0) / 25500.0;

                local_blocks.push(RawBlock {
                    data: to_block(sent),
                    depth: 4,
                    x: px + ox,
                    y: py + oy,
                    z: pz + oz,
                    layer_id: blocks[d3i].layer_id,
                    parent_idx: d3i as u32,
                    child_count: 0,
                    importance: 5,
                });
            }
            local_blocks
        })
        .collect();

    for (i, local) in d4_results.into_iter().enumerate() {
        let d3i = depth3_start + i;
        blocks[d3i].child_count = local.len() as u16;
        for b in local {
            blocks.push(b);
            depth4_parents.push(blocks.len() - 1);
        }
    }

    // ═══ DEPTH 5: Tokens (words) ═══
    let mut depth5_parents: Vec<usize> = Vec::new();
    let depth4_parents_clone = depth4_parents.clone();
    let d5_results: Vec<Vec<RawBlock>> = depth4_parents
        .into_par_iter()
        .map(|d4i| {
            let text_owned = String::from_utf8_lossy(&blocks[d4i].data).to_string();
            let px = blocks[d4i].x;
            let py = blocks[d4i].y;
            let pz = blocks[d4i].z;
            let lid = blocks[d4i].layer_id;

            let tokens: Vec<String> = text_owned
                .split_whitespace()
                .take(8)
                .map(|s| s.to_string())
                .collect();
            let mut local_blocks = Vec::new();
            for tok in &tokens {
                if tok.len() < 2 {
                    continue;
                }
                let h = tok
                    .as_bytes()
                    .iter()
                    .fold(0u64, |a, &b| a.wrapping_mul(31).wrapping_add(b as u64));
                let ox = ((h & 0xFF) as f32 - 128.0) / 255000.0;
                let oy = (((h >> 8) & 0xFF) as f32 - 128.0) / 255000.0;
                let oz = (((h >> 16) & 0xFF) as f32 - 128.0) / 255000.0;

                local_blocks.push(RawBlock {
                    data: to_block(tok),
                    depth: 5,
                    x: px + ox,
                    y: py + oy,
                    z: pz + oz,
                    layer_id: lid,
                    parent_idx: d4i as u32,
                    child_count: 0,
                    importance: 5,
                });
            }
            local_blocks
        })
        .collect();

    for (i, local) in d5_results.into_iter().enumerate() {
        let d4i = depth4_parents_clone[i];
        blocks[d4i].child_count = local.len() as u16;
        for b in local {
            blocks.push(b);
            depth5_parents.push(blocks.len() - 1);
        }
    }

    // ═══ DEPTH 6: Syllables / morphemes (sub-word) ═══
    let mut depth6_parents: Vec<usize> = Vec::new();
    let d6_results: Vec<Vec<RawBlock>> = depth5_parents
        .clone()
        .into_par_iter()
        .map(|d5i| {
            let text_owned = String::from_utf8_lossy(&blocks[d5i].data).to_string();
            let px = blocks[d5i].x;
            let py = blocks[d5i].y;
            let pz = blocks[d5i].z;
            let lid = blocks[d5i].layer_id;

            let chars: Vec<char> = text_owned.chars().collect();
            if chars.len() < 3 {
                return vec![];
            }
            let chunk_size = 3.max(chars.len() / 3).min(5);
            let mut local_blocks = Vec::new();
            for chunk in chars.chunks(chunk_size) {
                let syl: String = chunk.iter().collect();
                if syl.trim().is_empty() {
                    continue;
                }
                let h = syl
                    .as_bytes()
                    .iter()
                    .fold(0u64, |a, &b| a.wrapping_mul(37).wrapping_add(b as u64));
                let ox = ((h & 0xFF) as f32 - 128.0) / 2550000.0;
                let oy = (((h >> 8) & 0xFF) as f32 - 128.0) / 2550000.0;
                let oz = (((h >> 16) & 0xFF) as f32 - 128.0) / 2550000.0;

                local_blocks.push(RawBlock {
                    data: to_block(&syl),
                    depth: 6,
                    x: px + ox,
                    y: py + oy,
                    z: pz + oz,
                    layer_id: lid,
                    parent_idx: d5i as u32,
                    child_count: 0,
                    importance: 5,
                });
            }
            local_blocks
        })
        .collect();

    for (i, local) in d6_results.into_iter().enumerate() {
        let d5i = depth5_parents[i];
        blocks[d5i].child_count = local.len() as u16;
        for b in local {
            blocks.push(b);
            depth6_parents.push(blocks.len() - 1);
        }
    }

    // ═══ DEPTH 7: Characters ═══
    let mut depth7_parents: Vec<usize> = Vec::new();
    let d7_results: Vec<Vec<RawBlock>> = depth6_parents
        .clone()
        .into_par_iter()
        .map(|d6i| {
            let text_owned = String::from_utf8_lossy(&blocks[d6i].data).to_string();
            let px = blocks[d6i].x;
            let py = blocks[d6i].y;
            let pz = blocks[d6i].z;
            let lid = blocks[d6i].layer_id;

            let mut local_blocks = Vec::new();
            for ch in text_owned.chars() {
                if ch.is_whitespace() {
                    continue;
                }
                let h = (ch as u64).wrapping_mul(0x517cc1b727220a95);
                let ox = ((h & 0xFF) as f32 - 128.0) / 25500000.0;
                let oy = (((h >> 8) & 0xFF) as f32 - 128.0) / 25500000.0;
                let oz = (((h >> 16) & 0xFF) as f32 - 128.0) / 25500000.0;

                let ch_str = ch.to_string();
                local_blocks.push(RawBlock {
                    data: to_block(&ch_str),
                    depth: 7,
                    x: px + ox,
                    y: py + oy,
                    z: pz + oz,
                    layer_id: lid,
                    parent_idx: d6i as u32,
                    child_count: 0,
                    importance: 5,
                });
            }
            local_blocks
        })
        .collect();

    for (i, local) in d7_results.into_iter().enumerate() {
        let d6i = depth6_parents[i];
        blocks[d6i].child_count = local.len() as u16;
        for b in local {
            blocks.push(b);
            depth7_parents.push(blocks.len() - 1);
        }
    }

    // ═══ DEPTH 8: Raw bytes — the atomic level. Below this, data corrupts. ═══
    let d8_results: Vec<Vec<RawBlock>> = depth7_parents
        .clone()
        .into_par_iter()
        .map(|d7i| {
            let text_owned = String::from_utf8_lossy(&blocks[d7i].data).to_string();
            let px = blocks[d7i].x;
            let py = blocks[d7i].y;
            let pz = blocks[d7i].z;
            let lid = blocks[d7i].layer_id;

            let bytes = text_owned.as_bytes();
            let mut local_blocks = Vec::new();
            for &byte in bytes {
                let hex = format!("0x{:02X}", byte);
                let h = (byte as u64).wrapping_mul(0x9E3779B97F4A7C15);
                let ox = ((h & 0xFF) as f32 - 128.0) / 255000000.0;
                let oy = (((h >> 8) & 0xFF) as f32 - 128.0) / 255000000.0;
                let oz = (((h >> 16) & 0xFF) as f32 - 128.0) / 255000000.0;

                local_blocks.push(RawBlock {
                    data: to_block(&hex),
                    depth: 8,
                    x: px + ox,
                    y: py + oy,
                    z: pz + oz,
                    layer_id: lid,
                    parent_idx: d7i as u32,
                    child_count: 0, // LEAF. Below = corruption.
                    importance: 5,
                });
            }
            local_blocks
        })
        .collect();

    for (i, local) in d8_results.into_iter().enumerate() {
        let d7i = depth7_parents[i];
        blocks[d7i].child_count = local.len() as u16;
        for b in local {
            blocks.push(b);
        }
    }

    let n = blocks.len();
    println!("\n  {} blocks total", n);

    // Sort by depth (a stable no-op in practice — blocks are built in depth
    // order, so this leaves the layout untouched).
    let mut indices: Vec<usize> = (0..n).collect();
    indices.sort_by_key(|&i| blocks[i].depth);

    // Remap parent indices after the (no-op) sort — only used for header wiring.
    let mut sort_old_to_new = vec![0u32; n];
    for (new_i, &old_i) in indices.iter().enumerate() {
        sort_old_to_new[old_i] = new_i as u32;
    }

    // Content-based mapping from the PREVIOUS index layout to the new one.
    // Sidecar states reference blocks by position; positions shift whenever
    // layer entries are truncated (retention) or edited, so we match old block
    // content (depth + layer + bytes) to the freshly built layout. Blocks that
    // no longer exist map to u32::MAX and are dropped by the remap routines.
    let content_old_to_new = compute_content_remap(&old_snapshot, &blocks);

    // Write binary files
    let output_dir = Path::new(&config.paths.output_dir);
    fs::create_dir_all(output_dir).ok();

    let hdr_path = output_dir.join("microscope.bin");
    let dat_path = output_dir.join("data.bin");
    let meta_path = output_dir.join("meta.bin");

    let mut hdr_file = BufWriter::new(
        fs::File::create(&hdr_path).map_err(|e| format!("create microscope.bin: {}", e))?,
    );
    let mut dat_file =
        BufWriter::new(fs::File::create(&dat_path).map_err(|e| format!("create data.bin: {}", e))?);

    let mut depth_ranges: Vec<(u32, u32)> = vec![(0, 0); 9];
    let mut cur_depth: u8 = 0;
    let mut range_start: u32 = 0;

    for (new_i, &old_i) in indices.iter().enumerate() {
        let b = &blocks[old_i];
        let offset = dat_file
            .stream_position()
            .map_err(|e| format!("data.bin stream_position: {}", e))? as u32;
        let len = b.data.len().min(BLOCK_DATA_SIZE) as u16;
        dat_file
            .write_all(&b.data[..len as usize])
            .map_err(|e| format!("write data.bin: {}", e))?;

        let parent = if b.parent_idx == u32::MAX {
            u32::MAX
        } else {
            sort_old_to_new[b.parent_idx as usize]
        };

        let crc = crc16_ccitt(&b.data[..len as usize]);
        let hdr = BlockHeader {
            x: b.x,
            y: b.y,
            z: b.z,
            zoom: b.depth as f32 / 8.0,
            depth: b.depth,
            layer_id: b.layer_id,
            data_offset: offset,
            data_len: len,
            parent_idx: parent,
            child_count: b.child_count,
            crc16: crc.to_le_bytes(),
            project_id: config.project_id,
            importance: b.importance,
            flags: {
                // Stamp epistemic class from evidence ledger into flags bits 0-2.
                let text_str = std::str::from_utf8(&b.data[..len as usize])
                    .unwrap_or("")
                    .trim_end_matches('\0')
                    .trim();
                let ch = crate::epistemic::content_hash(text_str);
                match ledger.records.get(&ch) {
                    Some(rec) => rec.class as u8 & 0x07,
                    None => 0,
                }
            },
        };

        let bytes: &[u8] = unsafe {
            std::slice::from_raw_parts(&hdr as *const BlockHeader as *const u8, HEADER_SIZE)
        };
        hdr_file
            .write_all(bytes)
            .map_err(|e| format!("write microscope.bin: {}", e))?;

        // Track depth ranges
        if b.depth != cur_depth {
            depth_ranges[cur_depth as usize] = (range_start, new_i as u32 - range_start);
            range_start = new_i as u32;
            cur_depth = b.depth;
        }
    }
    depth_ranges[cur_depth as usize] = (range_start, n as u32 - range_start);
    hdr_file
        .flush()
        .map_err(|e| format!("flush microscope.bin: {}", e))?;
    dat_file
        .flush()
        .map_err(|e| format!("flush data.bin: {}", e))?;

    // ═══ Optional zstd compression of data.bin ═══
    #[cfg(feature = "compression")]
    if config.performance.compression {
        let raw_data =
            fs::read(&dat_path).map_err(|e| format!("read data.bin for compression: {}", e))?;
        let raw_size = raw_data.len();
        let compressed = zstd::encode_all(std::io::Cursor::new(&raw_data), 3)
            .map_err(|e| format!("zstd compress: {}", e))?;
        let comp_size = compressed.len();
        let zst_path = output_dir.join("data.bin.zst");
        fs::write(&zst_path, &compressed).map_err(|e| format!("write data.bin.zst: {}", e))?;
        let ratio = if comp_size > 0 {
            raw_size as f64 / comp_size as f64
        } else {
            0.0
        };
        println!(
            "  {}: {} → {} bytes ({:.1}x ratio)",
            "zstd".green(),
            raw_size,
            comp_size,
            ratio,
        );
    }

    // ═══ Merkle tree: SHA-256 over all block data ═══
    let merkle_path = output_dir.join("merkle.bin");
    // Re-read data.bin to get all block data slices for Merkle leaves
    hdr_file
        .flush()
        .map_err(|e| format!("flush microscope.bin: {}", e))?;
    dat_file
        .flush()
        .map_err(|e| format!("flush data.bin: {}", e))?;

    let dat_bytes = fs::read(&dat_path).map_err(|e| format!("read data.bin for merkle: {}", e))?;
    let hdr_bytes =
        fs::read(&hdr_path).map_err(|e| format!("read microscope.bin for merkle: {}", e))?;
    let mut leaf_slices: Vec<&[u8]> = Vec::with_capacity(n);
    for i in 0..n {
        let hdr_off = i * HEADER_SIZE;
        let data_offset =
            u32::from_le_bytes(hdr_bytes[hdr_off + 18..hdr_off + 22].try_into().unwrap()) as usize;
        let data_len =
            u16::from_le_bytes(hdr_bytes[hdr_off + 22..hdr_off + 24].try_into().unwrap()) as usize;
        if data_offset + data_len <= dat_bytes.len() {
            leaf_slices.push(&dat_bytes[data_offset..data_offset + data_len]);
        } else {
            leaf_slices.push(&[]);
        }
    }

    let merkle_tree = merkle::MerkleTree::build(&leaf_slices);
    let merkle_tmp = merkle_path.with_extension("bin.tmp");
    fs::write(&merkle_tmp, merkle_tree.to_bytes())
        .map_err(|e| format!("write merkle.bin: {}", e))?;
    fs::rename(&merkle_tmp, &merkle_path).map_err(|e| format!("rename merkle.bin: {}", e))?;
    println!(
        "  {}: {} leaves, root={}",
        "merkle".green(),
        merkle_tree.leaf_count,
        hex_str(&merkle_tree.root)
    );

    // meta.bin — MSC4 format with merkle root + layers hash
    let mut meta_buf = Vec::with_capacity(META_HEADER_SIZE + 9 * DEPTH_ENTRY_SIZE + 32 + 32);
    meta_buf.extend_from_slice(b"MSC4"); // magic v4
    meta_buf.extend_from_slice(&4u32.to_le_bytes()); // version
    meta_buf.extend_from_slice(&(n as u32).to_le_bytes()); // block_count
    meta_buf.extend_from_slice(&9u32.to_le_bytes()); // depth_count
    for &(start, count) in &depth_ranges {
        meta_buf.extend_from_slice(&start.to_le_bytes());
        meta_buf.extend_from_slice(&count.to_le_bytes());
    }
    meta_buf.extend_from_slice(&merkle_tree.root); // 32 bytes merkle root
    meta_buf.extend_from_slice(&layers_hash); // 32 bytes layers content hash
    let meta_tmp = meta_path.with_extension("bin.tmp");
    fs::write(&meta_tmp, &meta_buf).map_err(|e| format!("write meta.bin: {}", e))?;
    fs::rename(&meta_tmp, meta_path).map_err(|e| format!("rename meta.bin: {}", e))?;

    // Report
    let hdr_size = n * HEADER_SIZE;
    let dat_size = dat_file.stream_position().unwrap_or(0) as usize; // Get final data size
    let meta_size = meta_buf.len();
    println!(
        "\n  {}: {} bytes ({:.1} KB)",
        "headers".green(),
        hdr_size,
        hdr_size as f64 / 1024.0
    );
    println!(
        "  {}:    {} bytes ({:.1} KB)",
        "data".green(),
        dat_size,
        dat_size as f64 / 1024.0
    );
    println!("  {}:    {} bytes", "meta".green(), meta_size);
    println!(
        "  {}:   {:.1} KB",
        "TOTAL".yellow().bold(),
        (hdr_size + dat_size + meta_size) as f64 / 1024.0
    );

    let fits = if hdr_size < 32768 {
        "L1d (32KB)"
    } else if hdr_size < 262144 {
        "L2 (256KB)"
    } else {
        "L3"
    };
    println!("  cache:   {}", fits.green().bold());

    for (d, &(_start, count)) in depth_ranges.iter().enumerate() {
        println!("  Depth {}: {:>5} blocks", d, count);
    }

    // ═══ Embedding index + structural fingerprinting (heavy sidecars) ═══
    // Regenerated only on explicit full builds. The light (auto) rebuild path
    // drops stale copies instead of serving positional vectors/links that no
    // longer match the rebuilt block layout.
    if heavy {
        // ═══ Embedding index (mock provider, or candle if enabled) ═══
        if config.embedding.provider != "none" {
            println!("\n  Building embedding index...");
            let emb_path = output_dir.join("embeddings.bin");
            let reader = MicroscopeReader::open(config)?;
            let max_depth = config.embedding.max_depth;

            let provider: Box<dyn crate::embeddings::EmbeddingProvider> =
                crate::embeddings::provider_from_config(&config.embedding, config.embedding.dim);

            match crate::embedding_index::build_embedding_index(
                &*provider, &reader, max_depth, &emb_path,
            ) {
                Ok(()) => println!("  {} embeddings.bin built", "OK".green()),
                Err(e) => eprintln!("  {} embedding build: {}", "ERR".red(), e),
            }
        }

        // ═══ Inverted text index (fast find_text / recall prefilter) ═══
        let text_index_result = (|| -> Result<(), String> {
            let reader = MicroscopeReader::open(config)?;
            crate::text_index::build_text_index(
                (0..reader.block_count).map(|i| reader.text(i)),
                reader.block_count,
                &output_dir.join("text_index.bin"),
            )
        })();
        match text_index_result {
            Ok(()) => println!("  {} text_index.bin built", "OK".green()),
            Err(e) => eprintln!("  {} text index build: {}", "ERR".red(), e),
        }

        // ═══ Hebbian delta integration ═══
        let hebb_path = output_dir.join("activations.bin");
        if hebb_path.exists() {
            let hebb = crate::hebbian::HebbianState::load_or_init(output_dir, n);
            let drifted = hebb
                .activations
                .iter()
                .filter(|r| {
                    r.drift_x.abs() > 0.001 || r.drift_y.abs() > 0.001 || r.drift_z.abs() > 0.001
                })
                .count();

            if drifted > 0 {
                apply_hebbian_deltas(output_dir, &hebb, n)?;
                println!(
                    "  {} Hebbian deltas applied to {} blocks",
                    "HEBBIAN".magenta(),
                    drifted
                );
            }
        }

        // ═══ Structural fingerprinting ═══
        {
            let reader = MicroscopeReader::open(config)?;
            let texts: Vec<&str> = (0..reader.block_count).map(|i| reader.text(i)).collect();
            let table = crate::fingerprint::LinkTable::build(&texts);
            table.save(output_dir)?;
            let stats = table.stats();
            println!(
                "  {} {} links across {} blocks",
                "FINGERPRINT".cyan(),
                stats.link_count,
                stats.block_count
            );
        }
    } else if layers_changed {
        // Light rebuild with content change: stale *positional* sidecars must
        // not be served — drop them and rebuild on the next explicit `build`.
        //
        // `text_index.bin` is in this list on purpose: its postings are block
        // ids, and a rebuild reassigns those, so the file left over from the
        // previous layout would answer queries with the wrong blocks. The index
        // is cheap to regenerate (a heavy build takes milliseconds), but this
        // path deliberately does not spend them; `tests/integration.rs::
        // test_light_rebuild_drops_stale_text_index` pins the behaviour.
        //
        // Consequence for the recall speedup: after a light rebuild the reader
        // falls back to a full scan until the next heavy build, or until the
        // index is regenerated by hand (the `index_text` example).
        for name in [
            "embeddings.bin",
            "links.bin",
            "fingerprints.idx",
            "text_index.bin",
        ] {
            let p = output_dir.join(name);
            if p.exists() {
                let _ = fs::remove_file(&p);
            }
        }
        println!(
            "  {} heavy sidecars skipped (embeddings, fingerprint); stale copies removed",
            "LIGHT".cyan()
        );
    } else {
        println!(
            "  {} layers unchanged — heavy sidecars kept",
            "LIGHT".cyan()
        );
    }

    // ═══ Emotion log → emotions.bin ═══
    {
        let reader = MicroscopeReader::open(config)?;
        crate::reader::build_emotions_from_log(output_dir, &reader)?;
    }

    // ═══ Remap cognitive state after the layout change ═══
    // Rebuild re-derives block positions from layer files; every positional
    // sidecar must be remapped through the content-based mapping so learned
    // state keeps pointing at the right blocks after a retention shift or an
    // edit. (MM-001 only covered Hebbian/PredictiveCache/Mirror and used the
    // depth-sort mapping, which is the identity — the fix here matches content.)
    {
        let mut hebb = crate::hebbian::HebbianState::load_or_init(output_dir, n);
        hebb.remap_indexes(&content_old_to_new);
        // The remap sizes the vector to the highest surviving block; pad back
        // to the full new block count so every block has a record again.
        if hebb.activations.len() < n {
            hebb.activations
                .resize(n, crate::hebbian::ActivationRecord::default());
        }
        hebb.save(output_dir)
            .map_err(|e| format!("save hebbian after remap: {e}"))?;
        println!("  {} Hebbian state remapped (content)", "REMAP".magenta());
    }
    {
        let mut pc = crate::predictive_cache::PredictiveCache::load_or_init(output_dir);
        pc.remap_indexes(&content_old_to_new);
        pc.save(output_dir)
            .map_err(|e| format!("save predictive_cache after remap: {e}"))?;
        println!("  {} PredictiveCache remapped", "REMAP".magenta());
    }
    {
        let mut mirror = crate::mirror::MirrorState::load_or_init(output_dir);
        mirror.remap_indexes(&content_old_to_new);
        mirror
            .save(output_dir)
            .map_err(|e| format!("save mirror after remap: {e}"))?;
        println!("  {} MirrorState remapped", "REMAP".magenta());
    }
    {
        let mut spaced = crate::spaced_repetition::SpacedRepetition::load_or_init(output_dir);
        spaced.remap_indexes(&content_old_to_new);
        spaced
            .save(output_dir)
            .map_err(|e| format!("save spaced after remap: {e}"))?;
        println!("  {} SpacedRepetition remapped", "REMAP".magenta());
    }
    {
        let mut archetypes = crate::archetype::ArchetypeState::load_or_init(output_dir);
        archetypes.remap_indexes(&content_old_to_new);
        archetypes
            .save(output_dir)
            .map_err(|e| format!("save archetypes after remap: {e}"))?;
        println!("  {} Archetypes remapped", "REMAP".magenta());
    }
    {
        let mut tg = crate::thought_graph::ThoughtGraphState::load_or_init(output_dir);
        tg.remap_indexes(&content_old_to_new);
        tg.save(output_dir)
            .map_err(|e| format!("save thought_graph after remap: {e}"))?;
        println!("  {} ThoughtPatterns remapped", "REMAP".magenta());
    }
    {
        let mut eureka = crate::eureka::EurekaLog::load_or_init(output_dir);
        eureka.remap_indexes(&content_old_to_new);
        eureka
            .save(output_dir)
            .map_err(|e| format!("save eureka after remap: {e}"))?;
        println!("  {} EurekaLog remapped", "REMAP".magenta());
    }

    // ═══ Text index: regenerate when missing (light path dropped it) ═══
    // The layout is final here (all remaps above are done), so a fresh index
    // is never stale. The heavy path already built it, so this is a no-op
    // there; the light path dropped it as a stale positional sidecar, and
    // this restores the indexed recall path instead of leaving every query
    // on a full scan until the next heavy build. Content-derived and cheap
    // (milliseconds for tens of thousands of blocks), unlike the embedding
    // index which stays dropped by policy.
    {
        let tix_path = output_dir.join("text_index.bin");
        if !tix_path.exists() {
            let tix_result = (|| -> Result<(), String> {
                let reader = MicroscopeReader::open(config)?;
                crate::text_index::build_text_index(
                    (0..reader.block_count).map(|i| reader.text(i)),
                    reader.block_count,
                    &tix_path,
                )
            })();
            match tix_result {
                Ok(()) => println!("  {} text_index.bin regenerated", "OK".green()),
                Err(e) => eprintln!("  {} text index regenerate: {}", "ERR".red(), e),
            }
        }
    }

    println!("\n{}", "ZERO JSON. Pure binary. Done.".green().bold());
    Ok(())
}

/// Post-process: apply Hebbian drift deltas to microscope.bin header coordinates.
fn apply_hebbian_deltas(
    output_dir: &Path,
    hebb: &crate::hebbian::HebbianState,
    block_count: usize,
) -> Result<(), String> {
    let hdr_path = output_dir.join("microscope.bin");
    let mut data = fs::read(&hdr_path).map_err(|e| format!("read microscope.bin: {}", e))?;

    for i in 0..block_count.min(hebb.activations.len()) {
        let rec = &hebb.activations[i];
        if rec.drift_x.abs() < 0.001 && rec.drift_y.abs() < 0.001 && rec.drift_z.abs() < 0.001 {
            continue;
        }

        let off = i * HEADER_SIZE;
        if off + 12 > data.len() {
            break;
        }

        // Read current x, y, z (first 12 bytes of header, 3×f32 LE)
        let x = f32::from_le_bytes(data[off..off + 4].try_into().unwrap());
        let y = f32::from_le_bytes(data[off + 4..off + 8].try_into().unwrap());
        let z = f32::from_le_bytes(data[off + 8..off + 12].try_into().unwrap());

        // Apply drift
        let new_x = x + rec.drift_x;
        let new_y = y + rec.drift_y;
        let new_z = z + rec.drift_z;

        data[off..off + 4].copy_from_slice(&new_x.to_le_bytes());
        data[off + 4..off + 8].copy_from_slice(&new_y.to_le_bytes());
        data[off + 8..off + 12].copy_from_slice(&new_z.to_le_bytes());
    }

    let hdr_tmp = hdr_path.with_extension("bin.tmp");
    fs::write(&hdr_tmp, &data).map_err(|e| format!("write microscope.bin: {}", e))?;
    fs::rename(&hdr_tmp, &hdr_path).map_err(|e| format!("rename microscope.bin: {}", e))?;

    // Clear drift values after integration (they're now baked in)
    let mut hebb_clone = crate::hebbian::HebbianState {
        activations: hebb.activations.clone(),
        coactivations: hebb.coactivations.clone(),
        fingerprints: hebb.fingerprints.clone(),
    };
    for rec in &mut hebb_clone.activations {
        rec.drift_x = 0.0;
        rec.drift_y = 0.0;
        rec.drift_z = 0.0;
    }
    hebb_clone
        .save(output_dir)
        .map_err(|e| format!("save cleared Hebbian: {}", e))
}

/// Snapshot of one block from the pre-rebuild index, used to remap positional
/// sidecar state by content after a rebuild changes block positions.
struct OldBlockSnapshot {
    depth: u8,
    layer_id: u8,
    text: Vec<u8>,
}

/// Read every block of the current index into memory before it is overwritten.
fn capture_old_snapshot(config: &Config) -> Option<Vec<OldBlockSnapshot>> {
    let reader = MicroscopeReader::open(config).ok()?;
    let snapshots: Vec<OldBlockSnapshot> = (0..reader.block_count)
        .map(|i| {
            let h = reader.header(i);
            OldBlockSnapshot {
                depth: h.depth,
                layer_id: h.layer_id,
                text: reader.text(i).as_bytes().to_vec(),
            }
        })
        .collect();
    Some(snapshots)
}

/// Read the layers content hash stored in the pre-rebuild meta.bin (MSC4).
fn read_stored_layers_hash(output_dir: &Path) -> Option<[u8; 32]> {
    let meta = fs::read(output_dir.join("meta.bin")).ok()?;
    if meta.len() < 152 || &meta[0..4] != b"MSC4" {
        return None;
    }
    let mut hash = [0u8; 32];
    hash.copy_from_slice(&meta[120..152]);
    Some(hash)
}

/// Build a positional remap from the OLD index layout to the NEW one by
/// matching block content (depth + layer + bytes). Old blocks that have no
/// counterpart in the new layout map to `u32::MAX` and are dropped by the
/// sidecar remap routines. Duplicate content maps in order so repeated entries
/// keep their relative positions.
fn compute_content_remap(old: &Option<Vec<OldBlockSnapshot>>, new_blocks: &[RawBlock]) -> Vec<u32> {
    let old = match old {
        Some(v) => v,
        None => return Vec::new(),
    };
    if old.is_empty() {
        return Vec::new();
    }

    // key = (depth, layer_id, data bytes) → new layout positions (ascending).
    let mut index: HashMap<(u8, u8, Vec<u8>), Vec<u32>> = HashMap::new();
    for (j, b) in new_blocks.iter().enumerate() {
        index
            .entry((b.depth, b.layer_id, b.data.clone()))
            .or_default()
            .push(j as u32);
    }

    let mut cursor: HashMap<(u8, u8, Vec<u8>), usize> = HashMap::new();
    let mut map = vec![u32::MAX; old.len()];
    for (i, s) in old.iter().enumerate() {
        let key = (s.depth, s.layer_id, s.text.clone());
        if let Some(list) = index.get(&key) {
            let c = cursor.entry(key).or_insert(0);
            if *c < list.len() {
                map[i] = list[*c];
                *c += 1;
            }
        }
    }
    map
}

#[cfg(test)]
mod tests {
    use super::*;

    fn snap(depth: u8, layer_id: u8, text: &str) -> OldBlockSnapshot {
        OldBlockSnapshot {
            depth,
            layer_id,
            text: text.as_bytes().to_vec(),
        }
    }

    fn block(depth: u8, layer_id: u8, text: &str) -> RawBlock {
        RawBlock {
            data: to_block(text),
            depth,
            x: 0.0,
            y: 0.0,
            z: 0.0,
            layer_id,
            parent_idx: u32::MAX,
            child_count: 0,
            importance: 5,
        }
    }

    /// Retention shift: the first layer entry is truncated before rebuild, so
    /// every surviving block moves one position earlier. The content remap must
    /// follow the surviving blocks to their new positions.
    #[test]
    fn content_remap_follows_retention_shift() {
        let old = Some(vec![
            snap(3, 1, "Alpha first entry"),
            snap(3, 1, "Beta second entry"),
            snap(3, 1, "Gamma third entry"),
        ]);
        // New layout: Alpha is gone (retention), Beta and Gamma shift down.
        let new_blocks = vec![
            block(0, 0, "identity"),
            block(3, 1, "Beta second entry"),
            block(3, 1, "Gamma third entry"),
            block(4, 1, "Beta second entry"),
        ];
        let map = compute_content_remap(&old, &new_blocks);
        // Alpha has no counterpart → u32::MAX.
        assert_eq!(map[0], u32::MAX);
        // Beta (old #1) → first Beta in the new layout (#1). Note the D4
        // sentence "Beta second entry" also matches; ordered matching assigns
        // the first (D3) occurrence, which is what recall returns.
        assert_eq!(map[1], 1);
        assert_eq!(map[2], 2);
    }

    /// Rebuild with unchanged content: every old block maps to itself.
    #[test]
    fn content_remap_is_identity_for_unchanged_content() {
        let old = Some(vec![
            snap(3, 1, "Alpha first entry"),
            snap(3, 1, "Beta second entry"),
        ]);
        let new_blocks = vec![
            block(0, 0, "identity"),
            block(3, 1, "Alpha first entry"),
            block(3, 1, "Beta second entry"),
        ];
        let map = compute_content_remap(&old, &new_blocks);
        assert_eq!(map, vec![1, 2]);
    }

    /// No old index (first build): empty mapping is safe.
    #[test]
    fn content_remap_empty_without_old_index() {
        let map = compute_content_remap(&None, &[block(3, 1, "x")]);
        assert!(map.is_empty());
    }

    /// An entry longer than one block must be split, not truncated.
    ///
    /// This is the SciFact regression. 4,300 of 5,183 abstracts exceed
    /// BLOCK_DATA_SIZE, and `to_block` used to cut them at 1,021 characters,
    /// so the tail of most of the corpus was never stored. Asserting on the
    /// reassembled word set is what makes the test meaningful: a truncation
    /// passes a "does it fit in a block" check, it fails this one.
    #[test]
    fn oversized_entry_is_split_without_losing_bytes() {
        let sentence = "The quick brown fox jumps over the lazy dog. ";
        // Sized past BLOCK_DATA_SIZE (16 KiB), not to a fixed byte count: this
        // constant was raised from 1024 and the test has to follow it, or it
        // silently stops exercising the split path it exists to cover.
        let entry = sentence.repeat(400);
        assert!(entry.len() > BLOCK_DATA_SIZE);

        let pieces = split_oversized_entry(entry.trim());
        assert!(pieces.len() > 1, "expected the entry to be split");
        for p in &pieces {
            assert!(
                p.len() <= BLOCK_DATA_SIZE,
                "piece of {} bytes exceeds the block size",
                p.len()
            );
        }

        let rejoined = pieces.join(" ");
        let original_words: Vec<&str> = entry.split_whitespace().collect();
        let rejoined_words: Vec<&str> = rejoined.split_whitespace().collect();
        assert_eq!(
            original_words, rejoined_words,
            "splitting changed the content of the entry"
        );
    }

    /// The split must also survive a single sentence longer than a block.
    #[test]
    fn oversized_single_sentence_is_split_on_whitespace() {
        let entry = "word ".repeat(4000); // no sentence terminator at all
        assert!(entry.len() > BLOCK_DATA_SIZE);
        let pieces = split_oversized_entry(entry.trim());
        assert!(pieces.len() > 1);
        for p in &pieces {
            assert!(p.len() <= BLOCK_DATA_SIZE);
        }
        assert_eq!(pieces.join(" "), entry.trim());
    }

    /// Ordinary short entries are untouched: no needless fragmentation.
    #[test]
    fn short_entry_is_returned_whole() {
        let entry = "A short fact that fits in a block. Another sentence.";
        let pieces = split_oversized_entry(entry);
        assert_eq!(pieces, vec![entry.to_string()]);
    }

    /// A layer file with one entry per line must yield one block per line.
    ///
    /// This is the regression behind 504 blocks for 5,183 abstracts. The reader
    /// used to pack consecutive lines together until BLOCK_DATA_SIZE was
    /// reached, which was invisible while the limit was 1,024 and a typical
    /// abstract overflowed it alone. At 16 KiB the same code merged about ten
    /// documents per block, and the benchmark would have scored retrieval over
    /// merged documents while reporting a plausible-looking R@k.
    ///
    /// The entries here are sized well below the limit, so any packing
    /// whatsoever fails the assertion.
    #[test]
    fn one_line_per_entry_is_not_packed_into_shared_blocks() {
        let n = 40usize;
        let mut file = String::new();
        for i in 0..n {
            file.push_str(&format!("document number {} with some body text\n", i));
        }
        let tmp = tempfile::tempdir().expect("tempdir");
        let path = tmp.path().join("long_term.txt");
        fs::write(&path, &file).unwrap();

        let texts = extract_texts_from_file(&path);
        assert_eq!(
            texts.len(),
            n,
            "one line must be one block; {} lines produced {} blocks",
            n,
            texts.len()
        );
        for (i, (text, _)) in texts.iter().enumerate() {
            assert!(
                text.contains(&format!("document number {}", i)),
                "block {} does not hold its own entry: {:?}",
                i,
                text
            );
        }
    }
}