formal-ai 0.304.0

//! The whole-repository source ↔ links projection (issue #558).
//!
//! Issue #558 ("Auto learning") asks for a meta-algorithm advanced enough to
//! *"recompile itself"*: it must be able to *"translate the entire source code of
//! our system to links / meta language (that must be present in our data), and
//! back to the source code"*. [`crate::agentic_coding::self_ast`] already proves
//! the lossless source → links → source round-trip for a *single* pinned module
//! (the planner); this module lifts that from one module to **every owned Rust
//! source file in the repository**.
//!
//! The entire source is embedded at build time (see `build.rs`, which generates
//! `OWNED_SOURCE_FILES`), so the projection needs no filesystem access — it works
//! from an agent CLI's sandbox workdir, exactly like the other self-inspection
//! recipes. Two views are offered, deliberately split by cost:
//!
//! * [`owned_manifest`] / [`owned_manifest_notation`] — a **cheap** content-addressed
//!   manifest of *every* owned file (`path`, `byte_len`, `content_id`). This is the
//!   "entire source present in our data as links" view: enumerating and content-
//!   addressing all of it costs only a hash per file, so it is safe to build on
//!   every request and in every test.
//! * [`SourceLinks`] — the **full** projection that parses each file through the
//!   sole CST/AST engine ([meta-language](https://github.com/link-foundation/meta-language))
//!   and verifies the byte-for-byte
//!   round-trip. [`SourceLinks::compile`] projects any file list;
//!   [`SourceLinks::owned`] projects the *whole* repository. Parsing every file is
//!   deliberately non-trivial (seconds per file in debug), so the whole-repository
//!   projection is exercised by the example binary and an exhaustive (ignored by
//!   default) test rather than on every `cargo test`.
//!
//! Nothing here writes anything back: the projection is an auditable, read-only
//! artifact — the "recompile itself" guardrail (observable, testable, reversible,
//! human-approved) is preserved because translating *to* links and *back* is proven
//! lossless before any edit could ever be expressed against it. Neural inference
//! stays a NON-GOAL: every number is a deterministic function of the embedded
//! source and the real tree-sitter parse.

use std::fmt::Write as _;
use std::sync::OnceLock;

use crate::agentic_coding::self_ast::ast_census;
use crate::engine::stable_id;

// The compile-time manifest of every owned `src/**/*.rs` file, generated by
// `build.rs` as `pub const OWNED_SOURCE_FILES: &[(&str, &str)]` sorted by path.
include!(concat!(env!("OUT_DIR"), "/owned_source_files.rs"));

/// The `(repo_relative_path, source_text)` pairs for every owned Rust source file,
/// embedded at build time and sorted by path.
///
/// This is the concrete realisation of issue #558's *"the entire source code of our
/// system … must be present in our data"*: the whole owned source tree ships in the
/// binary, addressable as data with no filesystem access.
#[must_use]
pub const fn owned_source_files() -> &'static [(&'static str, &'static str)] {
    OWNED_SOURCE_FILES
}

/// A cheap, content-addressed digest of one source file — no parse required.
///
/// This is the unit of the whole-repository *manifest*: it proves the file is
/// enumerated and content-addressed (so it is "present in our data as links") at
/// the cost of a single hash, without paying for a full CST/AST parse.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SourceModuleDigest {
    /// The repository-relative path of the source file.
    pub path: String,
    /// The byte length of the source (a stable size signal).
    pub byte_len: usize,
    /// A stable content-addressed id for the source text.
    pub content_id: String,
}

impl SourceModuleDigest {
    /// Digest `source` (identified by `path`) — hash only, no parse.
    #[must_use]
    pub fn of(path: impl Into<String>, source: &str) -> Self {
        Self {
            path: path.into(),
            byte_len: source.len(),
            content_id: stable_id("source_module", source),
        }
    }
}

/// The full source ↔ links projection of one module, verified through the sole
/// CST/AST engine in this repo.
///
/// A projection is *faithful* exactly when `source → links → source` reproduces the
/// input byte-for-byte, i.e. the links/meta representation lost nothing and an edit
/// could in principle be expressed against the links and rendered back to
/// compilable Rust — the "recompile itself" round-trip issue #558 asks for.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SourceModuleProjection {
    /// The repository-relative path of the projected module.
    pub path: String,
    /// The byte length of the source.
    pub byte_len: usize,
    /// A stable content-addressed id for the source text.
    pub content_id: String,
    /// Total links in the lossless network (a size signal for the parse).
    pub total_link_count: usize,
    /// Named abstract-syntax nodes recovered from the parse (the AST proper).
    pub named_node_count: usize,
    /// Whether `source → links → source` reproduced the input byte-for-byte.
    pub faithful: bool,
}

impl SourceModuleProjection {
    /// Project `source` (identified by `path`) through the meta-language links
    /// network and record its census and round-trip verdict.
    #[must_use]
    pub fn project(path: impl Into<String>, source: &str) -> Self {
        // A single parse yields both the census and the round-trip verdict:
        // `AstCensus::text_preserved` *is* `reconstruct_text() == source`.
        let census = ast_census(source);
        Self {
            path: path.into(),
            byte_len: source.len(),
            content_id: stable_id("source_module", source),
            total_link_count: census.total_link_count,
            named_node_count: census.named_node_count,
            faithful: census.text_preserved,
        }
    }

    /// The cheap digest for this module (drops the parse-derived counts).
    #[must_use]
    pub fn digest(&self) -> SourceModuleDigest {
        SourceModuleDigest {
            path: self.path.clone(),
            byte_len: self.byte_len,
            content_id: self.content_id.clone(),
        }
    }
}

/// The whole-repository source ↔ links projection: every projected module plus the
/// aggregate coverage of the lossless round-trip.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SourceLinks {
    /// One projection per source file, in the embedded (path-sorted) order.
    pub modules: Vec<SourceModuleProjection>,
}

impl SourceLinks {
    /// Project every `(path, source)` pair in `files` to links and back.
    ///
    /// The general primitive: it works over any file list, so callers can project a
    /// representative slice cheaply or the whole repository exhaustively.
    #[must_use]
    pub fn compile(files: &[(&str, &str)]) -> Self {
        let modules = files
            .iter()
            .map(|(path, source)| SourceModuleProjection::project(*path, source))
            .collect();
        Self { modules }
    }

    /// The whole-repository projection over every owned source file, computed once
    /// per process.
    ///
    /// Parsing every file through the CST/AST engine is deliberately expensive, and
    /// a server or test may ask for the projection repeatedly, so the result is
    /// memoised; it is a deterministic function of the embedded source, so memoising
    /// changes nothing but latency.
    #[must_use]
    pub fn owned() -> &'static Self {
        static LINKS: OnceLock<SourceLinks> = OnceLock::new();
        LINKS.get_or_init(|| Self::compile(OWNED_SOURCE_FILES))
    }

    /// How many modules the links projection projected.
    #[must_use]
    pub const fn module_count(&self) -> usize {
        self.modules.len()
    }

    /// How many modules round-tripped losslessly (`source → links → source`).
    #[must_use]
    pub fn faithful_count(&self) -> usize {
        self.modules.iter().filter(|module| module.faithful).count()
    }

    /// Whether *every* projected module round-trips byte-for-byte — the invariant
    /// issue #558's "translate the entire source … and back" requires.
    #[must_use]
    pub fn is_fully_faithful(&self) -> bool {
        !self.modules.is_empty() && self.faithful_count() == self.modules.len()
    }

    /// The lossless-round-trip coverage in permille (parts per thousand), computed
    /// with integer math so the artifact stays float-free and deterministic. A fully
    /// faithful projection is `1000`.
    #[must_use]
    pub fn coverage_permille(&self) -> u32 {
        if self.modules.is_empty() {
            return 0;
        }
        let faithful = self.faithful_count() as u64;
        let total = self.modules.len() as u64;
        u32::try_from(faithful * 1000 / total).unwrap_or(0)
    }

    /// The modules that did *not* round-trip, if any — the triage list a human
    /// reviews before trusting a whole-repository recompile.
    #[must_use]
    pub fn unfaithful_modules(&self) -> Vec<&SourceModuleProjection> {
        self.modules
            .iter()
            .filter(|module| !module.faithful)
            .collect()
    }

    /// Total links across every projected module (an aggregate size signal).
    #[must_use]
    pub fn total_link_count(&self) -> usize {
        self.modules.iter().map(|m| m.total_link_count).sum()
    }

    /// Total named abstract-syntax nodes across every projected module.
    #[must_use]
    pub fn total_named_node_count(&self) -> usize {
        self.modules.iter().map(|m| m.named_node_count).sum()
    }

    /// Total bytes of source across every projected module.
    #[must_use]
    pub fn total_byte_len(&self) -> usize {
        self.modules.iter().map(|m| m.byte_len).sum()
    }

    /// A one-line human-readable summary of the whole-repository projection.
    #[must_use]
    pub fn summary(&self) -> String {
        format!(
            "Projected {faithful}/{total} owned source modules losslessly through the meta-language \
             links network ({permille}‰ round-trip coverage, {nodes} named AST nodes).",
            faithful = self.faithful_count(),
            total = self.module_count(),
            permille = self.coverage_permille(),
            nodes = self.total_named_node_count(),
        )
    }

    /// Render the whole-repository projection as Links Notation — the auditable
    /// artifact of the source ↔ links translation. Ends trimmed of trailing
    /// whitespace.
    #[must_use]
    pub fn links_notation(&self) -> String {
        let mut out = String::from("source_links\n");
        let _ = writeln!(out, "  engine meta_language");
        let _ = writeln!(out, "  language rust");
        let _ = writeln!(out, "  module_count {}", self.module_count());
        let _ = writeln!(out, "  faithful_count {}", self.faithful_count());
        let _ = writeln!(out, "  coverage_permille {}", self.coverage_permille());
        let _ = writeln!(out, "  fully_faithful {}", self.is_fully_faithful());
        let _ = writeln!(out, "  total_link_count {}", self.total_link_count());
        let _ = writeln!(
            out,
            "  total_named_node_count {}",
            self.total_named_node_count()
        );
        let _ = writeln!(out, "  modules");
        for module in &self.modules {
            let _ = writeln!(out, "    module");
            let _ = writeln!(out, "      path \"{}\"", quote(&module.path));
            let _ = writeln!(out, "      byte_len {}", module.byte_len);
            let _ = writeln!(out, "      content_id \"{}\"", quote(&module.content_id));
            let _ = writeln!(out, "      total_link_count {}", module.total_link_count);
            let _ = writeln!(out, "      named_node_count {}", module.named_node_count);
            let _ = writeln!(out, "      faithful {}", module.faithful);
        }
        out.trim_end().to_owned()
    }
}

/// The cheap content-addressed manifest of *every* owned source file — hashes only,
/// no parse. Fast enough to build on every request and in every test.
#[must_use]
pub fn owned_manifest() -> Vec<SourceModuleDigest> {
    OWNED_SOURCE_FILES
        .iter()
        .map(|(path, source)| SourceModuleDigest::of(*path, source))
        .collect()
}

/// The number of owned source files present in our data.
#[must_use]
pub const fn owned_file_count() -> usize {
    OWNED_SOURCE_FILES.len()
}

/// The total byte length of the entire owned source tree.
#[must_use]
pub fn owned_total_bytes() -> usize {
    OWNED_SOURCE_FILES
        .iter()
        .map(|(_, source)| source.len())
        .sum()
}

/// A single stable content-addressed id for the *entire* owned source tree.
///
/// Hashes the content-addressed manifest (every file's path + `content_id`), so the
/// whole repository collapses to one id that changes iff any owned source changes.
/// Cheap (hash only) and deterministic — it lets an agentic response reference "the
/// entire source, as one link" without enumerating every file inline.
#[must_use]
pub fn owned_manifest_content_id() -> String {
    stable_id("source_tree", &owned_manifest_notation())
}

/// Render the cheap whole-repository *manifest* as Links Notation: every owned file
/// enumerated and content-addressed. Ends trimmed of trailing whitespace.
///
/// This is the "entire source present in our data as links" artifact — it accounts
/// for every owned file at hash cost, so it is safe to embed in an agentic recipe's
/// response. The *lossless* proof is [`SourceLinks`]; this is the *coverage* proof.
#[must_use]
pub fn owned_manifest_notation() -> String {
    let manifest = owned_manifest();
    let mut out = String::from("source_manifest\n");
    let _ = writeln!(out, "  engine meta_language");
    let _ = writeln!(out, "  language rust");
    let _ = writeln!(out, "  file_count {}", manifest.len());
    let _ = writeln!(out, "  total_bytes {}", owned_total_bytes());
    let _ = writeln!(out, "  files");
    for digest in &manifest {
        let _ = writeln!(out, "    file");
        let _ = writeln!(out, "      path \"{}\"", quote(&digest.path));
        let _ = writeln!(out, "      byte_len {}", digest.byte_len);
        let _ = writeln!(out, "      content_id \"{}\"", quote(&digest.content_id));
    }
    out.trim_end().to_owned()
}

fn quote(value: &str) -> String {
    value
        .replace('\\', "\\\\")
        .replace('"', "'")
        .replace('\n', "\\n")
        .replace('\r', "\\r")
        .replace('\t', "\\t")
}