memstead_schema/

source.rs

//! Schema source-file collection for publishing mem archives.
//!
//! A published `.mem` archive is portable only if the schema it pins
//! travels inside it — otherwise opening the archive on a foreign
//! machine without the matching schema registered would fail at
//! resolve time. `collect_schema_source` resolves a `SchemaRef` to the
//! raw YAML bytes callers need to zip into the archive's `schema/`
//! tree.
//!
//! Resolution order (first match wins):
//! 1. `<workspace>/<schemas_dir>/<name>/` — workspace-level shared
//!    schemas (optional; when the caller supplies a workspace path)
//! 2. `<workspace>/.memstead.cache/schemas/<name>-<version>/` — cache
//!    extracted from a previously loaded archive (workspace-wide)
//! 3. Embedded built-in (via `include_dir!`) — ships inside the binary
//!
//! In every case the manifest's declared version is checked against the
//! pin; a name collision at the wrong version falls through rather than
//! silently embedding a mismatched schema.

/// Filename of the install-time provenance stamp `memstead schema
/// install` writes INTO the sealed package when the install source was
/// an authoring directory: `{"authoring_path": "<canonical path>"}`.
/// The stamp is the detection basis for the authoring-drift health
/// axis — a schema without one (sealed pre-stamp, built-in, name- or
/// archive-sourced install) is simply not checked, because a guessed
/// provenance is worse than an absent one. The stamp is workspace-
/// local by design: every package collector in this module reads
/// selectively (`schema.yaml`, `types/*.yaml`, `mem-template.json`,
/// `README.md`) and therefore never picks it up, and the git-ref
/// export path excludes it by name — a published `.mem` archive never
/// carries another machine's filesystem path.
pub const INSTALL_PROVENANCE_FILE: &str = "install-provenance.json";

use std::path::{Path, PathBuf};

use crate::builtins::builtin_schemas_dir;
use crate::config::SchemaRef;

/// One source file destined for the archive's `.memstead/schema/` tree.
///
/// `archive_path` is the relative path *inside* the archive (e.g.
/// `"schema.yaml"` or `"types/spec.yaml"`). Callers prefix it with
/// whatever root they want (the archive writer uses `".memstead/schema/"`);
/// the canonical re-pack uses the same prefix so byte-identical
/// archives round-trip.
#[derive(Debug, Clone)]
pub struct SchemaSourceFile {
    pub archive_path: String,
    pub bytes: Vec<u8>,
}

#[derive(Debug, thiserror::Error)]
pub enum SchemaSourceError {
    #[error(
        "schema {schema_ref} not found — candidate paths tried: [{}]",
        .candidates.iter().map(|p| p.display().to_string()).collect::<Vec<_>>().join(", ")
    )]
    NotFound {
        schema_ref: String,
        /// Every filesystem location (mem-local, workspace-level,
        /// cache) consulted before falling through to the embedded
        /// builtins. Listed in resolution order so the error message
        /// matches the precedence the resolver walked.
        candidates: Vec<PathBuf>,
    },

    #[error("i/o error reading schema source at {}: {source}", .path.display())]
    Io {
        path: PathBuf,
        #[source]
        source: std::io::Error,
    },

    /// Two distinct on-disk schema directories share the same
    /// `<name>-<version>` cache key. Indicates a bug in the cache
    /// extraction pipeline — silent overwrite would mask the issue.
    #[error(
        "schema cache collision: '{name}-{version}' has more than one source directory ({} and {})",
        .first.display(),
        .second.display()
    )]
    CacheCollision {
        name: String,
        version: String,
        first: PathBuf,
        second: PathBuf,
    },

    #[error(
        "schema manifest at {} does not declare version '{expected}' (found '{found}')",
        .path.display()
    )]
    VersionMismatch {
        path: PathBuf,
        expected: String,
        found: String,
    },

    #[error("schema manifest at {} is malformed: {reason}", .path.display())]
    MalformedManifest { path: PathBuf, reason: String },
}

/// Resolve the schema pinned by `schema_ref` to a sorted set of source
/// files ready to embed under `.memstead/schema/` in a mem archive.
///
/// The returned vector is sorted by `archive_path` so archive bytes are
/// deterministic — callers don't need to re-sort.
///
/// Resolution order (first match wins):
/// 1. `<workspace_schemas_dir>/<name>/` (when provided)
/// 2. `<workspace_root>/.memstead.cache/schemas/<name>-<version>/`
///    (when `workspace_root` is provided)
/// 3. Embedded builtins
///
/// When every filesystem layer misses, the returned `NotFound` lists
/// every concrete path that was consulted so callers (and agents) can
/// inspect the resolution trace without re-walking the filesystem.
pub fn collect_schema_source(
    workspace_root: Option<&Path>,
    workspace_schemas_dir: Option<&Path>,
    schema_ref: &SchemaRef,
) -> Result<Vec<SchemaSourceFile>, SchemaSourceError> {
    let mut candidates: Vec<PathBuf> = Vec::new();

    if let Some(ws_dir) = workspace_schemas_dir {
        // Two directory shapes: `<name>@<version>/` is what `memstead
        // schema install` writes; the bare `<name>/` form predates the
        // versioned layout and stays supported for hand-authored dirs.
        let versioned_dir = ws_dir.join(format!("{}@{}", schema_ref.name, schema_ref.version));
        candidates.push(versioned_dir.clone());
        if versioned_dir.is_dir()
            && let Some(files) = try_collect_dir(&versioned_dir, schema_ref)?
        {
            return Ok(files);
        }
        let ws_schema_dir = ws_dir.join(&schema_ref.name);
        candidates.push(ws_schema_dir.clone());
        if ws_schema_dir.is_dir()
            && let Some(files) = try_collect_dir(&ws_schema_dir, schema_ref)?
        {
            return Ok(files);
        }
    }

    if let Some(ws_root) = workspace_root {
        let cache_dir = ws_root
            .join(".memstead.cache/schemas")
            .join(format!("{}-{}", schema_ref.name, schema_ref.version));
        candidates.push(cache_dir.clone());
        if cache_dir.is_dir()
            && let Some(files) = try_collect_dir(&cache_dir, schema_ref)?
        {
            return Ok(files);
        }
    }

    if let Some(files) = collect_builtin_source(schema_ref)? {
        return Ok(files);
    }

    Err(SchemaSourceError::NotFound {
        schema_ref: schema_ref.as_display(),
        candidates,
    })
}

/// Read `<dir>/schema.yaml` + `<dir>/types/*.yaml` and return them
/// only if the manifest's declared version matches `schema_ref`.
/// A mismatched version returns `Ok(None)` so the caller can fall
/// through to the next resolution layer rather than hard-failing.
fn try_collect_dir(
    dir: &Path,
    schema_ref: &SchemaRef,
) -> Result<Option<Vec<SchemaSourceFile>>, SchemaSourceError> {
    let manifest_path = dir.join("schema.yaml");
    let manifest_bytes = std::fs::read(&manifest_path).map_err(|e| SchemaSourceError::Io {
        path: manifest_path.clone(),
        source: e,
    })?;

    if !manifest_matches(&manifest_bytes, schema_ref, &manifest_path)? {
        return Ok(None);
    }

    let mut out = vec![SchemaSourceFile {
        archive_path: "schema.yaml".to_string(),
        bytes: manifest_bytes,
    }];

    // The sealed format marker rides as-found — it records the
    // package's metadata-polarity generation, so a collected source
    // seals with the same reading its origin carries.
    let marker_path = dir.join(crate::loader::SCHEMA_FORMAT_MARKER_FILE);
    if marker_path.is_file() {
        let bytes = std::fs::read(&marker_path).map_err(|e| SchemaSourceError::Io {
            path: marker_path.clone(),
            source: e,
        })?;
        out.push(SchemaSourceFile {
            archive_path: crate::loader::SCHEMA_FORMAT_MARKER_FILE.to_string(),
            bytes,
        });
    }

    let types_dir = dir.join("types");
    if types_dir.is_dir() {
        let entries = std::fs::read_dir(&types_dir).map_err(|e| SchemaSourceError::Io {
            path: types_dir.clone(),
            source: e,
        })?;
        for entry in entries {
            let entry = entry.map_err(|e| SchemaSourceError::Io {
                path: types_dir.clone(),
                source: e,
            })?;
            let path = entry.path();
            if path.extension().and_then(|s| s.to_str()) != Some("yaml") {
                continue;
            }
            let Some(stem) = path.file_stem().and_then(|s| s.to_str()) else {
                continue;
            };
            let bytes = std::fs::read(&path).map_err(|e| SchemaSourceError::Io {
                path: path.clone(),
                source: e,
            })?;
            out.push(SchemaSourceFile {
                archive_path: format!("types/{stem}.yaml"),
                bytes,
            });
        }
    }

    out.sort_by(|a, b| a.archive_path.cmp(&b.archive_path));
    Ok(Some(out))
}

fn collect_builtin_source(
    schema_ref: &SchemaRef,
) -> Result<Option<Vec<SchemaSourceFile>>, SchemaSourceError> {
    // Name-exact directory first — the current generation lives there.
    if let Some(schema_dir) = builtin_schemas_dir().get_dir(schema_ref.name.as_str())
        && let Some(files) = collect_builtin_dir(schema_dir, schema_ref)?
    {
        return Ok(Some(files));
    }
    // Retained older generations live in suffixed sibling directories
    // (`planning` vs `planning-0.2`, …) under the append-only
    // retention pattern — the directory name is organisational only,
    // identity comes from each package's manifest. Scan the whole
    // embedded catalogue for the (name, version) the ref pins; the
    // registry registers every retained version, so the collect path
    // must resolve them all.
    for schema_dir in builtin_schemas_dir().dirs() {
        if let Some(files) = collect_builtin_dir(schema_dir, schema_ref)? {
            return Ok(Some(files));
        }
    }
    Ok(None)
}

/// Collect one embedded schema directory's files when its manifest
/// matches `schema_ref` — `Ok(None)` on a manifest mismatch (or a
/// directory without a manifest, e.g. a non-package entry) so the
/// caller can keep scanning.
fn collect_builtin_dir(
    schema_dir: &include_dir::Dir<'static>,
    schema_ref: &SchemaRef,
) -> Result<Option<Vec<SchemaSourceFile>>, SchemaSourceError> {
    // `include_dir`'s paths are always relative to the include root (the
    // `builtins/schemas` dir), so every entry's path starts with the
    // directory name. Build lookups by constructing the same prefix.
    let prefix = schema_dir.path().display().to_string();
    let manifest_key = format!("{prefix}/schema.yaml");
    let Some(manifest_file) = schema_dir.get_file(manifest_key.as_str()) else {
        return Ok(None);
    };
    let manifest_bytes = manifest_file.contents().to_vec();
    if !manifest_matches(
        &manifest_bytes,
        schema_ref,
        &PathBuf::from(format!("<builtin:{prefix}>/schema.yaml")),
    )? {
        return Ok(None);
    }

    let mut out = vec![SchemaSourceFile {
        archive_path: "schema.yaml".to_string(),
        bytes: manifest_bytes,
    }];

    // The sealed format marker rides as-found (new builtin
    // generations ship it; retained older generations don't).
    let marker_key = format!("{prefix}/{}", crate::loader::SCHEMA_FORMAT_MARKER_FILE);
    if let Some(marker) = schema_dir.get_file(marker_key.as_str()) {
        out.push(SchemaSourceFile {
            archive_path: crate::loader::SCHEMA_FORMAT_MARKER_FILE.to_string(),
            bytes: marker.contents().to_vec(),
        });
    }

    let types_key = format!("{prefix}/types");
    if let Some(types_dir) = schema_dir.get_dir(types_key.as_str()) {
        for file in types_dir.files() {
            if file.path().extension().and_then(|s| s.to_str()) != Some("yaml") {
                continue;
            }
            let Some(stem) = file.path().file_stem().and_then(|s| s.to_str()) else {
                continue;
            };
            out.push(SchemaSourceFile {
                archive_path: format!("types/{stem}.yaml"),
                bytes: file.contents().to_vec(),
            });
        }
    }

    out.sort_by(|a, b| a.archive_path.cmp(&b.archive_path));
    Ok(Some(out))
}

/// True when `manifest_bytes` parses and its `name`+`version` match
/// `schema_ref`. Uses `serde_yaml_ng` with a narrow intermediate struct
/// rather than the full `SchemaManifest` so callers can compare versions
/// without paying for the full manifest validation (the caller is often
/// the publish path, where the real loader runs separately over the
/// source directory for full validation).
fn manifest_matches(
    manifest_bytes: &[u8],
    schema_ref: &SchemaRef,
    source_path: &Path,
) -> Result<bool, SchemaSourceError> {
    #[derive(serde::Deserialize)]
    struct ManifestId {
        name: String,
        version: String,
    }
    let id: ManifestId = serde_yaml_ng::from_slice(manifest_bytes).map_err(|e| {
        SchemaSourceError::MalformedManifest {
            path: source_path.to_path_buf(),
            reason: e.to_string(),
        }
    })?;
    if id.name != schema_ref.name {
        return Ok(false);
    }
    let declared =
        semver::Version::parse(&id.version).map_err(|e| SchemaSourceError::MalformedManifest {
            path: source_path.to_path_buf(),
            reason: format!("invalid semver '{}': {e}", id.version),
        })?;
    if declared != schema_ref.version {
        // Surface the mismatch as a hard error only for the on-disk
        // paths that selected `dir` by name — a workspace author bumping
        // the pin without editing the manifest should hear about it
        // loudly, not get a silent fallthrough to the next layer.
        // Builtin retention keeps every generation in sibling
        // directories (`planning`, `planning-0.2`, …), so the
        // `collect_builtin_source` path turns `Ok(false)` into "keep
        // scanning; NotFound only when no directory matches". String
        // prefix, deliberately: `Path::starts_with` is
        // component-based and never matched the `<builtin:…>` marker
        // (which made every non-name-exact builtin version refuse
        // with a hard VersionMismatch instead of falling through).
        if source_path.to_string_lossy().starts_with("<builtin:") {
            return Ok(false);
        }
        return Err(SchemaSourceError::VersionMismatch {
            path: source_path.to_path_buf(),
            expected: schema_ref.version.to_string(),
            found: declared.to_string(),
        });
    }
    Ok(true)
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::TempDir;

    fn write_schema(dir: &Path, name: &str, version: &str, types: &[&str]) {
        let manifest = format!(
            r#"name: {name}
version: {version}
description: test
when_to_use: test
types:
  - {type_list}
relationships:
  mode: strict
  definitions:
    - name: _default
      description: default
      default_weight: 1.0
    - name: PART_OF
      description: hier
      default_weight: 3.0
community:
  resolution: 1.0
  seed: 42
"#,
            name = name,
            version = version,
            type_list = types.join("\n  - "),
        );
        std::fs::write(dir.join("schema.yaml"), manifest).unwrap();
        for t in types {
            let td = format!(
                r#"name: {t}
description: test
when_to_use: test
sections:
  - key: body
    heading: Body
    required: true
    search_weight: 10.0
    catch_all: true
metadata_fields: []
title_weight: 1.0
text_fields: [body]
hierarchy_relationship: PART_OF
no_self_loop_relationships: []
updatable_fields: [title, body]
health_required_fields: [body]
staleness_threshold_days: 30
write_rules: []
"#
            );
            std::fs::write(dir.join(format!("types/{t}.yaml")), td).unwrap();
        }
    }

    /// The collectors carry the sealed format marker as-found: the
    /// current builtin generation ships it, a retained pre-flip
    /// generation doesn't — so seals stay faithful in both directions.
    #[test]
    fn collectors_carry_format_marker_as_found() {
        let marked: SchemaRef = "default@1.3.0".parse().unwrap();
        let files = collect_schema_source(None, None, &marked).unwrap();
        assert!(
            files
                .iter()
                .any(|f| f.archive_path == crate::loader::SCHEMA_FORMAT_MARKER_FILE),
            "current generation collects its marker"
        );

        let legacy: SchemaRef = "default@1.2.0".parse().unwrap();
        let files = collect_schema_source(None, None, &legacy).unwrap();
        assert!(
            !files
                .iter()
                .any(|f| f.archive_path == crate::loader::SCHEMA_FORMAT_MARKER_FILE),
            "retained pre-flip generation stays unmarked"
        );
    }

    #[test]
    fn collects_builtin_default_source() {
        let schema_ref = SchemaRef::new("default", semver::Version::new(1, 0, 0));
        let files = collect_schema_source(None, None, &schema_ref).unwrap();

        assert!(
            files.iter().any(|f| f.archive_path == "schema.yaml"),
            "embedded builtin must expose schema.yaml"
        );
        let type_count = files
            .iter()
            .filter(|f| f.archive_path.starts_with("types/"))
            .count();
        assert_eq!(type_count, 10, "default schema has 10 types");
        for pair in files.windows(2) {
            assert!(pair[0].archive_path < pair[1].archive_path, "sorted");
        }
    }

    /// The append-only retention pattern keeps older generations in
    /// suffixed sibling directories (`planning` holds 0.1.0,
    /// `planning-0.3` holds 0.3.0, …). Every RETAINED version the
    /// registry registers must resolve through the collect path — the
    /// historical name-exact-only lookup refused every version but
    /// the one in the name-exact directory.
    #[test]
    fn collects_builtin_source_for_every_retained_version() {
        for (name, version) in [
            ("planning", semver::Version::new(0, 2, 0)),
            ("planning", semver::Version::new(0, 4, 0)),
            ("ingest", semver::Version::new(0, 1, 0)),
            ("ingest", semver::Version::new(0, 5, 0)),
        ] {
            let schema_ref = SchemaRef::new(name, version.clone());
            let files = collect_schema_source(None, None, &schema_ref)
                .unwrap_or_else(|e| panic!("{name}@{version} must resolve: {e}"));
            let manifest = files
                .iter()
                .find(|f| f.archive_path == "schema.yaml")
                .expect("manifest present");
            let text = String::from_utf8_lossy(&manifest.bytes);
            assert!(
                text.contains(&format!("version: {version}")),
                "{name}@{version}: collected manifest must carry the requested version"
            );
        }

        // An unregistered version still refuses — the scan resolves
        // retained versions, it never invents one.
        let ghost = SchemaRef::new("planning", semver::Version::new(9, 9, 9));
        assert!(matches!(
            collect_schema_source(None, None, &ghost),
            Err(SchemaSourceError::NotFound { .. })
        ));
    }

    #[test]
    fn workspace_schema_wins_over_builtin() {
        let tmp = TempDir::new().unwrap();
        let ws_dir = tmp.path().join("schemas");
        let schema_dir = ws_dir.join("default");
        std::fs::create_dir_all(schema_dir.join("types")).unwrap();
        // Override builtin default with a skeletal variant. collect_schema_source
        // must return this, not the 10-type builtin.
        write_schema(&schema_dir, "default", "1.0.0", &["spec"]);
        let schema_ref = SchemaRef::new("default", semver::Version::new(1, 0, 0));
        let files = collect_schema_source(Some(tmp.path()), Some(&ws_dir), &schema_ref).unwrap();
        let type_count = files
            .iter()
            .filter(|f| f.archive_path.starts_with("types/"))
            .count();
        assert_eq!(type_count, 1, "workspace override takes priority");
    }

    #[test]
    fn workspace_mismatched_version_errors() {
        let tmp = TempDir::new().unwrap();
        let ws_dir = tmp.path().join("schemas");
        let schema_dir = ws_dir.join("recipe");
        std::fs::create_dir_all(schema_dir.join("types")).unwrap();
        write_schema(&schema_dir, "recipe", "1.0.0", &["spec"]);
        let schema_ref = SchemaRef::new("recipe", semver::Version::new(2, 0, 0));
        let err = collect_schema_source(Some(tmp.path()), Some(&ws_dir), &schema_ref).unwrap_err();
        assert!(matches!(err, SchemaSourceError::VersionMismatch { .. }));
    }

    #[test]
    fn cache_schema_resolves_when_workspace_layer_absent() {
        let tmp = TempDir::new().unwrap();
        // Cache dir encodes version in the folder name: `<name>-<version>`.
        let dir = tmp.path().join(".memstead.cache/schemas/recipe-1.0.0");
        std::fs::create_dir_all(dir.join("types")).unwrap();
        write_schema(&dir, "recipe", "1.0.0", &["spec"]);
        let schema_ref = SchemaRef::new("recipe", semver::Version::new(1, 0, 0));
        let files = collect_schema_source(Some(tmp.path()), None, &schema_ref).unwrap();
        assert!(files.iter().any(|f| f.archive_path == "schema.yaml"));
    }

    #[test]
    fn unknown_schema_returns_not_found() {
        let schema_ref = SchemaRef::new("nonexistent", semver::Version::new(1, 0, 0));
        let err = collect_schema_source(None, None, &schema_ref).unwrap_err();
        assert!(matches!(err, SchemaSourceError::NotFound { .. }));
    }

    #[test]
    fn workspace_schema_wins_over_cache() {
        let tmp = TempDir::new().unwrap();
        // Cache layer carries a 2-type variant.
        let cache_dir = tmp.path().join(".memstead.cache/schemas/software-1.0.0");
        std::fs::create_dir_all(cache_dir.join("types")).unwrap();
        write_schema(&cache_dir, "software", "1.0.0", &["spec", "memo"]);
        // Workspace layer carries a 1-type variant — must win.
        let ws_dir = tmp.path().join("schemas");
        let ws_schema = ws_dir.join("software");
        std::fs::create_dir_all(ws_schema.join("types")).unwrap();
        write_schema(&ws_schema, "software", "1.0.0", &["spec"]);

        let schema_ref = SchemaRef::new("software", semver::Version::new(1, 0, 0));
        let files = collect_schema_source(Some(tmp.path()), Some(&ws_dir), &schema_ref).unwrap();
        let type_count = files
            .iter()
            .filter(|f| f.archive_path.starts_with("types/"))
            .count();
        assert_eq!(type_count, 1, "workspace layer must win over cache");
    }

    #[test]
    fn not_found_lists_every_candidate_path() {
        let tmp = TempDir::new().unwrap();
        let ws_dir = tmp.path().join("schemas");
        std::fs::create_dir_all(&ws_dir).unwrap();

        let schema_ref = SchemaRef::new("missing", semver::Version::new(2, 3, 4));
        let err = collect_schema_source(Some(tmp.path()), Some(&ws_dir), &schema_ref).unwrap_err();
        match err {
            SchemaSourceError::NotFound {
                schema_ref: name,
                candidates,
            } => {
                assert_eq!(name, "missing@2.3.4");
                // Every filesystem candidate listed in resolution order:
                // the versioned install shape first, then the bare
                // workspace schemas dir, then the workspace cache dir.
                assert_eq!(candidates.len(), 3);
                assert!(candidates[0].ends_with("schemas/missing@2.3.4"));
                assert!(candidates[1].ends_with("schemas/missing"));
                assert!(
                    candidates[2]
                        .to_string_lossy()
                        .contains(".memstead.cache/schemas/missing-2.3.4")
                );
            }
            other => panic!("expected NotFound, got {other:?}"),
        }
    }
}