memstead_schema/

loader.rs

//! Schema loader — reads on-disk schema directories (or in-memory YAML) into
//! validated `Schema` values with `edge_weights` resolved.
//!
//! Three validation layers coordinate here:
//! 1. Structural (serde + `deny_unknown_fields`) — handled by the deserialize.
//! 2. Semantic (this module) — cross-field rules listed in `SchemaLoadError`.
//! 3. Editor (JSON Schemas) — generated by `emit_json_schemas`, consumed by
//!    schema authors via `# yaml-language-server: $schema=...`.

use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;

use indexmap::IndexMap;
use thiserror::Error;

use crate::base_metadata;
use crate::manifest::SchemaManifest;
use crate::schema::Schema;
use crate::types::TypeDefinition;

#[derive(Debug, Error)]
pub enum SchemaLoadError {
    #[error("i/o error reading {}: {source}", .path.display())]
    Io {
        path: PathBuf,
        #[source]
        source: std::io::Error,
    },

    #[error("failed to parse manifest {}: {source}", .path.display())]
    ParseManifest {
        path: PathBuf,
        #[source]
        source: serde_yaml_ng::Error,
    },

    #[error("failed to parse type file {}: {source}", .path.display())]
    ParseType {
        path: PathBuf,
        #[source]
        source: serde_yaml_ng::Error,
    },

    #[error("invalid version '{value}': must be semver (e.g. 1.0.0)")]
    InvalidVersion { value: String },

    #[error("invalid schema name '{value}': {reason}")]
    InvalidName { value: String, reason: &'static str },

    #[error(
        "schema type file mismatch — declared in manifest: [{}], found in types/: [{}]",
        declared.join(", "),
        found.join(", ")
    )]
    TypeFileMismatch {
        declared: Vec<String>,
        found: Vec<String>,
    },

    #[error(
        "type file '{file}.yaml' has `name: {declared}` — filename and `name` field must match"
    )]
    TypeNameMismatch { file: String, declared: String },

    #[error("schema relationship vocabulary must include a '_default' definition")]
    MissingDefaultWeight,

    #[error("duplicate relationship definition: '{name}'")]
    DuplicateRelationship { name: String },

    #[error(
        "type '{type_name}' references relationship '{relationship}' in field '{field}' — not declared in schema. Available: [{}]. {}",
        available.join(", "),
        format_suggestion(relationship, available)
    )]
    UndeclaredRelationship {
        type_name: String,
        field: &'static str,
        relationship: String,
        available: Vec<String>,
    },

    #[error(
        "type '{type_name}' must have exactly one section with `catch_all: true` (found {count})"
    )]
    CatchAllViolation { type_name: String, count: usize },

    #[error(
        "type '{type_name}' field '{field}' references unknown key '{reference}' — not a section or metadata field"
    )]
    UnknownFieldReference {
        type_name: String,
        field: &'static str,
        reference: String,
    },

    #[error(
        "type '{type_name}' metadata field '{field}' default '{default}' is not listed in enum_values: [{}]",
        allowed.join(", ")
    )]
    DefaultValueNotInEnum {
        type_name: String,
        field: String,
        default: String,
        allowed: Vec<String>,
    },

    #[error(
        "type '{type_name}' redeclares engine-implicit metadata key '{field}' — remove it from the YAML; the loader injects it automatically"
    )]
    RedeclaredBaseField { type_name: String, field: String },

    #[error(
        "relationship '{relationship}' field '{field}' references unknown type '{reference}'. Declared types: [{}]. {}",
        declared.join(", "),
        format_suggestion(reference, declared)
    )]
    UndeclaredRelationshipType {
        relationship: String,
        field: &'static str,
        reference: String,
        declared: Vec<String>,
    },

    /// Schema declares a regular section or metadata field whose key
    /// collides with an engine-invariant key. The reserved set covers
    /// section key `relationships` (the parser's auto-managed
    /// `## Relationships` section) and metadata field key `type` (the
    /// engine-set frontmatter type discriminator).
    #[error(
        "type '{type_name}' declares {kind} with reserved key '{offending_key}' — reserved keys: [{}]",
        reserved_keys.join(", ")
    )]
    ReservedSchemaKey {
        type_name: String,
        kind: &'static str,
        offending_key: String,
        reserved_keys: Vec<String>,
    },

    /// A `cross_mem_relationships:` entry's `to_schema:` field is
    /// not a bare schema name. Cross-mem eligibility is name-based —
    /// versioned (`software@1.0.0`) and range (`software@^1.0`) forms
    /// are refused so a version component can never silently re-enter
    /// the eligibility path.
    #[error(
        "cross_mem_relationships[].to_schema '{value}' {reason} — expected a bare schema name (e.g. 'software', not 'software@1.0.0')"
    )]
    InvalidCrossMemToSchema { value: String, reason: String },

    /// Two `cross_mem_relationships:` entries declare the same
    /// `to_schema:`. A schema declares each target-schema at most once
    /// — the second entry would otherwise silently shadow or split
    /// the vocabulary.
    #[error("cross_mem_relationships declares duplicate to_schema '{to_schema}'")]
    DuplicateCrossMemToSchema { to_schema: String },

    /// A `cross_mem_relationships[].definitions[*].source_types` entry
    /// references a type name not declared in the source schema's
    /// `types` list. Source types belong to the source schema's
    /// namespace; unknown names raise this error at load time.
    /// (Target types are accepted as opaque strings — they belong to
    /// the target schema's namespace, which is not in scope here.)
    #[error(
        "cross_mem_relationships[to_schema='{to_schema}'].definitions[name='{relationship}'].source_types references unknown type '{reference}'. Declared types: [{}]. {}",
        declared.join(", "),
        format_suggestion(reference, declared)
    )]
    UndeclaredCrossMemSourceType {
        to_schema: String,
        relationship: String,
        reference: String,
        declared: Vec<String>,
    },

    /// The schema's `alias_target_rel_type:` pointer names a rel-type
    /// not declared in `relationships.definitions`. Surfaces at
    /// schema-load time so the alias-synthesis pass can trust the
    /// pointer is resolvable at every later mutation call.
    #[error(
        "schema '{schema}' alias_target_rel_type '{target}' is not declared in relationships. Declared: [{}]. {}",
        declared.join(", "),
        format_suggestion(target, declared)
    )]
    AliasTargetRelTypeNotDeclared {
        schema: String,
        target: String,
        declared: Vec<String>,
    },
}

/// Engine-invariant section keys reserved against schema use. The
/// parser's auto-managed `## Relationships` section is the only entry
/// today.
pub fn reserved_section_keys() -> &'static [&'static str] {
    &["relationships"]
}

/// Engine-invariant metadata-field keys reserved against schema use.
/// `type` is the engine-set frontmatter discriminator.
pub fn reserved_metadata_field_keys() -> &'static [&'static str] {
    &["type"]
}

fn format_suggestion(needle: &str, candidates: &[String]) -> String {
    let mut best: Option<(usize, &String)> = None;
    for cand in candidates {
        let d = strsim::levenshtein(needle, cand);
        match best {
            Some((bd, _)) if bd <= d => {}
            _ => best = Some((d, cand)),
        }
    }
    match best {
        Some((d, cand)) if d > 0 && d <= needle.len().saturating_add(3) => {
            format!("Did you mean '{cand}'?")
        }
        _ => String::new(),
    }
}

/// Load a schema from a directory containing `schema.yaml` and `types/*.yaml`.
pub fn load_schema_from_dir(path: &Path) -> Result<Schema, SchemaLoadError> {
    let manifest_path = path.join("schema.yaml");
    let manifest_text =
        std::fs::read_to_string(&manifest_path).map_err(|e| SchemaLoadError::Io {
            path: manifest_path.clone(),
            source: e,
        })?;

    let types_dir = path.join("types");
    let mut type_files: Vec<(String, String)> = Vec::new();
    if types_dir.is_dir() {
        let entries = std::fs::read_dir(&types_dir).map_err(|e| SchemaLoadError::Io {
            path: types_dir.clone(),
            source: e,
        })?;
        for entry in entries {
            let entry = entry.map_err(|e| SchemaLoadError::Io {
                path: types_dir.clone(),
                source: e,
            })?;
            let p = entry.path();
            if p.extension().and_then(|s| s.to_str()) != Some("yaml") {
                continue;
            }
            let Some(stem) = p.file_stem().and_then(|s| s.to_str()).map(str::to_owned) else {
                continue;
            };
            let contents = std::fs::read_to_string(&p).map_err(|e| SchemaLoadError::Io {
                path: p.clone(),
                source: e,
            })?;
            type_files.push((stem, contents));
        }
    }

    load_with_context(
        &manifest_text,
        &type_files,
        Some(&manifest_path),
        Some(&types_dir),
    )
}

/// Load a schema from in-memory YAML strings.
///
/// `types_yamls` is a slice of `(filename_stem, contents)` tuples — the stems
/// must match `manifest.types` exactly.
pub fn load_schema_from_memory(
    manifest_yaml: &str,
    types_yamls: &[(String, String)],
) -> Result<Schema, SchemaLoadError> {
    load_with_context(manifest_yaml, types_yamls, None, None)
}

fn load_with_context(
    manifest_yaml: &str,
    types_yamls: &[(String, String)],
    manifest_path: Option<&Path>,
    types_dir: Option<&Path>,
) -> Result<Schema, SchemaLoadError> {
    let mut manifest: SchemaManifest =
        serde_yaml_ng::from_str(manifest_yaml).map_err(|e| SchemaLoadError::ParseManifest {
            path: manifest_path
                .map(Path::to_path_buf)
                .unwrap_or_else(|| PathBuf::from("<memory>")),
            source: e,
        })?;

    validate_name(&manifest.name)?;

    let version =
        semver::Version::parse(&manifest.version).map_err(|_| SchemaLoadError::InvalidVersion {
            value: manifest.version.clone(),
        })?;

    // Relationship vocabulary: unique names + _default present
    let mut rel_names: HashSet<String> = HashSet::new();
    for def in &manifest.relationships.definitions {
        if !rel_names.insert(def.name.clone()) {
            return Err(SchemaLoadError::DuplicateRelationship {
                name: def.name.clone(),
            });
        }
    }
    if !rel_names.contains("_default") {
        return Err(SchemaLoadError::MissingDefaultWeight);
    }
    let available_rels: Vec<String> = manifest
        .relationships
        .definitions
        .iter()
        .map(|d| d.name.clone())
        .collect();

    // Validate that the schema-level alias_target_rel_type pointer (if
    // set) names a declared rel-type. The synthesis pass later relies
    // on this invariant — running it as a load-time check keeps the
    // mutation path's hot loop free of resolution failures.
    if let Some(target) = &manifest.alias_target_rel_type
        && !rel_names.contains(target)
    {
        let mut declared = available_rels.clone();
        declared.sort();
        return Err(SchemaLoadError::AliasTargetRelTypeNotDeclared {
            schema: manifest.name.clone(),
            target: target.clone(),
            declared,
        });
    }

    // Option C coupling — auto-force `manual_authoring: forbidden` on
    // the rel-type named by `alias_target_rel_type`. Schemas setting
    // the pointer opt the named rel-type out of explicit authoring;
    // the only path to a relation of that rel-type is via the
    // alias-synthesis pass that emits one per body wiki-link. This
    // closes the explicit/synthesised coexistence question: with the
    // coupling in place, edges of the pointer rel-type are always
    // engine-emitted, so `EdgeSource::BodyLink` is unambiguous and
    // GC can drop pointer-rel-type relations without risking
    // explicit-author data.
    //
    // The coupling is silent — a schema that writes
    // `manual_authoring: allow` (or `warn`) on the named rel-type
    // gets overridden to `forbidden` at load. The override is the
    // schema-strictness contract; explicit `allow`/`warn` on the
    // pointer rel-type is meaningless under the design and would
    // surprise the validator at runtime, so the loader corrects it
    // here.
    if let Some(pointer) = manifest.alias_target_rel_type.clone() {
        for def in &mut manifest.relationships.definitions {
            if def.name == pointer {
                def.manual_authoring = crate::manifest::ManualAuthoring::Forbidden;
            }
        }
    }

    // Cross-check `source_types` / `target_types` on each relationship
    // definition against the manifest's declared type list. Unknown
    // names raise `UndeclaredRelationshipType` with a "did you mean"
    // suggestion — the schema-author equivalent of `INVALID_REL_SHAPE`.
    for def in &manifest.relationships.definitions {
        for t in &def.source_types {
            if !manifest.types.iter().any(|d| d == t) {
                return Err(SchemaLoadError::UndeclaredRelationshipType {
                    relationship: def.name.clone(),
                    field: "source_types",
                    reference: t.clone(),
                    declared: manifest.types.clone(),
                });
            }
        }
        for t in &def.target_types {
            if !manifest.types.iter().any(|d| d == t) {
                return Err(SchemaLoadError::UndeclaredRelationshipType {
                    relationship: def.name.clone(),
                    field: "target_types",
                    reference: t.clone(),
                    declared: manifest.types.clone(),
                });
            }
        }
    }

    // Cross-mem relationships: validate `to_schema` is a bare schema
    // name (cross-mem eligibility is name-based — a version suffix or
    // range refuses), refuse duplicate target schemas, and cross-check
    // `source_types` against the source schema's types. `target_types`
    // are accepted as opaque strings — they belong to the target
    // schema's namespace, which is out of scope at source-schema load
    // time. The target schema may not even be present in the workspace
    // when the source schema loads (and cross-mem declarations
    // targeting absent schemas are legitimate for portable library
    // schemas).
    let mut seen_to_schemas: HashSet<String> = HashSet::new();
    for entry in &manifest.cross_mem_relationships {
        if entry.to_schema.contains('@') {
            return Err(SchemaLoadError::InvalidCrossMemToSchema {
                value: entry.to_schema.clone(),
                reason: "must not carry a version or range".into(),
            });
        }
        if let Err(reason) = name_shape(&entry.to_schema) {
            return Err(SchemaLoadError::InvalidCrossMemToSchema {
                value: entry.to_schema.clone(),
                reason: reason.into(),
            });
        }
        if !seen_to_schemas.insert(entry.to_schema.clone()) {
            return Err(SchemaLoadError::DuplicateCrossMemToSchema {
                to_schema: entry.to_schema.clone(),
            });
        }
        for def in &entry.definitions {
            for t in &def.source_types {
                if !manifest.types.iter().any(|d| d == t) {
                    return Err(SchemaLoadError::UndeclaredCrossMemSourceType {
                        to_schema: entry.to_schema.clone(),
                        relationship: def.name.clone(),
                        reference: t.clone(),
                        declared: manifest.types.clone(),
                    });
                }
            }
        }
    }

    // Type file cross-check: declared vs found, set equality (order-insensitive)
    let mut found_stems: Vec<String> = types_yamls.iter().map(|(s, _)| s.clone()).collect();
    found_stems.sort();
    let mut declared = manifest.types.clone();
    declared.sort();
    if found_stems != declared {
        return Err(SchemaLoadError::TypeFileMismatch {
            declared,
            found: found_stems,
        });
    }

    // Per-type defaults map for edge_weights resolution
    let defaults: IndexMap<String, f32> = manifest
        .relationships
        .definitions
        .iter()
        .map(|d| (d.name.clone(), d.default_weight))
        .collect();

    let mut types_map: HashMap<String, Arc<TypeDefinition>> = HashMap::new();

    for (stem, text) in types_yamls {
        let type_path = types_dir
            .map(|d| d.join(format!("{stem}.yaml")))
            .unwrap_or_else(|| PathBuf::from(format!("<memory>/{stem}.yaml")));

        let mut td: TypeDefinition =
            serde_yaml_ng::from_str(text).map_err(|e| SchemaLoadError::ParseType {
                path: type_path.clone(),
                source: e,
            })?;

        if td.name != *stem {
            return Err(SchemaLoadError::TypeNameMismatch {
                file: stem.clone(),
                declared: td.name.clone(),
            });
        }

        // Reserved-metadata-field-key check. `type` is the engine-set
        // frontmatter discriminator and must never appear in a schema's
        // metadata field list. Runs before the base-metadata merge so
        // the engine-injected fields don't false-positive here.
        for field in &td.metadata_fields {
            if reserved_metadata_field_keys().contains(&field.key.as_str()) {
                return Err(SchemaLoadError::ReservedSchemaKey {
                    type_name: td.name.clone(),
                    kind: "metadata_field",
                    offending_key: field.key.clone(),
                    reserved_keys: reserved_metadata_field_keys()
                        .iter()
                        .map(|s| s.to_string())
                        .collect(),
                });
            }
        }

        // Reject redeclarations of remaining engine-implicit base metadata
        // (`created_date`, `last_modified`, `tags`). The reserved `type`
        // case was already handled above with the typed reserved-key
        // error.
        for field in &td.metadata_fields {
            if base_metadata::is_base_key(&field.key)
                && !reserved_metadata_field_keys().contains(&field.key.as_str())
            {
                return Err(SchemaLoadError::RedeclaredBaseField {
                    type_name: td.name.clone(),
                    field: field.key.clone(),
                });
            }
        }

        // Merge base metadata around the type-declared fields. Canonical
        // order: type, created_date, last_modified, <declared>, tags.
        let mut merged = base_metadata::prefix_fields();
        merged.append(&mut td.metadata_fields);
        merged.extend(base_metadata::suffix_fields());
        td.metadata_fields = merged;

        validate_type(&td, &rel_names, &available_rels)?;

        // Resolve edge_weights: start with schema defaults, apply overrides.
        let mut weights = defaults.clone();
        for (k, v) in &td.edge_weight_overrides {
            weights.insert(k.clone(), *v);
        }
        td.edge_weights = weights;

        types_map.insert(stem.clone(), Arc::new(td));
    }

    Ok(Schema {
        manifest,
        version,
        types: types_map,
    })
}

fn validate_name(name: &str) -> Result<(), SchemaLoadError> {
    name_shape(name).map_err(|reason| SchemaLoadError::InvalidName {
        value: name.into(),
        reason,
    })
}

/// Author-time access to the schema-name shape rule — the same check
/// the loader runs on a manifest's `name:`. Exposed so scaffolding
/// tooling (`memstead schema new`) can refuse a bad name up front with
/// the loader's own reason string instead of a drifting copy of the
/// grammar.
pub fn validate_schema_name(name: &str) -> Result<(), &'static str> {
    name_shape(name)
}

/// Shared shape rule for schema names — the manifest's own `name:` and
/// every `cross_mem_relationships[].to_schema` follow the same
/// grammar; the two callers wrap violations in their field-specific
/// error variants.
fn name_shape(name: &str) -> Result<(), &'static str> {
    if name.is_empty() {
        return Err("must not be empty");
    }
    let mut chars = name.chars();
    let first = chars.next().unwrap();
    if !first.is_ascii_lowercase() {
        return Err("must start with a lowercase letter");
    }
    for c in chars {
        if !(c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') {
            return Err("must contain only lowercase letters, digits, and hyphens");
        }
    }
    Ok(())
}

fn validate_type(
    td: &TypeDefinition,
    rel_names: &HashSet<String>,
    available_rels: &[String],
) -> Result<(), SchemaLoadError> {
    // Reserved-section check. Section key `relationships` collides
    // with the parser's auto-managed `## Relationships` section.
    // Domain conventions (`identity`, `purpose`, ...) are NOT reserved.
    // The reserved-metadata-key check runs earlier in
    // `load_with_context` against the *raw* author-declared field list
    // so the base-metadata merge doesn't false-positive against
    // engine-injected keys.
    for section in &td.sections {
        if reserved_section_keys().contains(&section.key.as_str()) {
            return Err(SchemaLoadError::ReservedSchemaKey {
                type_name: td.name.clone(),
                kind: "section",
                offending_key: section.key.clone(),
                reserved_keys: reserved_section_keys()
                    .iter()
                    .map(|s| s.to_string())
                    .collect(),
            });
        }
    }

    check_rel(
        &td.name,
        "hierarchy_relationship",
        &td.hierarchy_relationship,
        rel_names,
        available_rels,
    )?;
    for r in &td.propagating_relationships {
        check_rel(
            &td.name,
            "propagating_relationships",
            r,
            rel_names,
            available_rels,
        )?;
    }
    for r in td.edge_weight_overrides.keys() {
        check_rel(
            &td.name,
            "edge_weight_overrides",
            r,
            rel_names,
            available_rels,
        )?;
    }
    for block in &td.required_outgoing {
        for r in &block.relationships {
            check_rel(&td.name, "required_outgoing", r, rel_names, available_rels)?;
        }
    }

    // Exactly one catch_all section
    let catch_all_count = td.sections.iter().filter(|s| s.catch_all).count();
    if catch_all_count != 1 {
        return Err(SchemaLoadError::CatchAllViolation {
            type_name: td.name.clone(),
            count: catch_all_count,
        });
    }

    // Field-reference integrity
    let section_keys: HashSet<&str> = td.sections.iter().map(|s| s.key.as_str()).collect();
    let meta_keys: HashSet<&str> = td.metadata_fields.iter().map(|m| m.key.as_str()).collect();

    for f in &td.text_fields {
        // text_fields point at section content — not metadata.
        if !section_keys.contains(f.as_str()) {
            return Err(SchemaLoadError::UnknownFieldReference {
                type_name: td.name.clone(),
                field: "text_fields",
                reference: f.clone(),
            });
        }
    }
    for f in &td.health_required_fields {
        if !section_keys.contains(f.as_str()) && !meta_keys.contains(f.as_str()) {
            return Err(SchemaLoadError::UnknownFieldReference {
                type_name: td.name.clone(),
                field: "health_required_fields",
                reference: f.clone(),
            });
        }
    }
    for f in &td.updatable_fields {
        // `title` is the entity's filename-derived title — always updatable.
        if f == "title" {
            continue;
        }
        if !section_keys.contains(f.as_str()) && !meta_keys.contains(f.as_str()) {
            return Err(SchemaLoadError::UnknownFieldReference {
                type_name: td.name.clone(),
                field: "updatable_fields",
                reference: f.clone(),
            });
        }
    }

    // Metadata default_value must be a member of enum_values when both present.
    for m in &td.metadata_fields {
        if let (Some(default), Some(allowed)) = (m.default_value.as_ref(), m.enum_values.as_ref())
            && !allowed.contains(default)
        {
            return Err(SchemaLoadError::DefaultValueNotInEnum {
                type_name: td.name.clone(),
                field: m.key.clone(),
                default: default.clone(),
                allowed: allowed.clone(),
            });
        }
    }

    Ok(())
}

fn check_rel(
    type_name: &str,
    field: &'static str,
    relationship: &str,
    rel_names: &HashSet<String>,
    available: &[String],
) -> Result<(), SchemaLoadError> {
    if rel_names.contains(relationship) {
        return Ok(());
    }
    Err(SchemaLoadError::UndeclaredRelationship {
        type_name: type_name.into(),
        field,
        relationship: relationship.into(),
        available: available.to_vec(),
    })
}