ai_tokenopt 0.5.7

Adaptive token optimization engine for LLM inference pipelines — compresses prompts, conversation history, tool schemas, and output streams to minimize token usage while preserving response quality.
Documentation
//! Build script for ai_tokenopt — generates YAML prompt constants.
//!
//! Scans `prompts/*.prompt.txt` at build time and generates a Rust
//! source file containing pre-converted YAML versions as static strings.

use std::env;
use std::fs;
use std::io::Write;
use std::path::Path;

#[allow(clippy::expect_used)]
fn main() {
    let out_dir = env::var("OUT_DIR").expect("OUT_DIR not set");
    let dest = Path::new(&out_dir).join("prompts.rs");

    let prompts_dir = Path::new("prompts");
    let mut entries: Vec<(String, String)> = Vec::new();

    // Re-run if the prompts directory changes or is created
    println!("cargo:rerun-if-changed=prompts/");

    if prompts_dir.is_dir() {
        if let Ok(dir) = fs::read_dir(prompts_dir) {
            for entry in dir.flatten() {
                let path = entry.path();
                let is_prompt = path
                    .file_name()
                    .and_then(|n| n.to_str())
                    .is_some_and(|n| n.ends_with(".prompt.txt"));

                if is_prompt {
                    if let Ok(content) = fs::read_to_string(&path) {
                        let stem = path
                            .file_stem()
                            .and_then(|s| s.to_str())
                            .unwrap_or("unknown");
                        let name = stem.strip_suffix(".prompt").unwrap_or(stem);
                        entries.push((name.to_string(), prose_to_yaml_build(&content)));
                    }
                }
            }
        }
    }

    // Sort for deterministic output
    entries.sort_by(|a, b| a.0.cmp(&b.0));

    let mut file = fs::File::create(dest).expect("failed to create prompts.rs");
    let _ = writeln!(
        file,
        "/// Pre-converted YAML prompts generated at build time."
    );
    let _ = writeln!(file, "///");
    let _ = writeln!(
        file,
        "/// Each entry is `(name, yaml_content)` from `prompts/*.prompt.txt`."
    );
    let _ = write!(file, "pub const YAML_PROMPTS: &[(&str, &str)] = &[");

    for (name, yaml) in &entries {
        let escaped_yaml = yaml.replace('\\', "\\\\").replace('"', "\\\"");
        let _ = write!(file, "\n    (\"{name}\", \"{escaped_yaml}\"),");
    }

    let _ = writeln!(file, "\n];");
}

/// Minimal prose-to-YAML converter for build time.
///
/// Splits sections by double newline, detects headings (lines ending with `:`
/// or starting with `#`), and converts bullet lists to YAML arrays.
fn prose_to_yaml_build(text: &str) -> String {
    let sections: Vec<&str> = text.split("\n\n").collect();
    let mut yaml_parts = Vec::new();

    for section in sections {
        let trimmed = section.trim();
        if trimmed.is_empty() {
            continue;
        }

        let lines: Vec<&str> = trimmed.lines().collect();
        if lines.is_empty() {
            continue;
        }

        let first = lines[0].trim();
        let heading = detect_heading(first);

        if let Some(heading) = heading {
            let body_lines = &lines[1..];
            let all_bullets = !body_lines.is_empty()
                && body_lines
                    .iter()
                    .all(|l| l.trim().starts_with("- ") || l.trim().is_empty());

            if all_bullets {
                yaml_parts.push(format!("{heading}:"));
                for line in body_lines {
                    let t = line.trim();
                    if !t.is_empty() {
                        yaml_parts.push(format!("  {t}"));
                    }
                }
            } else {
                let body = body_lines.join("\\n");
                yaml_parts.push(format!("{heading}: \"{body}\""));
            }
        } else {
            // Plain paragraph — quote it
            let escaped = trimmed.replace('\n', "\\n");
            yaml_parts.push(format!("content: \"{escaped}\""));
        }
    }

    yaml_parts.join("\\n")
}

/// Detect a section heading from a line.
///
/// Returns the heading text if the line starts with `#`/`##` or ends with `:`.
fn detect_heading(line: &str) -> Option<&str> {
    if let Some(h) = line.strip_prefix("# ") {
        return Some(h.trim_end_matches(':'));
    }
    if let Some(h) = line.strip_prefix("## ") {
        return Some(h.trim_end_matches(':'));
    }
    if line.ends_with(':') && !line.contains('.') {
        return Some(line.trim_end_matches(':'));
    }
    None
}