ai_tokenopt 0.5.10

#![forbid(unsafe_code)]
//! Token Optimization Engine for PiSovereign
//!
//! Adaptively compresses the full inference pipeline — input prompts,
//! conversation history, RAG context, tool definitions, tool results,
//! and output streams — to minimize token usage while preserving response
//! quality. Operates as a decorator in the inference chain.
//!
//! # Architecture
//!
//! The optimizer sits inside the sanitization decorator:
//!
//! ```text
//! SanitizedInferencePort  →  TokenOptimizedInferencePort  →  Cache  →  Ollama
//! ```
//!
//! # Strategy
//!
//! Uses an adaptive approach: lossless compression when within budget,
//! progressively lossy (rolling summaries, extractive truncation) under
//! token pressure. Falls through transparently on any error.

pub mod budget;
pub mod config;
#[cfg(feature = "pisovereign")]
pub mod decorator;
pub mod error;
pub mod estimator;
pub mod history;
pub mod optimizer;
pub mod ports;
pub mod prompt;
pub mod stream;
pub mod tools;
pub mod types;

#[cfg(feature = "hf-tokenizer")]
pub mod estimator_hf;
pub mod estimator_language;
pub mod estimator_tuning;
pub mod metrics;
pub mod output;
pub mod pipeline;
pub mod profile;

// ollama feature — built-in HTTP summarizer
#[cfg(feature = "ollama")]
pub mod ollama_summarizer;

pub use config::TokenOptimizationConfig;
#[cfg(feature = "pisovereign")]
pub use decorator::TokenOptimizedInferencePort;
pub use error::TokenOptError;
pub use estimator::TokenEstimator;
#[cfg(feature = "hf-tokenizer")]
pub use estimator_hf::HfTokenEstimator;
pub use optimizer::TokenOptimizer;
pub use ports::SummarizationPort;
pub use prompt::template_loader::TemplateLoader;

// Build-time generated YAML prompt constants
include!(concat!(env!("OUT_DIR"), "/prompts.rs"));

#[cfg(feature = "ollama")]
pub use ollama_summarizer::OllamaSummarizer;
pub use pipeline::Pipeline;
pub use types::{OptimizationMetadata, OptimizedPrompt};