car_inference/

service.rs

//! Inference service — exposes inference as built-in CAR tools.
//!
//! Provides `ToolSchema` definitions for `infer`, `embed`, and `classify`
//! so they can be registered as built-in tools with real implementations.

use car_ir::ToolSchema;
use serde_json::{json, Value};

use crate::{ClassifyRequest, EmbedRequest, GenerateRequest, InferenceEngine, InferenceError};

/// Execute an inference tool call. Returns the result as JSON.
///
/// This is the bridge between CAR's tool dispatch and the inference engine.
pub async fn execute_tool(
    engine: &InferenceEngine,
    tool_name: &str,
    params: &Value,
) -> Result<Value, InferenceError> {
    match tool_name {
        "infer" => {
            let req: GenerateRequest = serde_json::from_value(params.clone())
                .map_err(|e| InferenceError::InferenceFailed(format!("bad params: {e}")))?;
            let result = engine.generate(req).await?;
            Ok(json!({ "text": result }))
        }
        "embed" => {
            let req: EmbedRequest = serde_json::from_value(params.clone())
                .map_err(|e| InferenceError::InferenceFailed(format!("bad params: {e}")))?;
            let result = engine.embed(req).await?;
            Ok(json!({ "embeddings": result }))
        }
        "classify" => {
            let req: ClassifyRequest = serde_json::from_value(params.clone())
                .map_err(|e| InferenceError::InferenceFailed(format!("bad params: {e}")))?;
            let result = engine.classify(req).await?;
            Ok(json!({ "classifications": result }))
        }
        _ => Err(InferenceError::InferenceFailed(format!(
            "unknown inference tool: {tool_name}"
        ))),
    }
}

/// ToolSchema for text generation.
pub fn infer_schema() -> ToolSchema {
    ToolSchema {
        name: "infer".to_string(),
        description: "Generate text using a local Qwen3 model.".to_string(),
        parameters: json!({
            "type": "object",
            "properties": {
                "prompt": {
                    "type": "string",
                    "description": "The prompt to complete"
                },
                "model": {
                    "type": "string",
                    "description": "Model name (default: Qwen3-1.7B)"
                },
                "context": {
                    "type": "string",
                    "description": "Optional memory context to ground the model's response"
                },
                "params": {
                    "type": "object",
                    "properties": {
                        "temperature": { "type": "number", "default": 0.7 },
                        "top_p": { "type": "number", "default": 0.9 },
                        "top_k": { "type": "integer", "default": 0 },
                        "max_tokens": { "type": "integer", "default": 512 },
                        "stop": {
                            "type": "array",
                            "items": { "type": "string" }
                        }
                    }
                }
            },
            "required": ["prompt"]
        }),
        returns: Some(json!({
            "type": "object",
            "properties": {
                "text": { "type": "string" }
            }
        })),
        idempotent: false,
        cache_ttl_secs: None,
        rate_limit: None,
    }
}

/// ToolSchema for embedding generation.
pub fn embed_schema() -> ToolSchema {
    ToolSchema {
        name: "embed".to_string(),
        description: "Generate vector embeddings for text using a local Qwen3 model.".to_string(),
        parameters: json!({
            "type": "object",
            "properties": {
                "texts": {
                    "type": "array",
                    "items": { "type": "string" },
                    "description": "Texts to embed"
                },
                "model": {
                    "type": "string",
                    "description": "Model name (default: Qwen3-0.6B)"
                }
            },
            "required": ["texts"]
        }),
        returns: Some(json!({
            "type": "object",
            "properties": {
                "embeddings": {
                    "type": "array",
                    "items": {
                        "type": "array",
                        "items": { "type": "number" }
                    }
                }
            }
        })),
        idempotent: true,
        cache_ttl_secs: Some(3600),
        rate_limit: None,
    }
}

/// ToolSchema for classification.
pub fn classify_schema() -> ToolSchema {
    ToolSchema {
        name: "classify".to_string(),
        description: "Classify text against candidate labels using a local Qwen3 model."
            .to_string(),
        parameters: json!({
            "type": "object",
            "properties": {
                "text": {
                    "type": "string",
                    "description": "Text to classify"
                },
                "labels": {
                    "type": "array",
                    "items": { "type": "string" },
                    "description": "Candidate labels"
                },
                "model": {
                    "type": "string",
                    "description": "Model name (default: Qwen3-0.6B)"
                }
            },
            "required": ["text", "labels"]
        }),
        returns: Some(json!({
            "type": "object",
            "properties": {
                "classifications": {
                    "type": "array",
                    "items": {
                        "type": "object",
                        "properties": {
                            "label": { "type": "string" },
                            "score": { "type": "number" }
                        }
                    }
                }
            }
        })),
        idempotent: true,
        cache_ttl_secs: Some(300),
        rate_limit: None,
    }
}

/// ToolSchema for memory-grounded text generation.
pub fn infer_grounded_schema() -> ToolSchema {
    ToolSchema {
        name: "infer.grounded".to_string(),
        description: "Generate text grounded with memory context. Automatically queries the memgine for relevant context before generating.".to_string(),
        parameters: json!({
            "type": "object",
            "properties": {
                "prompt": {
                    "type": "string",
                    "description": "The prompt to complete"
                },
                "model": {
                    "type": "string",
                    "description": "Model name (default: Qwen3-1.7B)"
                },
                "params": {
                    "type": "object",
                    "properties": {
                        "temperature": { "type": "number", "default": 0.7 },
                        "top_p": { "type": "number", "default": 0.9 },
                        "top_k": { "type": "integer", "default": 0 },
                        "max_tokens": { "type": "integer", "default": 512 },
                        "stop": {
                            "type": "array",
                            "items": { "type": "string" }
                        }
                    }
                }
            },
            "required": ["prompt"]
        }),
        returns: Some(json!({
            "type": "object",
            "properties": {
                "text": { "type": "string" }
            }
        })),
        idempotent: false,
        cache_ttl_secs: None,
        rate_limit: None,
    }
}

/// ToolSchema for model management — list models in the unified registry.
pub fn list_models_schema() -> ToolSchema {
    ToolSchema {
        name: "models.list".to_string(),
        description: "List all registered models (local and remote) with their capabilities, availability, and performance profiles.".to_string(),
        parameters: json!({
            "type": "object",
            "properties": {
                "capability": {
                    "type": "string",
                    "description": "Filter by capability (generate, embed, classify, code, reasoning, summarize, tool_use, vision)"
                },
                "local_only": {
                    "type": "boolean",
                    "description": "Only show local models"
                },
                "available_only": {
                    "type": "boolean",
                    "description": "Only show available models"
                }
            }
        }),
        returns: Some(json!({
            "type": "object",
            "properties": {
                "models": {
                    "type": "array",
                    "items": {
                        "type": "object",
                        "properties": {
                            "id": { "type": "string" },
                            "name": { "type": "string" },
                            "provider": { "type": "string" },
                            "capabilities": { "type": "array", "items": { "type": "string" } },
                            "available": { "type": "boolean" },
                            "is_local": { "type": "boolean" }
                        }
                    }
                }
            }
        })),
        idempotent: true,
        cache_ttl_secs: Some(60),
        rate_limit: None,
    }
}

/// ToolSchema for model routing — show which model would be selected for a prompt.
pub fn route_model_schema() -> ToolSchema {
    ToolSchema {
        name: "models.route".to_string(),
        description: "Route a prompt to the best model without executing. Shows the routing decision, strategy, and fallback chain.".to_string(),
        parameters: json!({
            "type": "object",
            "properties": {
                "prompt": {
                    "type": "string",
                    "description": "The prompt to route"
                }
            },
            "required": ["prompt"]
        }),
        returns: Some(json!({
            "type": "object",
            "properties": {
                "model_id": { "type": "string" },
                "model_name": { "type": "string" },
                "strategy": { "type": "string" },
                "complexity": { "type": "string" },
                "predicted_quality": { "type": "number" },
                "reason": { "type": "string" },
                "fallbacks": { "type": "array", "items": { "type": "string" } }
            }
        })),
        idempotent: true,
        cache_ttl_secs: None,
        rate_limit: None,
    }
}

/// ToolSchema for model performance stats.
pub fn model_stats_schema() -> ToolSchema {
    ToolSchema {
        name: "models.stats".to_string(),
        description: "Get performance statistics for models based on observed outcomes.".to_string(),
        parameters: json!({
            "type": "object",
            "properties": {
                "model_id": {
                    "type": "string",
                    "description": "Model ID to get stats for (omit for all models)"
                }
            }
        }),
        returns: Some(json!({
            "type": "object",
            "properties": {
                "profiles": {
                    "type": "array",
                    "items": {
                        "type": "object",
                        "properties": {
                            "model_id": { "type": "string" },
                            "total_calls": { "type": "integer" },
                            "success_rate": { "type": "number" },
                            "avg_latency_ms": { "type": "number" },
                            "ema_quality": { "type": "number" }
                        }
                    }
                }
            }
        })),
        idempotent: true,
        cache_ttl_secs: Some(30),
        rate_limit: None,
    }
}

/// All inference tool schemas.
pub fn all_schemas() -> Vec<ToolSchema> {
    vec![
        infer_schema(),
        infer_grounded_schema(),
        embed_schema(),
        classify_schema(),
        list_models_schema(),
        route_model_schema(),
        model_stats_schema(),
    ]
}