harn-vm 0.10.153

//! Text completion / fill-in-the-middle dispatch. Completion providers
//! fall into three buckets: OpenAI-style `/completions`, Ollama's
//! `/api/generate`, and a chat-based fallback that wraps the prompt in a
//! user message and re-enters the chat path.

use crate::value::VmDictExt;

use crate::value::{VmError, VmValue};

use super::auth::apply_auth_headers;
use super::openai_normalize::extract_openai_choice_logprobs;
use super::options::LlmCallOptions;
use super::response::{extract_cache_read_tokens, extract_cache_write_tokens};
use super::result::{mock_completion_response, LlmResult};
use super::telemetry::{source as telemetry_source, ProviderTelemetry};

async fn completion_json_response(
    provider: &str,
    response: reqwest::Response,
) -> Result<serde_json::Value, VmError> {
    if !response.status().is_success() {
        let status = response.status();
        let headers = response.headers().clone();
        let body = response.text().await.unwrap_or_default();
        return Err(super::provider_http_error(
            None, provider, status, &headers, &body,
        ));
    }

    response.json().await.map_err(|e| {
        VmError::Thrown(VmValue::String(arcstr::ArcStr::from(format!(
            "{provider} completion response parse error: {e}"
        ))))
    })
}

/// Execute a text completion / fill-in-the-middle call owned by Harn.
pub(crate) async fn vm_call_completion_full(
    opts: &LlmCallOptions,
    prefix: &str,
    suffix: Option<&str>,
) -> Result<LlmResult, VmError> {
    crate::llm::admission::check_auxiliary(opts.budget.as_ref(), "fill-in-the-middle completion")?;
    if opts.provider == "mock" {
        return Ok(mock_completion_response(prefix, suffix));
    }

    let controls = super::data_controls::resolve(
        &opts.provider,
        &opts.model,
        crate::llm_config::DataControlDialect::OpenAiSse,
        crate::llm_config::DataPosture::Default,
    );
    let boundary_rule = super::inference_boundary::preflight(
        opts.inference_boundary,
        &opts.provider,
        &opts.model,
        &controls.receipt,
    )
    .map_err(VmError::Runtime)?;
    crate::llm::ensure_real_llm_allowed(&opts.provider)?;

    let resolved = crate::llm_config::provider_config(&opts.provider);
    let completion_endpoint = resolved.and_then(|p| p.completion_endpoint);

    let mut result = match completion_endpoint.as_deref() {
        Some("/api/generate") => {
            reject_completion_options(
                opts,
                &[
                    ("logprobs", opts.logprobs.is_some()),
                    ("logit_bias", !opts.logit_bias.is_empty()),
                    ("prediction", opts.prediction.is_some()),
                    ("verbosity", opts.verbosity.is_some()),
                    ("parallel_tool_calls", opts.parallel_tool_calls.is_some()),
                ],
                "Ollama generate",
            )?;
            vm_call_completion_ollama(opts, prefix, suffix).await
        }
        Some(_) => {
            reject_completion_options(
                opts,
                &[
                    ("min_p", opts.min_p.is_some()),
                    ("repetition_penalty", opts.repetition_penalty.is_some()),
                    ("prediction", opts.prediction.is_some()),
                    ("verbosity", opts.verbosity.is_some()),
                    ("mirostat", opts.mirostat.is_some()),
                    ("parallel_tool_calls", opts.parallel_tool_calls.is_some()),
                ],
                "OpenAI-compatible completions",
            )?;
            vm_call_completion_openai_style(opts, prefix, suffix).await
        }
        None => vm_call_completion_fallback(opts, prefix, suffix).await,
    }?;
    result.telemetry.inference_boundary_rule = boundary_rule.map(str::to_string);
    Ok(result)
}

fn reject_completion_options(
    opts: &LlmCallOptions,
    options: &[(&str, bool)],
    surface: &str,
) -> Result<(), VmError> {
    if let Some((option, _)) = options.iter().find(|(_, selected)| *selected) {
        return Err(crate::llm::call::invalid_request_error(
            format!("option `{option}` is not representable by {surface}"),
            &opts.provider,
            &opts.model,
        ));
    }
    Ok(())
}

async fn vm_call_completion_openai_style(
    opts: &LlmCallOptions,
    prefix: &str,
    suffix: Option<&str>,
) -> Result<LlmResult, VmError> {
    let llm_timeout = opts.resolve_timeout();

    let pdef = crate::llm_config::provider_config(&opts.provider);
    let base_url = pdef
        .as_ref()
        .map(crate::llm_config::resolve_base_url)
        .unwrap_or_else(|| "https://api.openai.com/v1".to_string());
    let endpoint = pdef
        .as_ref()
        .and_then(|p| p.completion_endpoint.as_deref())
        .unwrap_or("/completions");
    let client = crate::llm::blocking_client_for_base_url(&base_url);

    let wire_model = crate::llm_config::wire_model_id(&opts.model);
    let mut body = serde_json::json!({
        "model": wire_model,
        "prompt": prefix,
        "max_tokens": opts.max_tokens,
    });
    if let Some(suffix) = suffix.filter(|s| !s.is_empty()) {
        body["suffix"] = serde_json::json!(suffix);
    }
    if let Some(temp) = opts.temperature {
        body["temperature"] = serde_json::json!(temp);
    }
    if let Some(top_p) = opts.top_p {
        body["top_p"] = serde_json::json!(top_p);
    }
    if let Some(logprobs) = opts.logprobs {
        body["logprobs"] = serde_json::json!(logprobs.top.unwrap_or(0));
    }
    if !opts.logit_bias.is_empty() {
        body["logit_bias"] = serde_json::Value::Object(
            opts.logit_bias
                .iter()
                .map(|entry| (entry.token_id.to_string(), serde_json::json!(entry.bias)))
                .collect(),
        );
    }
    if let Some(stop) = &opts.stop {
        body["stop"] = serde_json::json!(stop);
    }
    if let Some(seed) = opts.seed {
        body["seed"] = serde_json::json!(seed);
    }
    crate::llm::provider::apply_provider_wire_overrides(
        &mut body,
        opts.provider_overrides.as_ref(),
    );
    if crate::llm::provider::provider_uses_anthropic_messages(&opts.provider, &opts.model) {
        crate::llm::providers::anthropic::reconcile_request_body(
            &mut body,
            &opts.provider,
            &opts.model,
            &opts.thinking,
            opts.provider_contract_probe,
        );
    }

    let req = client
        .post(format!("{base_url}{endpoint}"))
        .header("Content-Type", "application/json")
        .timeout(std::time::Duration::from_secs(llm_timeout))
        .json(&body);
    let req = apply_auth_headers(req, &opts.api_key, pdef.as_ref());

    let response = req
        .send()
        .await
        .map_err(|e| crate::llm::api::reqwest_send_error(&opts.provider, "completion", e))?;

    let json = completion_json_response(&opts.provider, response).await?;

    if let Some(err) = json["error"]["message"].as_str() {
        return Err(VmError::Thrown(VmValue::String(arcstr::ArcStr::from(
            format!("{} completion API error: {err}", opts.provider),
        ))));
    }

    let request_id = json["id"].as_str().filter(|value| !value.is_empty());
    let telemetry = ProviderTelemetry::from_openai_response(&json, request_id);
    Ok(LlmResult {
        attempts: Default::default(),
        text_projection: None,
        served_fast: false,
        text: json["choices"][0]["text"]
            .as_str()
            .unwrap_or("")
            .to_string(),
        tool_calls: Vec::new(),
        raw_tool_calls: Vec::new(),
        input_tokens: json["usage"]["prompt_tokens"].as_i64().unwrap_or(0),
        output_tokens: json["usage"]["completion_tokens"].as_i64().unwrap_or(0),
        cache_read_tokens: extract_cache_read_tokens(&json["usage"])?,
        cache_write_tokens: extract_cache_write_tokens(&json["usage"])?,
        cache_supported: true,
        model: opts.model.clone(),
        provider: opts.provider.clone(),
        thinking: None,
        thinking_summary: None,
        stop_reason: json["choices"][0]["finish_reason"]
            .as_str()
            .map(|s| s.to_string()),
        blocks: vec![serde_json::json!({
            "type": "output_text",
            "text": json["choices"][0]["text"].as_str().unwrap_or(""),
            "visibility": "public",
        })],
        logprobs: extract_openai_choice_logprobs(&json["choices"][0]),
        telemetry: Box::new(telemetry),
    })
}

async fn vm_call_completion_ollama(
    opts: &LlmCallOptions,
    prefix: &str,
    suffix: Option<&str>,
) -> Result<LlmResult, VmError> {
    let llm_timeout = opts.resolve_timeout();
    let pdef = crate::llm_config::provider_config(&opts.provider);
    let base_url = pdef
        .as_ref()
        .map(crate::llm_config::resolve_base_url)
        .unwrap_or_else(|| "http://localhost:11434".to_string());
    let endpoint = pdef
        .as_ref()
        .and_then(|p| p.completion_endpoint.as_deref())
        .unwrap_or("/api/generate");
    let client = crate::llm::blocking_client_for_base_url(&base_url);

    let mut options = serde_json::Map::new();
    if let Some(temp) = opts.temperature {
        options.insert("temperature".to_string(), serde_json::json!(temp));
    }
    if let Some(top_p) = opts.top_p {
        options.insert("top_p".to_string(), serde_json::json!(top_p));
    }
    if let Some(top_k) = opts.top_k {
        options.insert("top_k".to_string(), serde_json::json!(top_k));
    }
    if let Some(min_p) = opts.min_p {
        options.insert("min_p".to_string(), serde_json::json!(min_p));
    }
    if let Some(repetition_penalty) = opts.repetition_penalty {
        options.insert(
            "repeat_penalty".to_string(),
            serde_json::json!(repetition_penalty),
        );
    }
    if let Some(mirostat) = opts.mirostat {
        options.insert("mirostat".to_string(), serde_json::json!(mirostat.version));
        options.insert(
            "mirostat_tau".to_string(),
            serde_json::json!(mirostat.target_entropy),
        );
        options.insert(
            "mirostat_eta".to_string(),
            serde_json::json!(mirostat.learning_rate),
        );
    }
    if let Some(seed) = opts.seed {
        options.insert("seed".to_string(), serde_json::json!(seed));
    }
    if let Some(stop) = &opts.stop {
        options.insert("stop".to_string(), serde_json::json!(stop));
    }
    options.insert(
        "num_predict".to_string(),
        serde_json::json!(opts.max_tokens),
    );

    let wire_model = crate::llm_config::wire_model_id(&opts.model);
    let mut body = serde_json::json!({
        "model": wire_model,
        "prompt": prefix,
        "stream": false,
        "raw": true,
        "options": options,
    });
    if let Some(suffix) = suffix.filter(|s| !s.is_empty()) {
        body["suffix"] = serde_json::json!(suffix);
    }
    if let Some(system) = &opts.system {
        body["system"] = serde_json::json!(system);
    }
    super::apply_ollama_runtime_settings(
        &mut body,
        &opts.provider,
        &opts.model,
        opts.provider_overrides.as_ref(),
    );

    let req = client
        .post(format!("{base_url}{endpoint}"))
        .header("Content-Type", "application/json")
        .timeout(std::time::Duration::from_secs(llm_timeout))
        .json(&body);
    let req = apply_auth_headers(req, &opts.api_key, pdef.as_ref());

    let response = req
        .send()
        .await
        .map_err(|e| crate::llm::api::reqwest_send_error(&opts.provider, "completion", e))?;
    let json = completion_json_response(&opts.provider, response).await?;
    if let Some(err) = json["error"].as_str() {
        return Err(VmError::Thrown(VmValue::String(arcstr::ArcStr::from(
            format!("{} completion API error: {err}", opts.provider),
        ))));
    }

    let telemetry = ProviderTelemetry::from_ollama_done(&json, telemetry_source::OLLAMA_GENERATE);
    Ok(LlmResult {
        attempts: Default::default(),
        text_projection: None,
        served_fast: false,
        text: json["response"].as_str().unwrap_or("").to_string(),
        tool_calls: Vec::new(),
        raw_tool_calls: Vec::new(),
        input_tokens: json["prompt_eval_count"].as_i64().unwrap_or(0),
        output_tokens: json["eval_count"].as_i64().unwrap_or(0),
        // Native Ollama generate responses carry no cache field; 0 is
        // "unknown", not a real miss. `cache_supported: false` keeps the cost
        // layer from scoring a local model as a 100% cache miss.
        cache_read_tokens: 0,
        cache_write_tokens: 0,
        cache_supported: false,
        model: opts.model.clone(),
        provider: opts.provider.clone(),
        thinking: None,
        thinking_summary: None,
        stop_reason: json["done_reason"].as_str().map(|s| s.to_string()),
        blocks: vec![serde_json::json!({
            "type": "output_text",
            "text": json["response"].as_str().unwrap_or(""),
            "visibility": "public",
        })],
        logprobs: Vec::new(),
        telemetry: Box::new(telemetry),
    })
}

async fn vm_call_completion_fallback(
    opts: &LlmCallOptions,
    prefix: &str,
    suffix: Option<&str>,
) -> Result<LlmResult, VmError> {
    let mut fallback_opts = opts.clone();
    let has_suffix = suffix.is_some_and(|s| !s.is_empty());
    let mut bindings = crate::value::DictMap::new();
    bindings.put_str("prefix", prefix);
    bindings.insert(
        crate::value::intern_key("has_suffix"),
        VmValue::Bool(has_suffix),
    );
    bindings.put_str("suffix", suffix.unwrap_or_default());
    let instruction = crate::stdlib::template::render_stdlib_prompt_asset(
        "llm/prompts/completion_fallback_system.harn.prompt",
        Some(&bindings),
    )?;
    let user_prompt = crate::stdlib::template::render_stdlib_prompt_asset(
        "llm/prompts/completion_fallback_user.harn.prompt",
        Some(&bindings),
    )?;
    fallback_opts.messages = vec![serde_json::json!({
        "role": "user",
        "content": user_prompt,
    })];
    fallback_opts.system = match &opts.system {
        Some(system) => Some(format!("{system}\n\n{instruction}")),
        None => Some(instruction),
    };
    fallback_opts.context_manifest.record_system_transform(
        "completion_fallback",
        "stdlib:completion_fallback",
        "completion fallback appended its system instruction",
        fallback_opts.system.as_deref(),
    );
    super::vm_call_llm_full(&fallback_opts).await
}

#[cfg(test)]
mod inference_boundary_tests {
    use super::*;

    #[tokio::test]
    async fn local_ceiling_refuses_hosted_completion_before_provider_io() {
        let opts = LlmCallOptions {
            provider: "openai".into(),
            model: "gpt-4o".into(),
            api_key: "invalid-test-key".into(),
            inference_boundary: Some(super::super::InferenceBoundary {
                reach: super::super::inference_boundary::InferenceReach::LocalOnly,
                allow_training_discounts: false,
            }),
            ..Default::default()
        };
        let refusal = vm_call_completion_full(&opts, "prefix", None)
            .await
            .expect_err("a hosted route cannot pass the local ceiling");
        assert!(format!("{refusal:?}").contains("inference_boundary.local_only"));
    }
}