ferrum-cli 0.8.2

//! OpenAI client contract tests — drive `cli serve` with `async-openai`,
//! the community-maintained Rust client for OpenAI's API.
//!
//! Where `server_smoke.rs` tests *behaviour* (does the server respond
//! correctly for a given request), this file tests *contract*: can a
//! real OpenAI client successfully parse our responses? If we deviate
//! from the spec (renamed field, wrong type, missing required field,
//! malformed SSE), `async-openai` raises a parse error and the test
//! fails. Catches schema drift the hand-written `reqwest` tests would
//! miss.
//!
//! Loads a real model; `#[ignore]` by default. Opt in:
//!
//!     ferrum pull qwen3:0.6b
//!     cargo test --release -p ferrum-cli --features metal --test server_openai_compat \
//!       -- --ignored --test-threads=1
//!
//! The Python SDK smoke additionally requires:
//!
//!     python3 -m pip install openai
//!     # optionally: FERRUM_PYTHON=python3.12

use async_openai::{
    config::OpenAIConfig,
    types::{
        ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestUserMessageArgs,
        ChatCompletionStreamOptions, ChatCompletionToolArgs, ChatCompletionToolChoiceOption,
        CreateChatCompletionRequestArgs, FunctionObjectArgs, ResponseFormat,
        ResponseFormatJsonSchema,
    },
    Client,
};
use futures::StreamExt;
use reqwest::Client as ReqwestClient;
use std::net::TcpListener;
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};

const SMOKE_MODEL: &str = "qwen3:0.6b";
const STARTUP_TIMEOUT: Duration = Duration::from_secs(120);
const REQUEST_TIMEOUT: Duration = Duration::from_secs(300);

fn http_client() -> ReqwestClient {
    ReqwestClient::builder()
        .timeout(REQUEST_TIMEOUT)
        .build()
        .expect("build HTTP client")
}

fn ferrum_bin() -> PathBuf {
    if let Ok(bin) = std::env::var("CARGO_BIN_EXE_ferrum") {
        return PathBuf::from(bin);
    }
    let current = std::env::current_exe().expect("test exe path");
    let dir = current
        .parent()
        .and_then(|p| p.parent())
        .expect("target dir");
    let mut bin = dir.join("ferrum");
    if cfg!(windows) {
        bin.set_extension("exe");
    }
    assert!(bin.exists(), "ferrum binary not found at {}", bin.display());
    bin
}

fn free_port() -> u16 {
    let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port");
    listener.local_addr().expect("local_addr").port()
}

fn python_bin() -> String {
    std::env::var("FERRUM_PYTHON")
        .or_else(|_| std::env::var("PYTHON"))
        .unwrap_or_else(|_| "python3".to_string())
}

/// Same fixture as `server_smoke.rs`; duplicated rather than shared via
/// a `tests/common` module because cargo's test layout makes the common
/// pattern noisy. Move to a shared helper if a third HTTP test file
/// appears.
struct ServerFixture {
    base_url: String,
    child: Child,
}

impl ServerFixture {
    async fn spawn(model: &str) -> Self {
        let port = free_port();
        let base_url = format!("http://127.0.0.1:{port}");
        let child = Command::new(ferrum_bin())
            .args([
                "serve",
                model,
                "--disable-thinking",
                "--port",
                &port.to_string(),
            ])
            .env("NO_COLOR", "1")
            .stdout(Stdio::inherit())
            .stderr(Stdio::inherit())
            .spawn()
            .expect("spawn ferrum serve");

        let probe = http_client();
        let healthz = format!("{base_url}/health");
        let start = Instant::now();
        loop {
            if start.elapsed() > STARTUP_TIMEOUT {
                panic!("server did not become healthy within {STARTUP_TIMEOUT:?}");
            }
            let ok = probe
                .get(&healthz)
                .timeout(Duration::from_secs(2))
                .send()
                .await
                .map(|r| r.status().is_success())
                .unwrap_or(false);
            if ok {
                break;
            }
            tokio::time::sleep(Duration::from_millis(500)).await;
        }
        Self { base_url, child }
    }

    fn client(&self) -> Client<OpenAIConfig> {
        let config = OpenAIConfig::new()
            .with_api_base(format!("{}/v1", self.base_url))
            .with_api_key("dummy-key-not-checked");
        Client::with_config(config).with_http_client(http_client())
    }
}

impl Drop for ServerFixture {
    fn drop(&mut self) {
        let _ = self.child.kill();
        let _ = self.child.wait();
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// PR 9 — async-openai client contract tests
// ─────────────────────────────────────────────────────────────────────────────

#[tokio::test(flavor = "current_thread")]
#[ignore = "loads real model — run with `cargo test -- --ignored`"]
async fn test_openai_client_chat_basic() {
    // async-openai will reject our response if it doesn't match the
    // ChatCompletionResponse schema (missing required fields, wrong
    // types). Successful return here is the contract proof.
    let fx = ServerFixture::spawn(SMOKE_MODEL).await;
    let client = fx.client();

    let request = CreateChatCompletionRequestArgs::default()
        .model(SMOKE_MODEL)
        .messages([ChatCompletionRequestUserMessageArgs::default()
            .content("Say hi in one short sentence.")
            .build()
            .expect("build user msg")
            .into()])
        .max_tokens(8u32)
        .temperature(0.0)
        .build()
        .expect("build request");

    let response = client.chat().create(request).await.expect("chat request");
    assert!(!response.choices.is_empty(), "no choices in response");
    let content = response.choices[0].message.content.as_deref().unwrap_or("");
    assert!(!content.trim().is_empty(), "content empty");
}

#[tokio::test(flavor = "current_thread")]
#[ignore = "loads real model"]
async fn test_openai_client_chat_streaming() {
    // async-openai parses SSE events into typed `CreateChatCompletionStreamResponse`.
    // Bad event format (missing `data:` prefix, wrong JSON, `[DONE]` malformed)
    // causes the stream to error mid-way; we assert clean iteration to
    // end-of-stream + non-empty concatenated delta.
    let fx = ServerFixture::spawn(SMOKE_MODEL).await;
    let client = fx.client();

    let request = CreateChatCompletionRequestArgs::default()
        .model(SMOKE_MODEL)
        .messages([ChatCompletionRequestUserMessageArgs::default()
            .content("Say hi in one short sentence.")
            .build()
            .expect("build user msg")
            .into()])
        .max_tokens(8u32)
        .temperature(0.0)
        .stream(true)
        .build()
        .expect("build streaming request");

    let mut stream = client
        .chat()
        .create_stream(request)
        .await
        .expect("open stream");

    let mut content = String::new();
    let mut chunk_count = 0usize;
    while let Some(result) = stream.next().await {
        let chunk = result.expect("parse stream chunk");
        chunk_count += 1;
        if let Some(choice) = chunk.choices.first() {
            if let Some(delta) = &choice.delta.content {
                content.push_str(delta);
            }
        }
    }
    assert!(chunk_count > 0, "no stream chunks parsed");
    assert!(
        !content.trim().is_empty(),
        "concatenated stream content empty"
    );
}

#[tokio::test(flavor = "current_thread")]
#[ignore = "loads real model"]
async fn test_openai_client_tools_stream_options_include_usage() {
    // Exercises async-openai's typed request fields for tools, tool_choice,
    // and stream_options.include_usage. Ferrum does not implement tool-call
    // generation yet, but it must accept the SDK request shape, stream valid
    // chat chunks, and expose the final usage chunk.
    let fx = ServerFixture::spawn(SMOKE_MODEL).await;
    let client = fx.client();
    let weather_tool = ChatCompletionToolArgs::default()
        .function(
            FunctionObjectArgs::default()
                .name("get_weather")
                .description("Return a short weather summary for a city.")
                .parameters(serde_json::json!({
                    "type": "object",
                    "properties": {
                        "city": {"type": "string"}
                    },
                    "required": ["city"]
                }))
                .build()
                .expect("build function object"),
        )
        .build()
        .expect("build tool");

    let request = CreateChatCompletionRequestArgs::default()
        .model(SMOKE_MODEL)
        .messages([ChatCompletionRequestUserMessageArgs::default()
            .content("Say hi in one short sentence. Do not call a tool.")
            .build()
            .expect("build user msg")
            .into()])
        .max_tokens(16u32)
        .temperature(0.0)
        .stream(true)
        .stream_options(ChatCompletionStreamOptions {
            include_usage: true,
        })
        .tools([weather_tool])
        .tool_choice(ChatCompletionToolChoiceOption::Auto)
        .build()
        .expect("build tools streaming request");

    let mut stream = client
        .chat()
        .create_stream(request)
        .await
        .expect("open tools stream");

    let mut content = String::new();
    let mut tool_call_names = Vec::new();
    let mut chunk_count = 0usize;
    let mut usage_seen = false;
    while let Some(result) = stream.next().await {
        let chunk = result.expect("parse tools stream chunk");
        chunk_count += 1;
        if let Some(usage) = &chunk.usage {
            usage_seen = usage.total_tokens > 0;
        }
        if let Some(choice) = chunk.choices.first() {
            if let Some(delta) = &choice.delta.content {
                content.push_str(delta);
            }
            for call in choice.delta.tool_calls.iter().flatten() {
                if let Some(name) = call.function.as_ref().and_then(|f| f.name.as_deref()) {
                    tool_call_names.push(name.to_string());
                }
            }
        }
    }

    assert!(chunk_count > 0, "no stream chunks parsed");
    // This test pins the streaming *mechanics* (parsable chunks + final
    // usage), not the model's choice: with `tool_choice=auto` a greedy
    // 0.6B model may answer in text or call the declared tool, and that
    // choice is prompt-byte-sensitive. Either outcome must arrive as
    // well-formed SDK deltas.
    let has_text = !content.trim().is_empty();
    let has_valid_tool_call = tool_call_names.iter().any(|name| name == "get_weather");
    assert!(
        has_text || has_valid_tool_call,
        "stream produced neither text content nor a valid get_weather tool call"
    );
    assert!(
        usage_seen,
        "stream_options.include_usage did not produce final SDK usage"
    );
}

#[tokio::test(flavor = "current_thread")]
#[ignore = "loads real model"]
async fn test_openai_client_response_format_json_object() {
    // `response_format = json_object` activates ferrum's `JsonModeProcessor`
    // (continuous_engine.rs:148) which constrains the sampler to emit
    // valid JSON. Verifies that (a) the OpenAI response_format field is
    // wired through to the engine and (b) the resulting content is
    // actually parseable JSON.
    let fx = ServerFixture::spawn(SMOKE_MODEL).await;
    let client = fx.client();

    let request = CreateChatCompletionRequestArgs::default()
        .model(SMOKE_MODEL)
        .messages([ChatCompletionRequestUserMessageArgs::default()
            .content("Return exactly this JSON object and nothing else: {\"ok\":true}")
            .build()
            .expect("build user msg")
            .into()])
        .max_tokens(16u32)
        .temperature(0.0)
        .response_format(ResponseFormat::JsonObject)
        .build()
        .expect("build request");

    let response = client.chat().create(request).await.expect("chat request");
    let content = response.choices[0].message.content.as_deref().unwrap_or("");
    assert!(!content.trim().is_empty(), "json_object response empty");
    // Strict: the whole content must be valid JSON. The server strips
    // markdown fences in `strip_markdown_json_fence` before returning;
    // models that don't emit a fence at all pass through unchanged.
    let parsed: Result<serde_json::Value, _> = serde_json::from_str(content.trim());
    assert!(
        parsed.is_ok(),
        "response_format=json_object should produce parseable JSON \
         (server strips markdown fences); got: {content:?}"
    );
}

#[tokio::test(flavor = "current_thread")]
#[ignore = "loads real model"]
async fn test_openai_client_strict_json_schema_3_runs() {
    // Real-model smoke: a simple strict object schema should succeed on
    // repeated requests at temperature 0. The server validates before
    // returning, so any hard-mask/validation failure surfaces as an SDK
    // request error or non-JSON content.
    let fx = ServerFixture::spawn(SMOKE_MODEL).await;
    let client = fx.client();
    let response_format = ResponseFormat::JsonSchema {
        json_schema: ResponseFormatJsonSchema {
            description: Some("A short answer object.".to_string()),
            name: "answer_object".to_string(),
            schema: Some(serde_json::json!({
                "type": "object",
                "properties": {
                    "answer": {"type": "string", "enum": ["ok"]}
                },
                "required": ["answer"],
                "additionalProperties": false
            })),
            strict: Some(true),
        },
    };

    for run in 0..3 {
        let request = CreateChatCompletionRequestArgs::default()
            .model(SMOKE_MODEL)
            .messages([ChatCompletionRequestUserMessageArgs::default()
                .content("Return an object whose answer field is the string ok.")
                .build()
                .expect("build user msg")
                .into()])
            .max_tokens(16u32)
            .temperature(0.0)
            .response_format(response_format.clone())
            .build()
            .expect("build strict schema request");

        let response = client
            .chat()
            .create(request)
            .await
            .unwrap_or_else(|e| panic!("strict schema run {run} request failed: {e}"));
        let content = response.choices[0].message.content.as_deref().unwrap_or("");
        let parsed: serde_json::Value = serde_json::from_str(content).unwrap_or_else(|e| {
            panic!("strict schema run {run} returned invalid JSON: {e}; content={content:?}")
        });
        assert_eq!(
            parsed.get("answer").and_then(|v| v.as_str()),
            Some("ok"),
            "strict schema run {run} returned the wrong answer: {parsed}"
        );
    }
}

#[tokio::test(flavor = "current_thread")]
#[ignore = "loads real model"]
async fn test_openai_client_multi_turn() {
    // Verify that messages constructed via async-openai's typed builders
    // (User / Assistant variants) are accepted together and reach real-model
    // generation. Exact message preservation is covered by deterministic
    // server conversion tests; a tiny model's recall is not a stable SDK or
    // wire-compatibility contract across Metal runners.
    let fx = ServerFixture::spawn(SMOKE_MODEL).await;
    let client = fx.client();

    let user1 = ChatCompletionRequestUserMessageArgs::default()
        .content("Remember the code name.")
        .build()
        .expect("build user msg 1")
        .into();
    let asst1 = ChatCompletionRequestAssistantMessageArgs::default()
        .content("The code name is XiaoMing.")
        .build()
        .expect("build asst msg")
        .into();
    let user2 = ChatCompletionRequestUserMessageArgs::default()
        .content("Copy only the code name from the previous assistant message.")
        .build()
        .expect("build user msg 2")
        .into();

    let request = CreateChatCompletionRequestArgs::default()
        .model(SMOKE_MODEL)
        .messages([user1, asst1, user2])
        .max_tokens(16u32)
        .temperature(0.0)
        .build()
        .expect("build request");

    let response = client.chat().create(request).await.expect("chat request");
    let content = response.choices[0].message.content.as_deref().unwrap_or("");
    assert!(
        !content.trim().is_empty(),
        "typed multi-turn request returned empty content: {response:?}"
    );
}

#[tokio::test(flavor = "current_thread")]
#[ignore = "loads real model and requires the Python `openai` package"]
async fn test_python_openai_sdk_chat_and_stream_smoke() {
    // Exercises the official Python OpenAI SDK against Ferrum's local
    // OpenAI-compatible server. This catches client-side schema/SSE
    // incompatibilities outside Rust's async-openai type model.
    let fx = ServerFixture::spawn(SMOKE_MODEL).await;
    let script = r#"
import os
import sys

try:
    from openai import OpenAI
except Exception as exc:
    raise SystemExit(
        "Python package `openai` is required for this ignored smoke: "
        "python3 -m pip install openai\n"
        f"import error: {exc}"
    )

base_url = os.environ["FERRUM_OPENAI_BASE_URL"]
model = os.environ["FERRUM_OPENAI_MODEL"]
client = OpenAI(
    base_url=f"{base_url}/v1",
    api_key="dummy-key-not-checked",
    timeout=300.0,
    max_retries=0,
)

response = client.chat.completions.create(
    model=model,
    messages=[{"role": "user", "content": "Say hi in one short sentence."}],
    max_tokens=8,
    temperature=0,
)
content = response.choices[0].message.content or ""
if not content.strip():
    raise SystemExit("empty non-streaming Python SDK chat content")

stream = client.chat.completions.create(
    model=model,
    messages=[{"role": "user", "content": "Say hi in one short sentence."}],
    max_tokens=8,
    temperature=0,
    stream=True,
    stream_options={"include_usage": True},
)
chunks = 0
choice_chunks = 0
terminal_finish_reason_seen = False
usage_seen = False
for chunk in stream:
    chunks += 1
    if chunk.choices:
        choice_chunks += 1
        if chunk.choices[0].finish_reason is not None:
            terminal_finish_reason_seen = True
    if getattr(chunk, "usage", None) is not None:
        usage_seen = True

if chunks == 0:
    raise SystemExit("Python SDK stream yielded no chunks")
if choice_chunks == 0:
    raise SystemExit("Python SDK stream yielded no choice chunks")
if not terminal_finish_reason_seen:
    raise SystemExit("Python SDK stream exposed no terminal finish_reason")
if not usage_seen:
    raise SystemExit("Python SDK stream_options.include_usage did not expose usage")
"#;

    let output = Command::new(python_bin())
        .arg("-c")
        .arg(script)
        .env("FERRUM_OPENAI_BASE_URL", &fx.base_url)
        .env("FERRUM_OPENAI_MODEL", SMOKE_MODEL)
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .output()
        .expect("spawn Python OpenAI SDK smoke");

    assert!(
        output.status.success(),
        "Python OpenAI SDK smoke failed\nstatus: {:?}\nstdout:\n{}\nstderr:\n{}",
        output.status,
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr)
    );
}