use serde_json::json;
use super::model_pricing::pricing_test_mutex;
use super::request::{GenerationParams, Message, MessageContent};
use super::resolve::{normalize_request, normalize_request_with_hint, normalize_response};
use super::response::{
AnnotatedLlmResponse, ApiSpecificResponse, CostEstimate, CostSource, FinishReason,
PricingCatalog, PricingResolver, Usage, reset_active_pricing_resolver,
set_active_pricing_resolver,
};
use crate::api::llm::LlmRequest;
use crate::json::Json;
struct ResetPricingResolverGuard;
impl Drop for ResetPricingResolverGuard {
fn drop(&mut self) {
let _ = reset_active_pricing_resolver();
}
}
fn install_parity_pricing(model_id: &str) {
let catalog = PricingCatalog::from_json_str(
&json!({
"version": 1,
"entries": [
{
"provider": "test",
"model_id": model_id,
"pricing_as_of": "2026-06-05",
"pricing_source": "test",
"rates": {
"input_per_million": 0.15,
"output_per_million": 0.60,
"cache_read_per_million": 0.075
},
"prompt_cache": {
"read_accounting": "included_in_prompt_tokens"
}
}
]
})
.to_string(),
)
.unwrap();
set_active_pricing_resolver(PricingResolver::from_catalogs(vec![catalog])).unwrap();
}
fn req(content: Json) -> LlmRequest {
LlmRequest {
headers: serde_json::Map::new(),
content,
}
}
fn decode(raw: &Json) -> AnnotatedLlmResponse {
normalize_response(raw).unwrap_or_else(|| panic!("response should detect and decode: {raw}"))
}
fn chat_text_response(model: &str) -> Json {
json!({
"id": "chatcmpl-parity",
"object": "chat.completion",
"model": model,
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": "hello"},
"finish_reason": "stop"
}]
})
}
fn anthropic_text_response(model: &str) -> Json {
json!({
"id": "msg_parity",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": "hello"}],
"stop_reason": "end_turn"
})
}
fn responses_text_response(model: &str) -> Json {
json!({
"id": "resp_parity",
"model": model,
"status": "completed",
"output": [{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "hello"}]
}]
})
}
fn chat_response_with_usage(model: &str, extra_usage: Json) -> Json {
let mut raw = chat_text_response(model);
let mut usage = json!({
"prompt_tokens": 1000,
"completion_tokens": 500,
"total_tokens": 1500,
"prompt_tokens_details": {"cached_tokens": 200}
});
merge_object(&mut usage, extra_usage);
raw.as_object_mut().unwrap().insert("usage".into(), usage);
raw
}
fn anthropic_response_with_usage(model: &str, extra_usage: Json) -> Json {
let mut raw = anthropic_text_response(model);
let mut usage = json!({
"input_tokens": 1000,
"output_tokens": 500,
"cache_read_input_tokens": 200
});
merge_object(&mut usage, extra_usage);
raw.as_object_mut().unwrap().insert("usage".into(), usage);
raw
}
fn responses_response_with_usage(model: &str, extra_usage: Json) -> Json {
let mut raw = responses_text_response(model);
let mut usage = json!({
"input_tokens": 1000,
"output_tokens": 500,
"total_tokens": 1500,
"input_tokens_details": {"cached_tokens": 200}
});
merge_object(&mut usage, extra_usage);
raw.as_object_mut().unwrap().insert("usage".into(), usage);
raw
}
fn merge_object(target: &mut Json, extra: Json) {
if let (Some(target), Json::Object(extra)) = (target.as_object_mut(), extra) {
target.extend(extra);
}
}
#[test]
fn test_response_model_name_and_text_parity() {
let chat = decode(&chat_text_response("parity-shared-model"));
let anthropic = decode(&anthropic_text_response("parity-shared-model"));
let responses = decode(&responses_text_response("parity-shared-model"));
for decoded in [&chat, &anthropic, &responses] {
assert_eq!(decoded.model.as_deref(), Some("parity-shared-model"));
assert_eq!(decoded.response_text(), Some("hello"));
assert_eq!(decoded.finish_reason, Some(FinishReason::Complete));
}
assert_eq!(chat.id.as_deref(), Some("chatcmpl-parity"));
assert_eq!(anthropic.id.as_deref(), Some("msg_parity"));
assert_eq!(responses.id.as_deref(), Some("resp_parity"));
}
#[test]
fn test_finish_reason_complete_parity() {
let raws = [
json!({"choices": [{"message": {"role": "assistant", "content": "x"}, "finish_reason": "stop"}]}),
json!({"type": "message", "content": [{"type": "text", "text": "x"}], "stop_reason": "end_turn"}),
json!({"status": "completed", "output": [{"type": "message", "content": [{"type": "output_text", "text": "x"}]}]}),
];
for raw in &raws {
assert_eq!(
decode(raw).finish_reason,
Some(FinishReason::Complete),
"expected Complete for {raw}",
);
}
}
#[test]
fn test_finish_reason_length_parity() {
let raws = [
json!({"choices": [{"message": {"role": "assistant", "content": "x"}, "finish_reason": "length"}]}),
json!({"type": "message", "content": [{"type": "text", "text": "x"}], "stop_reason": "max_tokens"}),
json!({
"status": "incomplete",
"incomplete_details": {"reason": "max_output_tokens"},
"output": []
}),
];
for raw in &raws {
assert_eq!(
decode(raw).finish_reason,
Some(FinishReason::Length),
"expected Length for {raw}",
);
}
}
#[test]
fn test_finish_reason_tool_use_parity_and_responses_divergence() {
let chat = decode(&json!({
"choices": [{
"message": {
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_parity_1",
"type": "function",
"function": {"name": "get_weather", "arguments": "{\"city\":\"NYC\"}"}
}]
},
"finish_reason": "tool_calls"
}]
}));
let anthropic = decode(&json!({
"type": "message",
"content": [{
"type": "tool_use",
"id": "call_parity_1",
"name": "get_weather",
"input": {"city": "NYC"}
}],
"stop_reason": "tool_use"
}));
let responses = decode(&json!({
"status": "completed",
"output": [{
"type": "function_call",
"call_id": "call_parity_1",
"name": "get_weather",
"arguments": "{\"city\":\"NYC\"}"
}]
}));
assert_eq!(chat.finish_reason, Some(FinishReason::ToolUse));
assert_eq!(anthropic.finish_reason, Some(FinishReason::ToolUse));
assert_eq!(responses.finish_reason, Some(FinishReason::Complete));
assert!(responses.has_tool_calls());
}
#[test]
fn test_response_tool_call_parity() {
let chat = decode(&json!({
"choices": [{
"message": {
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_parity_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"city\":\"NYC\",\"units\":\"c\"}"
}
}]
},
"finish_reason": "tool_calls"
}]
}));
let anthropic = decode(&json!({
"type": "message",
"content": [{
"type": "tool_use",
"id": "call_parity_1",
"name": "get_weather",
"input": {"city": "NYC", "units": "c"}
}],
"stop_reason": "tool_use"
}));
let responses = decode(&json!({
"status": "completed",
"output": [{
"type": "function_call",
"id": "fc_item_1",
"call_id": "call_parity_1",
"name": "get_weather",
"arguments": "{\"city\":\"NYC\",\"units\":\"c\"}"
}]
}));
let chat_calls = chat.tool_calls.expect("chat tool calls");
let anthropic_calls = anthropic.tool_calls.expect("anthropic tool calls");
let responses_calls = responses.tool_calls.expect("responses tool calls");
assert_eq!(chat_calls, anthropic_calls);
assert_eq!(chat_calls, responses_calls);
assert_eq!(chat_calls.len(), 1);
assert_eq!(chat_calls[0].id, "call_parity_1");
assert_eq!(chat_calls[0].name, "get_weather");
assert_eq!(
chat_calls[0].arguments,
json!({"city": "NYC", "units": "c"})
);
assert!(chat_calls[0].arguments.is_object());
}
#[test]
fn test_response_usage_parity() {
let chat = decode(&chat_response_with_usage("parity-usage-model", json!({})));
let anthropic = decode(&anthropic_response_with_usage(
"parity-usage-model",
json!({}),
));
let responses = decode(&responses_response_with_usage(
"parity-usage-model",
json!({}),
));
let expected = Usage {
prompt_tokens: Some(1000),
completion_tokens: Some(500),
total_tokens: Some(1500),
cache_read_tokens: Some(200),
cache_write_tokens: None,
cost: None,
};
assert_eq!(chat.usage, Some(expected.clone()));
assert_eq!(anthropic.usage, Some(expected.clone()));
assert_eq!(responses.usage, Some(expected));
}
#[test]
fn test_response_usage_schema_specific_extras() {
let chat = decode(&chat_response_with_usage("parity-usage-model", json!({})));
let anthropic = decode(&anthropic_response_with_usage(
"parity-usage-model",
json!({"cache_creation_input_tokens": 64}),
));
let responses = decode(&responses_response_with_usage(
"parity-usage-model",
json!({"output_tokens_details": {"reasoning_tokens": 128}}),
));
assert_eq!(
anthropic.usage.as_ref().unwrap().cache_write_tokens,
Some(64)
);
assert_eq!(chat.usage.as_ref().unwrap().cache_write_tokens, None);
assert_eq!(responses.usage.as_ref().unwrap().cache_write_tokens, None);
match responses.api_specific.as_ref().unwrap() {
ApiSpecificResponse::OpenAIResponses {
output_tokens_details,
..
} => {
assert_eq!(
output_tokens_details,
&Some(json!({"reasoning_tokens": 128}))
);
}
other => panic!("expected OpenAIResponses api_specific, got {other:?}"),
}
assert!(matches!(
chat.api_specific,
Some(ApiSpecificResponse::OpenAIChat { .. })
));
assert!(matches!(
anthropic.api_specific,
Some(ApiSpecificResponse::AnthropicMessages { .. })
));
for decoded in [&chat, &anthropic, &responses] {
let usage = decoded.usage.as_ref().unwrap();
assert_eq!(usage.prompt_tokens, Some(1000));
assert_eq!(usage.completion_tokens, Some(500));
assert_eq!(usage.total_tokens, Some(1500));
assert_eq!(usage.cache_read_tokens, Some(200));
}
}
#[test]
fn test_provider_reported_cost_object_parity() {
let cost = json!({"cost": {
"total": 0.0123,
"input": 0.004,
"output": 0.0083,
"currency": "USD"
}});
let chat = decode(&chat_response_with_usage(
"parity-reported-model",
cost.clone(),
));
let anthropic = decode(&anthropic_response_with_usage(
"parity-reported-model",
cost.clone(),
));
let responses = decode(&responses_response_with_usage(
"parity-reported-model",
cost,
));
let expected = CostEstimate {
total: Some(0.0123),
currency: "USD".to_string(),
input: Some(0.004),
output: Some(0.0083),
cache_read: None,
cache_write: None,
source: CostSource::ProviderReported,
pricing_provider: None,
pricing_model: None,
pricing_as_of: None,
pricing_source: None,
};
assert_eq!(chat.usage.unwrap().cost, Some(expected.clone()));
assert_eq!(anthropic.usage.unwrap().cost, Some(expected.clone()));
assert_eq!(responses.usage.unwrap().cost, Some(expected));
}
#[test]
fn test_provider_reported_scalar_cost_parity() {
let cost = json!({"cost_usd": 0.5});
let decoded = [
decode(&chat_response_with_usage(
"parity-reported-model",
cost.clone(),
)),
decode(&anthropic_response_with_usage(
"parity-reported-model",
cost.clone(),
)),
decode(&responses_response_with_usage(
"parity-reported-model",
cost,
)),
];
for response in decoded {
let cost = response.usage.unwrap().cost.expect("scalar reported cost");
assert_eq!(cost.total, Some(0.5));
assert_eq!(cost.currency, "USD");
assert_eq!(cost.source, CostSource::ProviderReported);
assert_eq!(cost.input, None);
assert_eq!(cost.output, None);
}
}
#[test]
fn test_estimated_cost_parity_for_identical_model_and_usage() {
let _pricing_guard = pricing_test_mutex()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
install_parity_pricing("parity-priced-model");
let _reset_guard = ResetPricingResolverGuard;
let chat = decode(&chat_response_with_usage("parity-priced-model", json!({})));
let anthropic = decode(&anthropic_response_with_usage(
"parity-priced-model",
json!({}),
));
let responses = decode(&responses_response_with_usage(
"parity-priced-model",
json!({}),
));
let chat_cost = chat.usage.unwrap().cost.expect("chat estimated cost");
let anthropic_cost = anthropic
.usage
.unwrap()
.cost
.expect("anthropic estimated cost");
let responses_cost = responses
.usage
.unwrap()
.cost
.expect("responses estimated cost");
assert_eq!(chat_cost, anthropic_cost);
assert_eq!(chat_cost, responses_cost);
let total = chat_cost.total.expect("estimated total");
assert!(
(total - 0.000_435).abs() < 1e-9,
"unexpected estimated total: {total}"
);
assert_eq!(chat_cost.currency, "USD");
assert_eq!(chat_cost.source, CostSource::ModelPricing);
assert_eq!(chat_cost.pricing_provider.as_deref(), Some("test"));
assert_eq!(
chat_cost.pricing_model.as_deref(),
Some("parity-priced-model")
);
}
#[test]
fn test_request_hint_never_overrides_strong_signals() {
let responses_request = req(json!({
"model": "gpt-parity",
"instructions": "You are terse.",
"input": "Summarize the docs.",
"max_output_tokens": 64
}));
let hinted = normalize_request_with_hint(&responses_request, Some("anthropic"))
.expect("responses request decodes despite wrong hint");
assert_eq!(
hinted,
normalize_request(&responses_request).expect("responses request decodes"),
);
assert_eq!(hinted.max_output_tokens, Some(64));
let anthropic_request = req(json!({
"model": "claude-parity",
"system": "You are terse.",
"messages": [{"role": "user", "content": "Summarize the docs."}],
"stop_sequences": ["END"]
}));
let hinted = normalize_request_with_hint(&anthropic_request, Some("openai.chat"))
.expect("anthropic request decodes despite wrong hint");
assert_eq!(
hinted,
normalize_request(&anthropic_request).expect("anthropic request decodes"),
);
let stop = hinted
.params
.as_ref()
.and_then(|params| params.stop.as_ref())
.expect("anthropic stop_sequences normalized");
assert_eq!(stop, &vec!["END".to_string()]);
assert!(!hinted.extra.contains_key("stop_sequences"));
}
#[test]
fn test_request_unknown_hint_matches_hintless_normalization() {
let bodies = [
json!({"model": "m", "instructions": "sys", "input": "hi"}),
json!({"model": "m", "system": "sys", "messages": [{"role": "user", "content": "hi"}]}),
json!({"model": "m", "messages": [{"role": "user", "content": "hi"}]}),
];
for body in &bodies {
let request = req(body.clone());
let baseline = normalize_request(&request).expect("canonical body decodes");
for hint in ["gemini", "not-a-provider", "anthropic.count_tokens"] {
assert_eq!(
normalize_request_with_hint(&request, Some(hint)).as_ref(),
Some(&baseline),
"hint {hint:?} must not change normalization for {body}",
);
}
}
}
#[test]
fn test_request_hint_none_equals_normalize_request() {
let bodies = [
json!({"model": "m", "instructions": "sys", "input": "hi"}),
json!({"model": "m", "system": "sys", "messages": [{"role": "user", "content": "hi"}]}),
json!({"model": "m", "messages": [{"role": "user", "content": "hi"}]}),
];
for body in &bodies {
let request = req(body.clone());
assert_eq!(
normalize_request_with_hint(&request, None),
normalize_request(&request),
"hint=None must equal normalize_request for {body}",
);
}
}
#[test]
fn test_request_normalization_parity() {
let chat = normalize_request(&req(json!({
"model": "parity-request-model",
"messages": [
{"role": "system", "content": "You are terse."},
{"role": "user", "content": "Summarize the docs."}
],
"temperature": 0.5,
"max_tokens": 256,
"stop": ["END"]
})))
.expect("chat request decodes");
let anthropic = normalize_request(&req(json!({
"model": "parity-request-model",
"system": "You are terse.",
"messages": [{"role": "user", "content": "Summarize the docs."}],
"temperature": 0.5,
"max_tokens": 256,
"stop_sequences": ["END"]
})))
.expect("anthropic request decodes");
let responses = normalize_request(&req(json!({
"model": "parity-request-model",
"instructions": "You are terse.",
"input": "Summarize the docs.",
"temperature": 0.5,
"max_output_tokens": 256
})))
.expect("responses request decodes");
let expected_messages = vec![
Message::System {
content: MessageContent::Text("You are terse.".to_string()),
name: None,
},
Message::User {
content: MessageContent::Text("Summarize the docs.".to_string()),
name: None,
},
];
assert_eq!(chat.messages, expected_messages);
let expected_user_messages = vec![Message::User {
content: MessageContent::Text("Summarize the docs.".to_string()),
name: None,
}];
assert_eq!(anthropic.messages, expected_user_messages);
assert_eq!(responses.messages, expected_user_messages);
assert_eq!(
anthropic.instructions,
Some(MessageContent::Text("You are terse.".to_string()))
);
assert_eq!(anthropic.instructions, responses.instructions);
for decoded in [&chat, &anthropic, &responses] {
assert_eq!(decoded.model.as_deref(), Some("parity-request-model"));
assert_eq!(decoded.system_prompt(), Some("You are terse."));
assert_eq!(decoded.last_user_message(), Some("Summarize the docs."));
}
let expected_params = GenerationParams {
temperature: Some(0.5),
max_tokens: Some(256),
top_p: None,
stop: Some(vec!["END".to_string()]),
};
assert_eq!(chat.params, Some(expected_params.clone()));
assert_eq!(anthropic.params, Some(expected_params));
assert_eq!(
responses.params,
Some(GenerationParams {
temperature: Some(0.5),
max_tokens: Some(256),
top_p: None,
stop: None,
})
);
assert_eq!(responses.max_output_tokens, Some(256));
assert_eq!(chat.max_output_tokens, None);
assert_eq!(anthropic.max_output_tokens, None);
}
#[test]
fn baseline_patching_keeps_raw_array_fields_with_reordered_logical_items() {
let original = json!([
{"type": "text", "text": "a", "provider_marker": "A"},
{"type": "text", "text": "b", "provider_marker": "B"},
{"type": "text", "text": "c", "provider_marker": "C"}
]);
let baseline = json!([
{"type": "text", "text": "a"},
{"type": "text", "text": "b"},
{"type": "text", "text": "c"}
]);
let edited = json!([
{"type": "text", "text": "b"},
{"type": "text", "text": "a"},
{"type": "text", "text": "c"}
]);
assert_eq!(
super::patch_changed_json(&original, &baseline, &edited).unwrap(),
json!([
{"type": "text", "text": "b", "provider_marker": "B"},
{"type": "text", "text": "a", "provider_marker": "A"},
{"type": "text", "text": "c", "provider_marker": "C"}
])
);
}
#[test]
fn baseline_patching_does_not_pair_reordered_deletion_with_unrelated_insertion() {
let original = json!([
{"type": "text", "text": "a", "provider_marker": "A"},
{"type": "text", "text": "b", "provider_marker": "B"}
]);
let baseline = json!([
{"type": "text", "text": "a"},
{"type": "text", "text": "b"}
]);
let edited = json!([
{"type": "text", "text": "new"},
{"type": "text", "text": "a"}
]);
assert_eq!(
super::patch_changed_json(&original, &baseline, &edited).unwrap(),
json!([
{"type": "text", "text": "new"},
{"type": "text", "text": "a", "provider_marker": "A"}
])
);
}
#[test]
fn baseline_patching_handles_insert_delete_and_duplicate_reorders() {
let duplicate_a = json!({"type": "text", "text": "a"});
let b = json!({"type": "text", "text": "b"});
let original = json!([
{"type": "text", "text": "a", "provider_marker": "A1"},
{"type": "text", "text": "a", "provider_marker": "A2"},
{"type": "text", "text": "b", "provider_marker": "B"}
]);
let baseline = Json::Array(vec![duplicate_a.clone(), duplicate_a.clone(), b.clone()]);
let edited = Json::Array(vec![
duplicate_a.clone(),
b,
duplicate_a,
json!({"type": "text", "text": "new"}),
]);
assert_eq!(
super::patch_changed_json(&original, &baseline, &edited).unwrap(),
json!([
{"type": "text", "text": "a", "provider_marker": "A1"},
{"type": "text", "text": "b", "provider_marker": "B"},
{"type": "text", "text": "a", "provider_marker": "A2"},
{"type": "text", "text": "new"}
])
);
}
#[test]
fn baseline_patching_rejects_multiple_reordered_and_edited_items_without_provenance() {
let original = json!([
{"type": "text", "text": "a", "provider_marker": "A"},
{"type": "text", "text": "b", "provider_marker": "B"}
]);
let baseline = json!([
{"type": "text", "text": "a"},
{"type": "text", "text": "b"}
]);
let edited = json!([
{"type": "text", "text": "b-edited"},
{"type": "text", "text": "a-edited"}
]);
let error = super::patch_changed_json(&original, &baseline, &edited).unwrap_err();
assert!(
error
.to_string()
.contains("multiple edited array items without stable identities")
);
}