import importlib.util
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
MODULE_PATH = REPO_ROOT / "scripts" / "openapi_drift.py"
SPEC = importlib.util.spec_from_file_location("openapi_drift", MODULE_PATH)
assert SPEC is not None and SPEC.loader is not None
openapi_drift = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(openapi_drift)
def build_spec(
*,
method="get",
path="/models",
operation_id="getModels",
parameters=None,
request_properties=None,
response_properties=None,
):
operation = {
"operationId": operation_id,
"responses": {
"200": {
"description": "ok",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": response_properties or {
"id": {"type": "string"},
},
}
}
},
}
},
}
if request_properties is not None:
operation["requestBody"] = {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": request_properties,
}
}
}
}
if parameters is not None:
operation["parameters"] = parameters
return {
"openapi": "3.1.0",
"info": {"title": "Test", "version": "1.0.0"},
"paths": {
path: {
method: operation,
}
},
}
class OpenApiDriftReportTests(unittest.TestCase):
def test_nullable_dialect_change_is_not_reported_as_drift(self):
baseline = build_spec(
response_properties={
"metadata": {
"type": "object",
"nullable": True,
"additionalProperties": {"nullable": True},
}
}
)
candidate = build_spec(
response_properties={
"metadata": {
"type": ["object", "null"],
"additionalProperties": {},
}
}
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertFalse(report["has_drift"])
def test_reduce_spec_for_baseline_sanitizes_openrouter_api_key_examples(self):
raw_key = "sk-or-v1-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ab"
spec = build_spec(
method="post",
path="/keys",
operation_id="createKey",
response_properties={
"key": {
"type": "string",
"example": raw_key,
},
"label": {
"type": "string",
"example": "sk-or-v1-...1c96",
},
},
)
spec["paths"]["/keys"]["post"]["responses"]["200"]["content"][
"application/json"
]["example"] = {"key": raw_key}
reduced = openapi_drift.reduce_spec_for_baseline(spec)
content = reduced["paths"]["/keys"]["post"]["responses"]["200"]["content"][
"application/json"
]
self.assertEqual(content["example"]["key"], "sk-or-v1-[REDACTED]")
self.assertEqual(
content["schema"]["properties"]["key"]["example"],
"sk-or-v1-[REDACTED]",
)
self.assertEqual(
content["schema"]["properties"]["label"]["example"],
"sk-or-v1-...1c96",
)
def test_validate_openapi_spec_sanitizes_openrouter_api_key_examples(self):
raw_key = "sk-or-v1-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ab"
spec = build_spec(
method="post",
path="/auth/keys",
operation_id="createAuthKey",
response_properties={
"key": {
"type": "string",
"example": raw_key,
},
},
)
validated = openapi_drift.validate_openapi_spec(spec, "test spec")
content = validated["paths"]["/auth/keys"]["post"]["responses"]["200"][
"content"
]["application/json"]
self.assertEqual(
content["schema"]["properties"]["key"]["example"],
"sk-or-v1-[REDACTED]",
)
def test_metadata_only_header_drift_is_classified_as_already_supported(self):
baseline = build_spec()
candidate = build_spec(
parameters=[
{
"in": "header",
"name": "HTTP-Referer",
"schema": {"type": "string"},
},
{
"in": "header",
"name": "X-OpenRouter-Title",
"schema": {"type": "string"},
"x-speakeasy-name-override": "appTitle",
},
{
"in": "header",
"name": "X-Title",
"schema": {"type": "string"},
},
{
"in": "header",
"name": "X-OpenRouter-Categories",
"schema": {"type": "string"},
"x-speakeasy-name-override": "appCategories",
},
]
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["summary"]["changed"], 1)
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(
report["changed"][0]["repo_impact"]["category"],
"already_supported",
)
def test_non_metadata_schema_change_remains_actionable(self):
baseline = build_spec(
parameters=[
{
"in": "header",
"name": "HTTP-Referer",
"schema": {"type": "string"},
}
]
)
candidate = build_spec(
parameters=[
{
"in": "header",
"name": "HTTP-Referer",
"schema": {"type": "string"},
}
],
response_properties={
"id": {"type": "string"},
"num_fetches": {"type": "integer", "nullable": True},
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertTrue(report["has_actionable_drift"])
self.assertEqual(report["summary"]["changed"], 1)
self.assertEqual(report["repo_summary"]["already_supported_changed"], 0)
self.assertEqual(report["repo_summary"]["actionable_changed"], 1)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "actionable")
def test_required_metadata_header_change_remains_actionable(self):
baseline = build_spec(
parameters=[
{
"in": "header",
"name": "X-OpenRouter-Title",
"schema": {"type": "string"},
"x-speakeasy-name-override": "appTitle",
}
]
)
candidate = build_spec(
parameters=[
{
"in": "header",
"name": "X-OpenRouter-Title",
"required": True,
"schema": {"type": "string"},
"x-speakeasy-name-override": "appTitle",
}
]
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertTrue(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 0)
self.assertEqual(report["repo_summary"]["actionable_changed"], 1)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "actionable")
self.assertEqual(
report["changed"][0]["repo_impact"]["supported_parameters"],
["header X-OpenRouter-Title"],
)
def test_dynamic_provider_enum_drift_is_classified_as_already_supported(self):
baseline = build_spec(
response_properties={
"provider_name": {
"enum": ["Anthropic", "Google", "OpenAI"],
"type": "string",
"x-speakeasy-unknown-values": "allow",
}
}
)
candidate = build_spec(
response_properties={
"provider_name": {
"enum": ["Anthropic", "Google", "Nex AGI", "OpenAI"],
"type": "string",
"x-speakeasy-unknown-values": "allow",
}
}
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "already_supported")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["dynamic provider name enum"],
)
def test_dynamic_provider_slug_drift_is_classified_as_already_supported(self):
baseline = build_spec(
response_properties={
"provider": {
"enum": ["anthropic", "google-vertex", "openai"],
"type": "string",
"x-speakeasy-unknown-values": "allow",
}
}
)
candidate = build_spec(
response_properties={
"provider": {
"enum": ["anthropic", "coreweave", "google-vertex", "openai"],
"type": "string",
"x-speakeasy-unknown-values": "allow",
}
}
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["dynamic provider name enum"],
)
def test_hidden_operation_metadata_is_not_reported_as_drift(self):
baseline = build_spec()
candidate = build_spec()
candidate["paths"]["/models"]["get"]["x-hidden"] = True
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertFalse(report["has_drift"])
def test_dynamic_enum_with_object_members_does_not_crash(self):
baseline = build_spec(
response_properties={
"provider_name": {
"enum": ["Anthropic", "Google", "OpenAI", {"custom": True}],
"type": "string",
"x-speakeasy-unknown-values": "allow",
}
}
)
candidate = build_spec(
response_properties={
"provider_name": {
"enum": ["Anthropic", "Google", "Nex AGI", "OpenAI", {"custom": True}],
"type": "string",
"x-speakeasy-unknown-values": "allow",
}
}
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "already_supported")
def test_dynamic_modality_enum_drift_is_classified_as_already_supported(self):
baseline = build_spec(
response_properties={
"output_modalities": {
"items": {
"enum": ["audio", "image", "text", "tts", "video"],
"type": "string",
"x-speakeasy-unknown-values": "allow",
},
"type": "array",
}
}
)
candidate = build_spec(
response_properties={
"output_modalities": {
"items": {
"enum": ["audio", "image", "speech", "text", "video"],
"type": "string",
"x-speakeasy-unknown-values": "allow",
},
"type": "array",
}
}
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "already_supported")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["dynamic output modality enum"],
)
def test_provider_options_map_drift_is_classified_as_already_supported(self):
provider_option = {
"additionalProperties": {"nullable": True},
"type": "object",
}
baseline = build_spec(
response_properties={
"provider": {
"properties": {
"anthropic": provider_option,
"google-vertex": provider_option,
"openai": provider_option,
},
"type": "object",
}
}
)
candidate = build_spec(
response_properties={
"provider": {
"properties": {
"anthropic": provider_option,
"google-vertex": provider_option,
"nex-agi": provider_option,
"openai": provider_option,
},
"type": "object",
}
}
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "already_supported")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["provider-specific options map"],
)
def test_responses_nullable_drift_is_classified_as_already_supported(self):
baseline = build_spec(
method="post",
path="/responses",
operation_id="createResponses",
response_properties={
"top_logprobs": {"nullable": True, "type": "integer"},
},
)
candidate = build_spec(
method="post",
path="/responses",
operation_id="createResponses",
response_properties={
"top_logprobs": {"type": "integer"},
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "already_supported")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["Responses flexible nullable fields"],
)
def test_non_responses_nullable_drift_remains_actionable(self):
baseline = build_spec(
response_properties={
"top_logprobs": {"nullable": True, "type": "integer"},
},
)
candidate = build_spec(
response_properties={
"top_logprobs": {"type": "integer"},
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertTrue(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 0)
self.assertEqual(report["repo_summary"]["actionable_changed"], 1)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "actionable")
def test_responses_request_nullable_drift_remains_actionable(self):
baseline = build_spec(
method="post",
path="/responses",
operation_id="createResponses",
request_properties={
"top_logprobs": {"nullable": True, "type": "integer"},
},
)
candidate = build_spec(
method="post",
path="/responses",
operation_id="createResponses",
request_properties={
"top_logprobs": {"type": "integer"},
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertTrue(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 0)
self.assertEqual(report["repo_summary"]["actionable_changed"], 1)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "actionable")
def test_flexible_plugin_property_drift_is_classified_as_already_supported(self):
baseline = build_spec(
method="post",
path="/chat/completions",
operation_id="sendChatCompletionRequest",
request_properties={
"plugins": {
"items": {
"oneOf": [
{
"properties": {
"id": {"enum": ["web"], "type": "string"},
"max_results": {"type": "integer"},
},
"required": ["id"],
"type": "object",
}
]
},
"type": "array",
}
},
)
candidate = build_spec(
method="post",
path="/chat/completions",
operation_id="sendChatCompletionRequest",
request_properties={
"plugins": {
"items": {
"oneOf": [
{
"properties": {
"id": {"enum": ["web"], "type": "string"},
"max_results": {"type": "integer"},
"max_uses": {"type": "integer"},
},
"required": ["id"],
"type": "object",
}
]
},
"type": "array",
}
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "already_supported")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["flexible plugin payload"],
)
def test_flexible_plugin_container_type_change_remains_actionable(self):
baseline = build_spec(
method="post",
path="/chat/completions",
operation_id="sendChatCompletionRequest",
request_properties={
"plugins": {
"items": {
"oneOf": [
{
"properties": {
"id": {"enum": ["web"], "type": "string"},
},
"required": ["id"],
"type": "object",
}
]
},
"type": "array",
}
},
)
candidate = build_spec(
method="post",
path="/chat/completions",
operation_id="sendChatCompletionRequest",
request_properties={
"plugins": {
"properties": {
"id": {"enum": ["web"], "type": "string"},
},
"type": "object",
}
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertTrue(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 0)
self.assertEqual(report["repo_summary"]["actionable_changed"], 1)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "actionable")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["flexible plugin payload"],
)
def test_chat_server_tool_drift_is_classified_as_already_supported(self):
baseline = build_spec(
method="post",
path="/chat/completions",
operation_id="sendChatCompletionRequest",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"function": {
"properties": {
"name": {"type": "string"},
},
"required": ["name"],
"type": "object",
},
"type": {"enum": ["function"], "type": "string"},
},
"required": ["type", "function"],
"type": "object",
}
]
},
"type": "array",
}
},
)
candidate = build_spec(
method="post",
path="/chat/completions",
operation_id="sendChatCompletionRequest",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"function": {
"properties": {
"name": {"type": "string"},
"strict": {"type": "boolean"},
},
"required": ["name"],
"type": "object",
},
"type": {"enum": ["function"], "type": "string"},
},
"required": ["type", "function"],
"type": "object",
},
{
"properties": {
"parameters": {
"properties": {
"max_results": {"type": "integer"},
},
"type": "object",
},
"type": {
"enum": ["openrouter:web_search"],
"type": "string",
},
},
"required": ["type"],
"type": "object",
},
]
},
"type": "array",
}
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "already_supported")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["Chat server-tool payload", "Chat supported function-tool fields"],
)
def test_chat_function_tool_shape_change_remains_actionable(self):
baseline = build_spec(
method="post",
path="/chat/completions",
operation_id="sendChatCompletionRequest",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"function": {
"properties": {
"name": {"type": "string"},
},
"required": ["name"],
"type": "object",
},
"type": {"enum": ["function"], "type": "string"},
},
"required": ["type", "function"],
"type": "object",
}
]
},
"type": "array",
}
},
)
candidate = build_spec(
method="post",
path="/chat/completions",
operation_id="sendChatCompletionRequest",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"function": {
"properties": {
"name": {"type": "string"},
"new_required": {"type": "string"},
},
"required": ["name", "new_required"],
"type": "object",
},
"type": {"enum": ["function"], "type": "string"},
},
"required": ["type", "function"],
"type": "object",
}
]
},
"type": "array",
}
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertTrue(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 0)
self.assertEqual(report["repo_summary"]["actionable_changed"], 1)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "actionable")
self.assertEqual(report["changed"][0]["repo_impact"]["schema_rules"], [])
def test_chat_tool_container_type_change_remains_actionable(self):
baseline = build_spec(
method="post",
path="/chat/completions",
operation_id="sendChatCompletionRequest",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"type": {"enum": ["function"], "type": "string"},
},
"required": ["type"],
"type": "object",
}
]
},
"type": "array",
}
},
)
candidate = build_spec(
method="post",
path="/chat/completions",
operation_id="sendChatCompletionRequest",
request_properties={
"tools": {
"properties": {
"type": {"enum": ["function"], "type": "string"},
},
"type": "object",
}
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertTrue(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 0)
self.assertEqual(report["repo_summary"]["actionable_changed"], 1)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "actionable")
self.assertEqual(report["changed"][0]["repo_impact"]["schema_rules"], [])
def test_responses_value_tool_and_output_drift_is_classified_as_already_supported(self):
baseline = build_spec(
method="post",
path="/responses",
operation_id="createResponses",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"name": {"type": "string"},
"type": {"enum": ["function"], "type": "string"},
},
"required": ["type", "name"],
"type": "object",
}
]
},
"type": "array",
}
},
response_properties={
"output": {
"items": {
"anyOf": [
{
"properties": {
"type": {"enum": ["message"], "type": "string"},
},
"required": ["type"],
"type": "object",
}
]
},
"type": "array",
}
},
)
candidate = build_spec(
method="post",
path="/responses",
operation_id="createResponses",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"name": {"type": "string"},
"type": {"enum": ["function"], "type": "string"},
},
"required": ["type", "name"],
"type": "object",
},
{
"properties": {
"type": {"enum": ["web_search_preview"], "type": "string"},
"max_results": {"type": "integer"},
},
"required": ["type"],
"type": "object",
},
]
},
"type": "array",
}
},
response_properties={
"output": {
"items": {
"anyOf": [
{
"properties": {
"type": {"enum": ["message"], "type": "string"},
},
"required": ["type"],
"type": "object",
},
{
"properties": {
"type": {"enum": ["code_interpreter_call"], "type": "string"},
"outputs": {
"items": {"type": "object"},
"type": "array",
},
},
"required": ["type"],
"type": "object",
},
]
},
"type": "array",
}
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "already_supported")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["Responses flexible output payload", "Responses flexible tool payload"],
)
def test_preset_responses_tool_drift_is_classified_as_already_supported(self):
baseline = build_spec(
method="post",
path="/presets/{slug}/responses",
operation_id="createResponsesPreset",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"name": {"type": "string"},
"type": {"enum": ["function"], "type": "string"},
},
"required": ["type", "name"],
"type": "object",
}
]
},
"type": "array",
}
},
)
candidate = build_spec(
method="post",
path="/presets/{slug}/responses",
operation_id="createResponsesPreset",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"name": {"type": "string"},
"type": {"enum": ["function"], "type": "string"},
},
"required": ["type", "name"],
"type": "object",
},
{
"properties": {
"type": {
"enum": ["openrouter:datetime"],
"type": "string",
},
},
"required": ["type"],
"type": "object",
},
]
},
"type": "array",
}
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "already_supported")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["Responses flexible tool payload"],
)
def test_responses_flexible_container_type_changes_remain_actionable(self):
baseline = build_spec(
method="post",
path="/responses",
operation_id="createResponses",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"name": {"type": "string"},
"type": {"enum": ["function"], "type": "string"},
},
"required": ["type", "name"],
"type": "object",
}
]
},
"type": "array",
}
},
response_properties={
"output": {
"items": {
"anyOf": [
{
"properties": {
"type": {"enum": ["message"], "type": "string"},
},
"required": ["type"],
"type": "object",
}
]
},
"type": "array",
}
},
)
candidate = build_spec(
method="post",
path="/responses",
operation_id="createResponses",
request_properties={
"tools": {
"properties": {
"type": {"enum": ["function"], "type": "string"},
},
"type": "object",
}
},
response_properties={
"output": {
"properties": {
"type": {"enum": ["message"], "type": "string"},
},
"type": "object",
}
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertTrue(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 0)
self.assertEqual(report["repo_summary"]["actionable_changed"], 1)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "actionable")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["Responses flexible output payload", "Responses flexible tool payload"],
)
def test_messages_flexible_tool_option_drift_is_classified_as_already_supported(self):
baseline = build_spec(
method="post",
path="/messages",
operation_id="createMessages",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"name": {"enum": ["web_search"], "type": "string"},
"type": {
"enum": ["web_search_20250305"],
"type": "string",
},
},
"required": ["type", "name"],
"type": "object",
}
]
},
"type": "array",
}
},
)
candidate = build_spec(
method="post",
path="/messages",
operation_id="createMessages",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"max_uses": {"nullable": True, "type": "integer"},
"name": {"enum": ["web_search"], "type": "string"},
"type": {
"enum": ["web_search_20250305"],
"type": "string",
},
"user_location": {
"properties": {
"type": {
"enum": ["approximate"],
"type": "string",
}
},
"type": "object",
},
},
"required": ["type", "name"],
"type": "object",
}
]
},
"type": "array",
}
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertFalse(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 1)
self.assertEqual(report["repo_summary"]["actionable_changed"], 0)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "already_supported")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["Messages flexible tool payload"],
)
def test_messages_flexible_tool_container_type_change_remains_actionable(self):
baseline = build_spec(
method="post",
path="/messages",
operation_id="createMessages",
request_properties={
"tools": {
"items": {
"anyOf": [
{
"properties": {
"name": {"enum": ["web_search"], "type": "string"},
"type": {
"enum": ["web_search_20250305"],
"type": "string",
},
},
"required": ["type", "name"],
"type": "object",
}
]
},
"type": "array",
}
},
)
candidate = build_spec(
method="post",
path="/messages",
operation_id="createMessages",
request_properties={
"tools": {
"properties": {
"name": {"enum": ["web_search"], "type": "string"},
"type": {
"enum": ["web_search_20250305"],
"type": "string",
},
},
"type": "object",
}
},
)
report = openapi_drift.build_report(
baseline_spec=baseline,
candidate_spec=candidate,
baseline_label="baseline",
candidate_label="candidate",
source_url="https://example.com/openapi.json",
max_diff_lines=20,
)
self.assertTrue(report["has_drift"])
self.assertTrue(report["has_actionable_drift"])
self.assertEqual(report["repo_summary"]["already_supported_changed"], 0)
self.assertEqual(report["repo_summary"]["actionable_changed"], 1)
self.assertEqual(report["changed"][0]["repo_impact"]["category"], "actionable")
self.assertEqual(
report["changed"][0]["repo_impact"]["schema_rules"],
["Messages flexible tool payload"],
)
if __name__ == "__main__":
unittest.main()