rolter 0.0.8

# rolter bootstrap config
#
# copy to rolter.toml and adjust. secrets should come from environment variables
# referenced by `api_key_env`, not be written inline.

[server]
host = "0.0.0.0"
port = 4000
# path the prometheus metrics endpoint is served on. change it when an upstream
# app or sidecar already owns /metrics behind the same reverse proxy. defaults
# to "/metrics"; must be rooted and must not collide with a built-in route.
# metrics_path = "/internal/metrics"
# secret mixed into virtual-key digests (keeps plaintext keys out of gateway
# memory). prefer the ROLTER_KEY_PEPPER env var over writing it inline.
# key_pepper = "change-me"
# max accepted request body size in bytes. raises axum's restrictive 2 MiB
# default so large prompts / base64 images / audio uploads aren't rejected; an
# oversized body gets an openai-style 413. defaults to 33554432 (32 MiB).
# max_body_bytes = 33554432

# --- outbound TLS trust ---
# PEM certificates in these files are added to the normal public-root trust
# store. ROLTER_CA_BUNDLE overrides this global list with one container path.
# [tls]
# ca_bundles = ["/etc/rolter/ca/private-root.pem"]

# --- providers: where rolter can forward to ---

[[providers]]
name = "openai"
kind = "openai"
api_base = "https://api.openai.com"
api_key_env = "OPENAI_API_KEY"
# egress_proxy = "http://corp-proxy:3128"   # optional outbound proxy
# ca_bundles = ["/etc/rolter/ca/openai.pem"] # optional provider override

[[providers]]
name = "anthropic"
kind = "anthropic"
api_base = "https://api.anthropic.com"
api_key_env = "ANTHROPIC_API_KEY"

[[providers]]
name = "openrouter"
kind = "openrouter"
api_base = "https://openrouter.ai/api/v1"
api_key_env = "OPENROUTER_API_KEY"

# OpenAI-compatible cloud-provider endpoints (uncomment the providers you use).
# Azure uses `api-key`; Bedrock and Vertex use bearer tokens.
# [[providers]]
# name = "azure"
# kind = "azure_openai"
# api_base = "https://RESOURCE.openai.azure.com/openai/v1"
# api_key_env = "AZURE_OPENAI_API_KEY"
#
# [[providers]]
# name = "bedrock"
# kind = "bedrock"
# api_base = "https://bedrock-runtime.us-east-1.amazonaws.com/v1"
# api_key_env = "AWS_BEARER_TOKEN_BEDROCK"
#
# [[providers]]
# name = "vertex"
# kind = "vertex"
# api_base = "https://aiplatform.googleapis.com/v1/projects/PROJECT/locations/global/endpoints/openapi"
# api_key_env = "VERTEX_ACCESS_TOKEN"

# direct Cloud access requires an environment-sourced bearer key
[[providers]]
name = "ollama-cloud"
kind = "ollama_cloud"
api_base = "https://ollama.com"
api_key_env = "OLLAMA_API_KEY"

[[providers]]
name = "vllm-1"
kind = "openai_compatible"
api_base = "http://localhost:8001"
# OpenAI-compatible providers default to `system_only`: developer instructions
# become system instructions, preserving their order. Override only when the
# configured model template explicitly supports developer messages.
# role_profile = "openai"
# [providers.model_role_profiles]
# "custom-developer-template" = "openai"

[[providers]]
name = "vllm-2"
kind = "openai_compatible"
api_base = "http://localhost:8002"

# self-hosted ollama needs no api key. use http://ollama:11434 from the
# docker compose network; rolter appends the /v1 endpoint paths itself.
[[providers]]
name = "ollama"
kind = "ollama"
api_base = "http://localhost:11434"

# --- routes: public model name -> upstream targets + strategy ---

[[routes]]
model = "gpt-4o"
strategy = "round_robin"
# admin-set default inference params injected into the request body. an unset
# param passes through untouched. param_policy governs whether callers may
# override a default: mode "allow" (default) lets callers override anything not
# in `deny`; mode "deny" pins every default except those in `allow`. when an
# override is denied the admin default silently wins.
# [routes.params]
# temperature = 0.0        # force deterministic output for this model
# max_tokens = 1024        # cap output length
# [routes.param_policy]
# mode = "deny"
# allow = ["max_tokens"]   # callers may still lower max_tokens
[[routes.targets]]
provider = "openai"
model = "gpt-4o"

[[routes]]
model = "claude"
strategy = "round_robin"
[[routes.targets]]
provider = "anthropic"
model = "claude-sonnet-4-20250514"

# a self-hosted pool of vllm instances balanced with cache-aware routing
[[routes]]
model = "llama"
strategy = "cache_aware"
[[routes.targets]]
provider = "vllm-1"
model = "meta-llama/Llama-3.1-8B-Instruct"
[[routes.targets]]
provider = "vllm-2"
model = "meta-llama/Llama-3.1-8B-Instruct"

# weighted routing: send 3x as much traffic to the first target as the second
# (smooth weighted round-robin — proportional but evenly interleaved)
[[routes]]
model = "llama-weighted"
strategy = "weighted"
[[routes.targets]]
provider = "vllm-1"
model = "meta-llama/Llama-3.1-8B-Instruct"
weight = 3
[[routes.targets]]
provider = "vllm-2"
model = "meta-llama/Llama-3.1-8B-Instruct"
weight = 1

# composable pipeline: filter → weighted-score → argmax. blends the target
# weight, in-flight load, and prefix-cache affinity scorers into one decision —
# the foundation future cost/latency/kv-cache scorers plug into
[[routes]]
model = "llama-pipeline"
strategy = "pipeline"
[[routes.targets]]
provider = "vllm-1"
model = "meta-llama/Llama-3.1-8B-Instruct"
weight = 2
[[routes.targets]]
provider = "vllm-2"
model = "meta-llama/Llama-3.1-8B-Instruct"
weight = 1

# weighted variants: A/B, canary, and key-split traffic under one schema. a
# request samples one variant by weight (the primary) and, on failure, falls
# back to the remaining variants in declared order; within a variant the
# targets are tried in order. variant-level params (below) layer over the
# route-level params. the chosen variant name is attributed in request logs
# (the `variant` column) for A/B analysis. a route uses variants instead of the
# top-level targets/strategy pool.
# [[routes]]
# model = "chat"
# [[routes.variants]]
# name = "control"
# weight = 9
# [[routes.variants.targets]]
# provider = "openai"
# model = "gpt-4o"
# [[routes.variants]]
# name = "canary"
# weight = 1
# [[routes.variants.targets]]
# provider = "anthropic"
# model = "claude-sonnet-4-20250514"

# --- virtual keys clients present to the gateway ---
# remove this section to run open (no auth) in local development

[[virtual_keys]]
key = "sk-rolter-dev"
name = "local-dev"
models = []           # empty = all models allowed
disabled = false      # revoke without deleting
# expires_at = "2027-01-01T00:00:00Z"   # rfc3339; key stops working at/after this
# cache = false   # per-key response-cache override, independent of a route's opt-in:
#                 # omit to inherit the route, false to bypass, true to cache even on
#                 # a non-opted-in route (the global [cache] switch is still required)

# --- per-model token pricing (usd per million tokens) ---
# drives cost_usd on each request log; models without a price log cost 0
# [[model_prices]]
# model = "gpt-4o"
# input_per_mtok = 2.5
# output_per_mtok = 10.0
# cached_input_per_mtok = 1.25   # optional; defaults to input_per_mtok

# --- spend caps (enforced against redis-tracked cost) ---
# a request is blocked with 402 when any matching budget's spend has reached its
# limit (most-restrictive-wins across the scope chain). requires --redis-url;
# without redis, budgets fail open. `id` matches the request's virtual-key scope
# ids (org/team/project/virtual-key id); config keys carry no scope, so these
# target database-defined keys.
# [[budgets]]
# scope = "org"        # org | team | project | key
# id = "org-uuid"
# limit_usd = 500.0
# period = "monthly"   # monthly (default) | daily | total

# --- throughput caps (redis sliding window, per-minute) ---
# a request is rejected with 429 (+ retry-after) when a matching limit's
# trailing-minute request or token count has reached its cap (most-restrictive-
# wins across the scope chain). requires --redis-url; without redis, limits fail
# open. `id` matches the request's virtual-key scope ids, like budgets above.
# [[rate_limits]]
# scope = "key"        # org | team | project | key
# id = "vk-uuid"
# rpm = 600            # requests per minute; omit to leave requests uncapped
# tpm = 150000         # tokens per minute; omit to leave tokens uncapped

# --- upstream retry policy ---
# on a transient upstream failure (http 408/429/5xx or a connection error) the
# gateway re-picks a target (excluding ones already tried, so retries fail over
# to sibling targets) and forwards again. backoff is exponential with jitter; a
# 429 retry-after header (whole seconds) overrides the computed delay. retries
# only happen before any body bytes reach the client. defaults shown below.
# [retry]
# max_retries = 2         # extra attempts after the first; 0 disables retries
# base_backoff_ms = 100   # base delay for the first retry
# max_backoff_ms = 2000   # ceiling for the backoff delay

# --- per-target cooldowns ---
# after a target returns a transient failure (http 429/5xx or a connection
# error) it is parked for base_secs (or the 429 retry-after, capped at max_secs)
# and the balancer skips it so healthy siblings absorb the load. when every
# sibling is parked the gateway fails open and still forwards. defaults shown.
# [cooldown]
# base_secs = 5     # how long a failing target is parked; 0 disables cooldowns
# max_secs = 300    # ceiling for a cooldown derived from a 429 retry-after

# --- upstream timeouts ---
# connect_secs bounds establishing the connection; request_secs bounds time to
# response headers (not the body, so long SSE streams are never cut short). a
# timeout surfaces as a transient error and feeds the retry/cooldown paths. set
# a field to 0 to disable it. defaults shown below.
# [timeouts]
# connect_secs = 10
# request_secs = 60

# --- per-provider queues and backpressure ---
# each provider gets an independent bounded request queue and worker set, so a
# slow or unavailable upstream cannot exhaust gateway admission capacity for
# healthy providers. workers limit concurrent waits for response headers; SSE
# bodies still stream directly to the client after headers arrive. defaults:
# [queue]
# enabled = true
# capacity = 256             # waiting requests per provider
# workers = 8                # concurrent header requests per provider
# backpressure = "error"      # drop | block | error
# block_timeout_ms = 1000     # only used by backpressure = "block"

# --- active upstream health checks ---
# when enabled, a background task periodically issues a lightweight GET
# {api_base}{path} to each provider; a provider that times out, fails to connect,
# or answers 5xx is marked unhealthy and the balancer skips its targets until a
# later probe recovers it (any non-5xx status, incl. 401/404, counts as healthy).
# when every target of a route is unhealthy the gateway fails open. disabled by
# default. defaults shown below.
# [health]
# enabled = false
# interval_secs = 10   # seconds between probe sweeps
# timeout_secs = 2     # per-probe timeout
# path = "/"           # appended to each provider's api_base when probing

# --- per-target circuit breaker ---
# complements the short per-failure cooldown with a longer-lived state machine.
# after failure_threshold consecutive transient failures a target trips OPEN and
# is skipped for open_secs; the first request after that window probes it
# (half-open), a success closes the breaker and a failure re-opens it. where a
# cooldown shrugs off one wobble, the breaker sheds load off a target that is down
# hard. when every target of a route is open the gateway fails open. disabled by
# default. defaults shown below.
# [breaker]
# enabled = false
# failure_threshold = 5   # consecutive failures that trip a target open; 0 disables
# open_secs = 30          # how long a tripped target stays open before a probe

# --- upstream engine metrics scrape ---
# when enabled, a background task periodically pulls each provider's prometheus
# {api_base}{path} and parses the scheduler queue depth (vLLM
# num_requests_waiting, SGLang/TGI fallbacks). the depth is folded into the
# balancer's in-flight load view so load-aware strategies (power_of_two,
# pipeline) steer away from backed-up engines. disabled by default; a scrape
# failure or missing metric contributes zero depth. defaults shown below.
# [metrics_scrape]
# enabled = false
# interval_secs = 5    # seconds between scrape sweeps
# timeout_secs = 2     # per-scrape timeout
# path = "/metrics"    # appended to each provider's api_base when scraping

# --- exact-match response cache (ROL-56, streaming: ROL-235) ---
# when enabled (and a redis url is configured), a successful response is stored
# in redis keyed by the exact request (path + post-injection body); an identical
# later request is served from the cache with no upstream call and an
# `x-rolter-cache: HIT` header. both non-streaming JSON and streaming SSE
# responses are cached: a streaming miss is buffered in full, stored, and
# replayed instantly on a later hit. this is a global master switch — a route
# only caches when it ALSO opts in via its own [routes.cache] table below.
# with no redis, or redis unreachable, every request is a miss (fail open).
# disabled by default. defaults shown below.
# [cache]
# enabled = false
# default_ttl_secs = 60        # entry TTL; overridable per route
# namespace = "rolter:cache"   # redis key prefix for cache entries
# max_entry_bytes = 1048576    # skip caching bodies larger than this (0 = no limit)
#
# per-route opt-in (nested under a [[routes]] entry):
# [routes.cache]
# enabled = true       # cache this route (global [cache] switch must also be on)
# ttl_secs = 600       # optional TTL override; falls back to default_ttl_secs
# per_key = false      # true isolates cached entries per virtual key

[logging]
# request logs are written asynchronously in batches, off the request hot path;
# logging is disabled entirely when clickhouse_url is unset
# clickhouse_url = "http://localhost:8123"
# batch_max = 1000        # flush once a batch reaches this many rows
# flush_ms = 1000         # also flush a partial batch at least this often
# queue_capacity = 10000  # rows are dropped (and counted) when this queue is full