opensourcellmrouter 0.7.0

A fast, local-first LLM router — proxy any OpenAI/Anthropic/Ollama client to your own provider pipeline with classifiers, cost/latency/random routing rules, plugins, a live dashboard, and a TUI.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169

# opensourcellmrouter configuration
#
# Providers:
#   local-llama — llama.cpp server on :8080 (OpenAI-compatible)
#   ollama      — Ollama on :11434 (native Ollama API)
#   cloudflare  — Cloudflare Workers AI  (CLOUDFLARE_API_TOKEN)
#   openai      — OpenAI                 (OPENAI_API_KEY)
#   anthropic   — Anthropic              (ANTHROPIC_API_KEY)
#   xai         — xAI / Grok             (XAI_API_KEY)
#
# API keys live in .env — run `source .env` or let demo.sh load it.
#
# Pipeline: classifiers tag the request → routers pick a provider → logging

[server]
host      = "127.0.0.1"
port      = 8090
dashboard = true
# Set host = "0.0.0.0" to accept connections from other machines (e.g. for
# launch-coding-agent.sh's remote host:port use), and set api_key_env
# whenever you do — see docs/security.md.
# api_key_env = "ROUTER_API_KEY"

[logging]
enabled = true
path    = "logs/requests.jsonl"

# OpenTelemetry traces/metrics/logs over OTLP/HTTP — see docs/telemetry.md.
# Disabled by default; omit this section entirely to leave it off.
# [telemetry]
# enabled       = true
# otlp_endpoint = "http://localhost:4318"
# service_name  = "opensourcellmrouter"
# sample_ratio  = 1.0

# ── providers ─────────────────────────────────────────────────────────────────

[[providers]]
name                      = "local-llama"
format                    = "openai"
base_url                  = "http://localhost:8080/v1"
cost_per_1m_tokens        = 0.0
quality                   = 60
latency_ms                = 900
throughput_tokens_per_sec = 20

# Ollama native API — base_url has no /v1 suffix.
# The "discover" router rule queries /api/tags at startup to know which
# models are pulled (llama3.1:8b, deepseek-r1:latest, gemma3:latest, etc.).
[[providers]]
name                      = "ollama"
format                    = "ollama"
base_url                  = "http://localhost:11434"
cost_per_1m_tokens        = 0.0
quality                   = 75
latency_ms                = 600
throughput_tokens_per_sec = 30

# Cloudflare Workers AI — OpenAI-compatible endpoint.
# Requires CLOUDFLARE_API_TOKEN in the environment (source .env).
[[providers]]
name               = "cloudflare"
format             = "openai"
base_url           = "https://api.cloudflare.com/client/v4/accounts/e4847a910eddf0d11d40af276cc478b7/ai/v1"
api_key_env        = "CLOUDFLARE_API_TOKEN"
cost_per_1m_tokens = 0.2
quality            = 80
latency_ms         = 500
throughput_tokens_per_sec = 60

# OpenAI — requires OPENAI_API_KEY in environment.
[[providers]]
name               = "openai"
format             = "openai"
base_url           = "https://api.openai.com/v1"
api_key_env        = "OPENAI_API_KEY"
cost_per_1m_tokens = 5.0
quality            = 90
latency_ms         = 400
throughput_tokens_per_sec = 80

# Anthropic — requires ANTHROPIC_API_KEY in environment.
[[providers]]
name               = "anthropic"
format             = "anthropic"
base_url           = "https://api.anthropic.com"
api_key_env        = "ANTHROPIC_API_KEY"
cost_per_1m_tokens = 15.0
quality            = 95
latency_ms         = 600
throughput_tokens_per_sec = 70

# xAI (Grok) — OpenAI-compatible endpoint. Requires XAI_API_KEY in environment.
[[providers]]
name               = "xai"
format             = "openai"
base_url           = "https://api.x.ai/v1"
api_key_env        = "XAI_API_KEY"
cost_per_1m_tokens = 5.0
quality            = 88
latency_ms         = 500
throughput_tokens_per_sec = 70

# ── classifiers ───────────────────────────────────────────────────────────────

[classifiers.keyword]
enabled = true

[classifiers.keyword.tags]
# Matches any message mentioning images/photos → routed to a vision-capable model
vision = ["image", "photo", "picture", "screenshot", "visual", "diagram", "chart"]
# Matches video content references
video  = ["video", "clip", "footage", "frame", "timestamp"]
# Matches code-heavy requests → routed to deepseek-r1 (strong at reasoning/code)
code   = ["function", "class", "import", "def ", "fn ", "bug", "error", "stack trace",
          "compile", "runtime", "algorithm", "refactor", "debug"]
# Adult/explicit content → kept on local-llama (private, no content policy)
nsfw   = ["nsfw", "adult", "explicit", "erotic", "nude", "naked",
          "sexual", "xxx", "porn", "hentai", "fetish", "lewd"]

# ── response classifiers ────────────────────────────────────────────────────────
# Tag the response after the provider replies (e.g. detect refusals). Tags land
# in the X-Router-Tags response header and in logs/dashboard — never in the
# OpenAI/Anthropic response body.

[response_classifiers.refusal]
enabled = true

# ── routers (first match wins) ────────────────────────────────────────────────

# Pick a random model from everything we have — applies to every request.
# Providers whose API key env var is unset are skipped automatically at startup.
[[routers]]
type = "random"
candidates = [
  # local
  { provider = "local-llama", model = "llama3.2-3b"                                  },
  # ollama
  { provider = "ollama",      model = "llama3.1:8b"                                  },
  { provider = "ollama",      model = "deepseek-r1:latest"                           },
  { provider = "ollama",      model = "gemma3:latest"                                },
  # cloudflare workers ai
  { provider = "cloudflare",  model = "@cf/meta/llama-3.1-8b-instruct"               },
  { provider = "cloudflare",  model = "@cf/meta/llama-3.2-3b-instruct"               },
  { provider = "cloudflare",  model = "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" },
  { provider = "cloudflare",  model = "@cf/google/gemma-3-12b-it"                    },
  # openai
  { provider = "openai",      model = "gpt-4o-mini"                                  },
  { provider = "openai",      model = "gpt-4o"                                       },
  # anthropic
  { provider = "anthropic",   model = "claude-haiku-4-5-20251001"                    },
  { provider = "anthropic",   model = "claude-sonnet-4-6"                            },
  # xai
  { provider = "xai",         model = "grok-4"                                       },
]

# ── plugins ───────────────────────────────────────────────────────────────────

[plugins.response-healing]
enabled = true

[plugins.pareto-router]
enabled      = false
default_tier = "medium"

[plugins.pareto-router.tiers]
low    = ["local-llama"]
medium = ["ollama"]
high   = ["ollama"]