rusty_agent_server/lib.rs
1//! # rusty-agent-server
2//!
3//! The network face of [`rusty_agent_runtime`]: an axum-based HTTP + SSE server
4//! implementing a pragmatic Agent-Protocol subset (see
5//! `docs/rusty-server-design.md`). The server ships as a **library** —
6//! users build their graphs, register them in a [`GraphRegistry`], and call
7//! [`serve`] (or [`router`] to embed the routes in a larger application):
8//!
9//! ```no_run
10//! use rusty_agent_runtime::prelude::*;
11//! use rusty_agent_server::{serve, GraphRegistry, ServerConfig};
12//!
13//! # async fn demo(graph: Graph, spec: StateSpec) -> std::io::Result<()> {
14//! let mut registry = GraphRegistry::new();
15//! registry.register("my_agent", graph, spec);
16//!
17//! let config = ServerConfig::new(
18//! "127.0.0.1:8080".parse().unwrap(),
19//! "./data/checkpoints",
20//! );
21//! serve(registry, config).await
22//! # }
23//! ```
24//!
25//! ## Endpoint inventory (v0.5)
26//!
27//! | Endpoint | Purpose |
28//! |---|---|
29//! | `GET /ok` | liveness |
30//! | `GET /info` | service version + registered graphs and their channels |
31//! | `POST /threads` | create a thread bound to a registered graph |
32//! | `POST /threads/{id}/fork` | time travel: copy the thread's checkpoint history (full or up to `checkpoint_id`) into a new thread |
33//! | `GET /threads/{id}/state` | latest checkpoint as `{values, next, checkpoint}` (the checkpoint ref carries its pinned `policy_version`, R0.8 wave 4) |
34//! | `POST /threads/{id}/state` | write a new checkpoint (`update_state` analog; `as_node` is accepted for LangGraph compatibility but not recorded; optional `enqueue: [...]` submits tasks atomically with the checkpoint through the R0.6 wave-2b transactional outbox — one transaction on Postgres, outbox-first best-effort ordering on the JSON-file backend) |
35//! | `POST /threads/{id}/history` | checkpoint list, newest first, `limit`/`before` |
36//! | `POST /threads/{id}/runs` | background run: `202 + run_id` |
37//! | `POST /threads/{id}/runs/wait` | blocking run: terminal result as JSON |
38//! | `POST /threads/{id}/runs/stream` | run with SSE streaming (`updates`/`values`/`messages`/`metadata`/`error`/`end`); a fresh run starts a new frame sequence, so `Last-Event-ID` is ignored here |
39//! | `GET /runs/{id}/stream` | attach to an existing run's SSE stream: replay honoring `Last-Event-ID`, then live frames |
40//! | `GET /runs/{id}/events` | Flight Recorder: the run's journaled `RunEvent`s as `{run_id, events, complete}` (snapshot flushed per checkpoint boundary and at run completion; persisted under `{store_path}/journals/` or the `server_journals` table; fetchable by run id even after the live run record is evicted or the process restarts) |
41//! | `GET /runs/{id}/fixture` | Flight Recorder: download the run as a portable `ReplayFixture` bundle (journal + graph topology hash + final checkpoint) for CI replay |
42//! | `POST /runs/replay` | Flight Recorder: re-drive a journaled run against its registered graph and verify the replayed evidence → `{run_id, verified, expected_events, actual_events, first_divergence}` (`422` when the graph is not registered in this process or the journal carries recorded effect calls) |
43//! | `GET /runs/diff?base=&branch=` | Flight Recorder: structural diff of two runs' journals (core's `BranchDiff` shape: `first_divergent_seq`, `added`/`removed` events, per-step channel diffs, token/cost totals) |
44//! | `DELETE /threads/{id}/runs/{run_id}` | rollback: delete a finished run's checkpoints (JSON-file checkpointer only; `409` on Postgres) |
45//! | `GET /runs/{run_id}` | run status polling (plus `output`/`error`/`interrupt` once terminal) |
46//! | `POST /assistants` | create a named graph alias with config metadata |
47//! | `GET /assistants` / `GET /assistants/{id}` | list / fetch assistants |
48//! | `POST /assistants/{id}/archive` / `POST /assistants/{id}/restore` | reversibly retire or return an assistant using an expected-active-version guard; archived assistants retain their immutable lineage and history but reject new runs |
49//! | `GET /assistants/{id}/versions` | list the bounded immutable configuration lineage and active serving pointer |
50//! | `POST /assistants/{id}/versions` | stage an immutable child version against the exact active base (never activates it) |
51//! | `GET /assistants/{id}/versions/{version_id}` | fetch one exact immutable configuration version for review |
52//! | `POST /assistants/{id}/versions/{version_id}/activate` | atomically activate or roll back to a reviewed version using an expected-active guard |
53//! | `POST /crons` | schedule recurring runs (interval secs or 5-field cron expr) |
54//! | `GET /crons` / `DELETE /crons/{id}` | list / delete crons |
55//! | `POST /triggers` | create an event-driven trigger: name, target (assistant or thread), action (`start_run` / `resume_thread` / `send_message`), `{{event.*}}` input template, per-trigger webhook secret, optional debounce window |
56//! | `GET /triggers` / `GET /triggers/{id}` / `PATCH /triggers/{id}` / `DELETE /triggers/{id}` | trigger registry (tenant-scoped) |
57//! | `POST /triggers/{id}/webhook` | signed event ingress: HMAC-SHA256 over the raw body, `X-Rusty-Signature: sha256=<hex>`, constant-time compare; `401` unsigned/invalid. Not behind the API-key layer — the signature is the credential and resolves the owning tenant |
58//! | `GET /triggers/{id}/events` | the trigger's event log (payload hash, action, run id, status), newest first |
59//! | `GET /triggers/{id}/dead-letter` | failed events, inspectable and replayable |
60//! | `POST /triggers/{id}/events/{event_id}/replay` | re-execute a logged event immediately (the replay is itself logged) |
61//! | `PUT /store/{ns}/{key}` | upsert a JSON value in a namespace (`201` create, `200` replace) |
62//! | `GET /store/{ns}/{key}` / `DELETE /store/{ns}/{key}` | fetch / delete one item |
63//! | `GET /store/{ns}` | list a namespace's items |
64//! | `POST /tasks` | R0.6 durable task queue: enqueue a task `{kind, payload, pool?, max_attempts?, idempotency_key?, worker_version?}` → `201 {task_id, deduplicated}` (`200` + `deduplicated: true` when the idempotency key already names a live task in this tenant; `429 quota_exceeded` when the tenant is over its wave-3 task quota — see `ServerConfig::with_task_quota`) |
65//! | `POST /tasks/outbox` | R0.6 wave 2b: enqueue through the transactional outbox (same payload as `POST /tasks`, same quota gate) → `202 {task_id, deduplicated}`; the relay publishes the task into the queue within one poll interval, at-least-once, deduped on the idempotency key |
66//! | `POST /tasks/claim` | R0.6: claim the oldest claimable task `{worker_id, pools?, worker_version?, lease_ms}` → `200 {task}` with a fresh lease, `204` when nothing is claimable. Wave 3: pools at their configured concurrency limit (`ServerConfig::with_pool_limit`) hand out nothing, and a task pinned with `worker_version` is leased only to a worker advertising that exact version |
67//! | `GET /tasks/metrics` | R0.6 wave 3: the autoscaling signals, tenant-scoped — per-pool queue depth, live leases, lease saturation against the configured limit, and oldest-visible-task age. Metrics, not a mechanism: the autoscaler is the operator's |
68//! | `POST /tasks/{id}/heartbeat` | R0.6: extend the held lease `{worker_id, lease_ms}` → `{lease_expires_at}`; `409` when the lease is lost |
69//! | `POST /tasks/{id}/complete` | R0.6: settle the held lease `{worker_id, result, receipt?}` → updated task record; `409` when the lease is lost. The optional `receipt` (wave 2b: `{provider, provider_id, idempotency_key, task_id?}`) is the worker's report of an idempotent effect's provider confirmation; the server journals it into the task's run as an `effect_receipt` event |
70//! | `POST /tasks/{id}/fail` | R0.6: record a failed attempt `{worker_id, error_class, message, retryable}` → `{requeued, next_attempt_at, dead, escalation}` (backoff + jitter requeue, or dead-letter); `409` when the lease is lost. R0.7 wave 2: a failed agent turn (non-`cancelled`) also feeds the agent's supervision policy — `escalation` reports where the escalation notice landed when the restart intensity was exceeded |
71//! | `GET /tasks/{id}` | R0.6: the task record (404 unknown/cross-tenant) |
72//! | `GET /tasks?status=…` | R0.6: the tenant's tasks, oldest first; `status=dead` is the DLQ listing |
73//! | `POST /agents` | R0.7 Agent Fabric (wave 1): register an agent `{agent_id, manifest, metadata?}` — the manifest is a core `CapabilityManifest` (`agent_kind`, `manifest_version`, `accepts`, …) → `201`; `409` when the id is taken |
74//! | `GET /agents` / `GET /agents/{id}` | R0.7: list the tenant's agents / fetch one registration (404 unknown/cross-tenant) |
75//! | `POST /agents/{id}/mailbox` | R0.7: send a message into the agent's mailbox `{kind, payload, idempotency_key?, …}` → `201 {task_id, deduplicated}` (`200` when the key already names a live message); `400` when the manifest's `accepts` does not declare the kind; same quota gate as `POST /tasks` |
76//! | `GET /agents/{id}/status` | R0.7: the agent's activation lease plus mailbox gauges (`queued` / `in_flight` / `dead` message counts) |
77//! | `POST /agents/{id}/activate` | R0.7: claim the agent's single activation lease `{worker_id, lease_ms}` → `200 {owner, fencing, lease_expires_at}`; `409` + the current lease when another host holds it live (expired leases are stolen, fencing bumped) |
78//! | `POST /agents/{id}/activate/heartbeat` | R0.7: renew the held activation `{worker_id, fencing, lease_ms}`; `409` on fencing loss |
79//! | `POST /agents/{id}/activate/release` | R0.7: drop the held activation `{worker_id, fencing}` so a draining host's replacement activates promptly; `409` on fencing loss |
80//! | `POST /agents/{id}/mailbox/next` | R0.7: claim the oldest queued mailbox message as one turn `{worker_id, fencing, lease_ms}` → `200 {task}` leased to the worker, `204` when empty or a turn is already in flight (one message at a time per agent is server-enforced), `409` when the activation lease is not held. The turn settles through the ordinary `/tasks/{id}/heartbeat|complete|fail` protocol |
81//! | `GET /agents/{id}/supervision` | R0.7 wave 2: the agent's supervision evidence — declared policy, latches, full attempt history, and the journaled `SupervisionEvent` / `AgentExit` events of its supervision journal (integrity re-verified on read) |
82//! | `POST /agents/{id}/cancel` | R0.7 wave 2: cancel the agent's outstanding mailbox traffic — queued/retry-scheduled messages go terminal-`cancelled`, the leased turn keeps its lease with `cancel_requested` set — and journal an `AgentExit`; idempotent (`200` with empty lists when nothing is outstanding) |
83//! | `POST /agents/{id}/restart` | R0.7 wave 2: the operator's manual restart — records a `manual_restart` supervision attempt, clears the escalation/deadline latches, and journals the event so the agent's next turn runs fresh |
84//! | `POST /teams/{team_id}/cancel` | R0.7 wave 2: cancel every agent registered with the `team_id` label, member by member with the agent-cancel semantics; `404` when no tenant agent declares the team |
85//! | `POST /memory` | R0.8 Rusty Learn (wave 1): write a governed memory record `{kind, scope, content, author, key?, tags?, priority?, confidence?, written_at?, valid_from?, valid_until?, expires_at?, supersedes?, evidence?, run_id?, parent?}` → `201 {memory_id, created, record}` (`200` + `created: false` when the content address is already stored — writes are idempotent by construction). Gates: `run` scope is runtime-only (`400`); `agent` scope requires the agent registered with `StateScope::Private` declared (`404` / `403`); `tenant` scope id must be the caller's tenant (`403`); confidence defaults to `1.0` for human authors and is required otherwise (`400`). With `run_id`, the write is journaled into that run as a `memory_write` event (best-effort) |
86//! | `GET /memory/{memory_id}` | R0.8: fetch one record by content address (`404` unknown/cross-tenant); artifact-spilled bodies are re-inlined, so the served record is self-contained |
87//! | `POST /memory/query` | R0.8: structured retrieval — the `MemoryQuery` filters plus optional `budget`, `run_id`, `parent`. `as_of` resolves at read time. With `budget`, answers the deterministic token-bounded `MemoryAssembly` (`422` when a hard budget overflows); without, the rank-ordered records. With `run_id` (budget required), the read is journaled into that run as a `memory_read` event (best-effort) |
88//! | `POST /memory/corrections` | R0.8 (wave 2): submit a human correction `{correction_id, author, target, corrected, scope, rationale?, run_id?, parent?}` → `201 {correction_id, attribution, candidate, memory_id, created, record, superseded, example_id}`. Author attribution is mandatory (validated at deserialization). Run scope adopts directly; agent scope or wider yields an attributed candidate (`candidacy: pending`, queryable via `candidates_only`). A `run_event` target additionally yields an `example`-kind record — the journaled input plus the corrected behavior — journaled into the corrected run through the memory-write seam; a `memory` target inherits the target's key, and same-key correction writes auto-supersede the prior record |
89//! | `POST /memory/consolidate` | R0.8 (wave 2): enqueue a consolidation as a durable `memory_consolidation` task `{scope, memory_ids, distiller, key?, tags?, priority?, pool?, run_id?, parent?}` → `201 {task_id, deduplicated, kind}` (deduped on scope + sorted source set). The claiming worker owns the distillation semantics and writes the `summary` record through the governed write path with the sources in `evidence.source_memory_ids` (which supersedes them) and the task payload's `written_at`; the task settles through the unchanged lease protocol |
90//! | `POST /memory/conflicts` | R0.8 (wave 2): the conflict review listing `{scope?}` → `{conflicts: [{scope, key, memory_ids, overlap}]}` — live same-key records with overlapping validity and contradictory content. Flags only; nothing is resolved |
91//! | `POST /memory/forget` | R0.8 (wave 2): erase one record `{memory_id, reason, run_id?, parent?}` → `200 {forgotten, invalidated, tombstone}`. Real deletion; dependent summaries are invalidated (deleted) transitively; a metadata-only `memory_forget` tombstone is journaled into the named run (best-effort) |
92//! | `POST /memory/forget_scope` | R0.8 (wave 2): erase every record at a scope address `{scope, reason, run_id?, parent?}` → `200 {forgotten, invalidated, tombstones}` — one tombstone per forgotten record; idempotent (empty scope → `200` with empty lists); tenant scope is self-only (`403`) |
93//! | `POST /learn/candidates` | R0.8 (wave 3): register a distilled candidate `{candidate, run_id, parent?}` → `201 {candidate_id, created, record}` (`200` + `created: false` when the content address is already stored — creation converges). The address is verified (`422`); the `candidate_created` event is journaled into `run_id`'s journal before the store write — hard-fail: an unresolvable run stops the transition (`404`) |
94//! | `GET /learn/candidates` / `GET /learn/candidates/{id}` | R0.8 (wave 3): the tenant's candidates (sorted by id) / fetch one record (`404` unknown/cross-tenant) |
95//! | `POST /learn/candidates/{id}/evaluate` | R0.8 (wave 3): drive the configured `CandidateEvaluator` `{request, run_id, parent?}` → `200 {candidate_id, status, evaluation}`; `409` when no evaluator is configured or the lifecycle forbids re-evaluation; `422` when the evaluation fails or violates the seam contract (it must name this candidate and the request's dataset version) |
96//! | `POST /learn/candidates/{id}/promote` | R0.8 (wave 3): run the promotion gate `{run_id, approval?, tag?, parent?}` → `200 {candidate_id, status, receipt, pointer}`. `403` on approval failures (out-of-envelope promotion needs an `ApprovalToken` scoped to the candidate's promotion effect id — non-transferable), `422` on evidence failures, `409` when the candidate is not `evaluated`. The status flip and the version-pointer move are one store transition. R0.11 (wave 1): the optional `tag` moves the pointer on the environment-tagged surface (`prompt:system@prod`) through the unchanged machinery; absent is the untagged, pre-R0.11 behavior |
97//! | `POST /learn/candidates/{id}/rollback` | R0.8 (wave 3): re-point the surface to the displaced version `{run_id, cause, tag?, parent?}` → `200 {candidate_id, status, receipt, pointer}`; `409` when the candidate is not `promoted` or the pointer no longer serves it. Byte-exact: the pointer's `to` is the promotion's recorded `previous`, and candidates are content-addressed — the restored version is the version that served. R0.11 (wave 1): `tag` rolls back the tagged surface's pointer |
98//! | `GET /learn/versions` | R0.8 (wave 3): the tenant's version pointers, sorted by surface |
99//! | `POST /registry/artifacts` | R0.11 (wave 1): declare a registry artifact `{family, name, owner}` → `201 {surface, created, artifact}` (`200` converged on an identical re-declaration; `409` when the surface is taken under a different family or owner; `422` on a name outside the naming rules). The artifact is an index over the candidate pipeline — a commit is a candidate, never a fork |
100//! | `GET /registry/artifacts?family=` / `GET /registry/artifacts/{family}/{name}` | R0.11 (wave 1): the tenant's artifacts (optionally one family's, sorted by surface) / fetch one (`404` unknown/cross-tenant) |
101//! | `POST /registry/artifacts/{family}/{name}/commits` | R0.11 (wave 1): commit a candidate `{candidate_id, committed_at?}` → `200 {surface, committed, commit, commits}` (`200` + `committed: false` when already committed; `404` unknown artifact/candidate; `422` family or surface mismatch; `409` concurrent commit) |
102//! | `GET /registry/artifacts/{family}/{name}/commits` | R0.11 (wave 1): the history walk — commits oldest first, joined with each candidate's status and author |
103//! | `GET /registry/artifacts/{family}/{name}/diff?from=&to=` | R0.11 (wave 1): the diff view between two committed versions, computed on read (never stored) — line diff for prompts, structural canonical-JSON diff for JSON families; `422` when either candidate is not committed to this artifact |
104//! | `POST /policy/versions` | R0.8 (wave 4): register an immutable executor policy body `{version?, policy}` → `201 {version, record}` (`200` converged when the version already names exactly this body; `409` when it names a different one — registry immutability; `400` invalid version). Without `version`, the content-derived `policy-{hash12}` name is minted. The reserved `static-v0` floor is never registerable |
105//! | `GET /policy/versions` / `GET /policy/versions/{version}` | R0.8 (wave 4): the tenant's registered policy bodies (sorted by version) / fetch one (`404` unknown/cross-tenant; the floor resolves as a synthetic record) |
106//! | `POST /policy/activations` | R0.8 (wave 4): move the active-version pointer `{version}` → `200 {version, active}`; `422` when the version is not registered (the floor is always activatable — reverting to pre-learning behavior needs no candidate) |
107//! | `GET /policy/active` | R0.8 (wave 4): the tenant's active policy — the last activation's body, or the floor when the registry never moved |
108//! | `GET /policy/epochs` | R0.8 (wave 4): the epoch history — each activation's reign window plus the admission bindings recorded inside it (and the implicit floor epoch covering pre-activation bindings) |
109//! | `GET /policy/drift` | R0.10 (wave 4): the drift check — the named (default active) candidate-derived version's journaled production decisions measured against the twin baseline it was promoted on; `422` for the floor and API-registered bodies (no promotion baseline) |
110//! | `POST /capsules` | R0.9 Rusty Capsules (wave 1): register an immutable capsule manifest `{manifest}` → `201 {capsule_id, record}` (`200` converged when the content address already names exactly this manifest; `409` when the `(name, version)` pin is claimed by a different address — registry immutability; `422` when the manifest fails validation). The manifest's content address is derived; callers never mint ids |
111//! | `GET /capsules` / `GET /capsules/{id}` | R0.9 (wave 1): the tenant's registered capsule manifests (sorted by content address) / fetch one (`404` unknown/cross-tenant) |
112//! | `POST /capsules/resolve` | R0.9 (wave 1): resolve a run's capsule pins `{pins: {name: version}, run_id, parent?}` → `200 {resolutions}` — each pin's stored manifest re-derives its content address before answering (a tampered record fails closed, `422`; an unknown pin is `404`), and one `capsule_resolved` event per pin is journaled into `run_id`'s journal (hard-fail: an unresolvable run is `404`). R0.9 wave 2 adds the optional `budget` field and, with the `capsules` feature, the admission composition: Cedar decides the admission and each declared grant (`403` on refusal, one journaled `capsule_denied` per forbidden grant), overlays intersect the effective grants, and budgets compose (`422` on a token/cost overspend) |
113//! | `POST /capsule_policies/versions` / `GET /capsule_policies/versions` / `GET /capsule_policies/versions/{version}` | R0.9 (wave 2, feature `capsules`): register an immutable Cedar policy body `{policy_text, version?}` → `201 {version, record}` (`200` converged; `409` when the version names a different body; `422` unparseable) / list the tenant's bodies (sorted by version) / fetch one (`404` unknown/cross-tenant). Without the feature: `503 capsule_policy_unavailable` |
114//! | `GET /capsule_policies/active` / `POST /capsule_policies/active` | R0.9 (wave 2, feature `capsules`): the tenant's active policy body (`404` in the unconfigured posture) / move the active-version pointer `{version}` → `200 {active}` (`422` unregistered version); the move eagerly refreshes the revocation cache. Without the feature: `503 capsule_policy_unavailable` |
115//! | `POST /capsules/overlays` / `GET /capsules/overlays` / `GET /capsules/overlays/{name}` | R0.9 (wave 2, feature `capsules`): attach (or replace) a tenant overlay `{overlay}` → `201` new / `200` replaced (`403` when the active policy refuses a widening attach; `422` invalid) / list the tenant's overlays (sorted by name) / fetch one (`404` unknown/cross-tenant). Without the feature: `503 capsule_policy_unavailable` |
116//! | `GET /runs/{id}/receipt` | R0.9 (wave 3): the run's signed `RunReceipt` — minted on first request over the run's reverified journal (manifest and executor policy read back from its last checkpoint header), then stored and served while the journal's head stands; a run whose journal advanced gets a fresh receipt. `409` when nothing is persisted yet (queued or pre-checkpoint); `404` unknown/cross-tenant |
117//! | `POST /receipts/verify` | R0.9 (wave 3): verify caller-supplied evidence `{snapshot, receipt, key_id?}` → `200` with the typed `VerifiedRun` summary, or `422 receipt_verification_failed` naming the mismatched component (`journal_head` / `manifest_digest` / `effect_ledger` / `denials_ledger` / `capsule_resolutions` / `capsule_policies` / `signer_key_id` / `signature`). The public key resolves from the deployment's key history by `key_id` (default: the receipt's `signer`); an unknown key id is `404` |
118//! | `GET /receipt_keys` | R0.9 (wave 3): the deployment's signing-key history (public keys, registration and retirement instants) plus the active key id — what an auditor needs to verify receipts offline |
119//! | `POST /receipt_keys/rotate` | R0.9 (wave 3): rotate the signing key → `201 {previous_key_id, key_id, public_key, event_id}`; the new key id is journaled as a `signing_key_rotated` event in the deployment's receipts journal, and receipts signed by the retired key keep verifying against the history |
120//! | `GET /receipt_keys/journal` | R0.9 (wave 3): the deployment's key-lineage journal — the chained `signing_key_rotated` events, integrity re-verified on read |
121//! | `POST /mcp` | R0.9 (wave 4): the MCP bridge (pinned revision `2025-03-26`) — every registered graph as one tool. `initialize` / `ping` / `notifications/*` / `tools/list` (tool schemas derived from each graph's state spec: append channels are arrays, deep-merge channels objects) / `tools/call` (runs the graph on a fresh thread; plain-JSON answer, or SSE `notifications/progress` + the final response when the request accepts `text/event-stream`; a mid-stream disconnect cancels the run). JSON-RPC errors in the envelope with HTTP 200; notifications answer `202` |
122//! | `GET /.well-known/agent-card.json` | R0.9 (wave 4): the A2A agent card (pinned spec `0.3.0`), derived from the registry on every read — one skill per registered graph; deterministic, no timestamps |
123//! | `POST /a2a` | R0.9 (wave 4): the A2A JSON-RPC task surface — `message/send` (enqueue a durable task, `kind = "a2a"`, idempotent on `messageId`; a capsule data part `{"capsule": {name, version}, "input"}` routes to the in-process executor over the `a2a-capsule` pool, plain messages queue on `a2a` for external workers) / `message/stream` (the same plus SSE status events) / `tasks/get` / `tasks/cancel`. The context id maps to one Flight Recorder journal (`a2a-{tenant}-{contextId}`), so capsule executions leave their evidence on the native `/runs/{id}/events` and `/fixture` endpoints |
124//! | `PUT /capsules/{id}/blob` | R0.9 (wave 4): upload the component bytes a registered manifest's `build_digest` commits to (raw body) → `201 {capsule_id, sha256, bytes}`; `404` unknown capsule, `422` digest mismatch, `409` different bytes under a taken address (registry immutability over bytes) |
125//! | `POST /connections` | R0.11 (wave 3): register a connection `{provider, subject?, scopes, token}` → `201 {connection}`; the token material is envelope-encrypted before it touches the store and the registration journaled — the bytes never travel beyond the broker. Wave 4: `{provider, subject?, scopes, authorization_code}` exchanges the code through the deployment's OAuth provider (`ServerConfig::with_oauth_provider`; `409` when none is configured) and seals the grant — the two credential paths are mutually exclusive (`422`) |
126//! | `GET /connections` | R0.11 (wave 3): the tenant's connection records (metadata only — sealed material never leaves the broker), sorted by id |
127//! | `GET /connections/health` | R0.11 (wave 4): the tenant-wide health surface — every connection with its status and health counters, sorted by id |
128//! | `GET /connections/{id}` | R0.11 (wave 3): one connection record → `200 {connection}`; `404` unknown/cross-tenant |
129//! | `POST /connections/{id}/consent` | R0.11 (wave 3): record a consent act `{scopes?, token?}` (one required, `422` otherwise) → `200 {connection, journaled}`; a scope-set change journals `connection_consented`, material-only `connection_refreshed`; re-recording the same fact converges without a second event. Re-activates a `needs_reauth` connection — this is the re-auth path. Wave 4: `authorization_code` is a third, mutually exclusive material path — exchanged through the OAuth provider and sealed |
130//! | `POST /connections/{id}/revoke` | R0.11 (wave 3): revoke `{reason?}` → `200 {connection, event_id?}` (re-revocation converges with no `event_id`); outstanding handles fail at their next use with a typed, journaled `connection_revoked` denial |
131//! | `DELETE /connections/{id}` | R0.11 (wave 3): revoke-then-erase → `200 {deleted: true}`; the sealed material is really deleted, and resolution fails closed `unknown_connection` thereafter |
132//! | `GET /connections/{id}/health` | R0.11 (wave 3): the connection's status and health counters (last failure class, consecutive failures, last refresh) |
133//! | `GET /broker/journal` | R0.11 (wave 3): the deployment's broker evidence chain — registrations, consents, refreshes, revocations, issuances, uses, denials, and (wave 4) `connection_needs_reauth` terminal-refusal transitions, integrity re-verified on read (the `receipt_keys/journal` precedent for a second control plane) |
134//! | `POST /deployments/revisions` | R0.12 (wave 3): register an immutable, content-addressed revision `{graph, assistant?, source_environment, surfaces, author}` → `201 {created, revision}` (`200` + `created: false` converged on an identical re-registration — the content address makes it the same declaration; `404` unknown graph/assistant or undeclared source environment; `422` a surface with no active version there to freeze, or a declaration outside the contract). The pins freeze the source environment's ACTIVE pointers; the server computes the graph's current topology hash at registration. The `revision_registered` act journals before the store write (hard-fail) |
135//! | `GET /deployments/revisions` / `GET /deployments/revisions/{id}` | R0.12 (wave 3): the tenant's revisions (sorted by id) / fetch one (`404` unknown/cross-tenant) |
136//! | `POST /deployments/environments` | R0.12 (wave 3): declare an environment `{name, gate?, approval_required, author}` → `201 {created, environment}` (`200` + `created: false` converged on an identical re-declaration; `409` a different rule under the same name — declarations are immutable; `400` invalid name). Gate and approval declarations are enforced from wave 4 (below). The `environment_declared` act journals before the store write (hard-fail) |
137//! | `GET /deployments/environments` / `GET /deployments/environments/{name}` | R0.12 (wave 3): the tenant's environments (sorted by name) / fetch one (`404` unknown/cross-tenant) |
138//! | `POST /deployments/environments/{name}/promote` | R0.12 (wave 3): move the environment's deployment pointer to a registered revision `{revision_id, author, approval?}` → `201 {applied, journaled, event_id, pointer}` (`200 {applied: false}` on a converged re-issue — no journal noise; `404` undeclared environment or unknown revision; `422` a revision failing its content-address check; `409` a lost race after exactly one rebuild — re-read and retry, never a lost move). Journal-then-pointer under the chain lock; the store's CAS arbitrates the move. Wave 4: a declared gate runs first — the decision journaled, allowed or refused (`422` a blocking verdict, `409` an unavailable gate — the fail-closed read); an approval-required environment then demands a token scoped to this revision's promotion effect id (`403` otherwise — a token minted for any other revision admits nothing here) |
139//! | `POST /deployments/environments/{name}/rollback` | R0.12 (wave 3): re-point the environment byte-exactly at what served before `{author, cause}` → `201` with the moved pointer (`200` converged; `400` empty cause — a rollback names its cause; `404` undeclared environment; `409` nothing serves, or no journaled history to restore to). The target re-derives from the chain's transition replay — the immutable revision that served, never a reconstruction |
140//! | `GET /deployments/environments/{name}/pointer` | R0.12 (wave 3): the environment's serving picture (`404` undeclared environment, or nothing ever promoted into it) |
141//! | `PUT /deployments/environments/{name}/canary` | R0.12 (wave 4): bind a revision to the environment's canary slot `{revision_id, fraction, author}` → `201` with the moved pointer (`200 {applied: false}` on an identical re-declaration — the binding already holds; `404` undeclared environment or unknown revision; `422` a fraction outside (0, 1] — never a clamp). A declared gate runs first, exactly as on promote — the gate protects the environment, not the pointer slot. The active keeps serving; bound runs split by the seeded draw, journaled on each run's own resolution |
142//! | `DELETE /deployments/environments/{name}/canary` | R0.12 (wave 4): clear the canary slot `{author}` → `201` with the moved pointer (`200 {applied: false}` when the slot is already empty — an empty slot is the state, not an error). The `canary_cleared` act names the binding it ended |
143//! | `GET /deployments/journal` | R0.12 (wave 3): the deployment evidence chain — revision registrations, environment declarations, promotions, rollbacks, secret acts and denials — integrity re-verified on read (`422` when it does not verify; the `receipt_keys/journal` and `broker/journal` precedent for a control plane) |
144//! | `PUT /deployments/secrets` | R0.12 (wave 3): set or rotate an environment secret `{name, environment, value, author}` → `201 {created, record}` (`200` + `created: false` on rotation — `rotated_at` marks it; `404` undeclared environment; `422` a name outside the naming rules). The value seals (XChaCha20-Poly1305 envelope under the deployment's `esk-` master key) before anything writes; the store only ever holds the envelope, the `env_secret_set` act journals before the write, and the response is metadata — the value never comes back through this route |
145//! | `GET /deployments/secrets?environment=` | R0.12 (wave 3): the tenant's secret metadata (never envelopes — a listing is an audit view), optionally one environment's, sorted by (environment, name) |
146//! | `POST /deployments/secrets/resolve` | R0.12 (wave 3): open a secret at use, inside its declared scope `{name, environment, holder}` → `200 {name, environment, value}`; `403 environment_scope_denied` — typed AND journaled — when the holder is not the scope; `404` unknown secret; `500` when this host does not hold the sealing key. Every failure fails closed |
147//! | `DELETE /deployments/secrets/{environment}/{name}` | R0.12 (wave 3): revoke by deletion `{author}` → `204` (`404` unknown/cross-tenant); the `env_secret_revoked` tombstone journals before the delete — the tombstone is the evidence the scope once held a value |
148//! | `POST /deployments/shadows` | R0.12 (wave 4): run a recorded run's twin under a candidate revision, behind the shadow admission boundary `{revision_id, source, input?, author}` — `source` is the journal snapshot, inline, so shadow evidence is self-contained → `201 {shadow_run_id, verdict}`; `404` unknown revision; `422` a revision failing its content-address check, a source snapshot failing integrity, or a failed run (the verdict journals either way — a failed shadow is evidence about the candidate). Pure and read-only effects execute; everything above is refused (typed, classified, journaled) and served from the recorded world when the source journal holds the outcome; the verdict names matched, unserved, and unrequested outcomes. The shadow journals under its own run id and holds no thread — a receipt request for it is a `404` by construction |
149//! | `GET /deployments/health` | R0.12 (wave 4): the health board — every environment's pointer, canary state with its bound-run tally, and the last gate decision the chain recorded for it, derived from journaled data alone (no new store) |
150//!
151//! Runs support `command.resume` (HITL), `config.recursion_limit`, the
152//! `reject` / `enqueue` multitask strategies (one active run per thread),
153//! `assistant_id` (resolved to its bound graph, with the assistant's
154//! `config.recursion_limit` as a default), and `checkpoint.checkpoint_id`
155//! (time-travel replay from that checkpoint instead of the latest).
156//! Runs may also declare their configuration at submission: `registry`
157//! (R0.11 wave 2) binds named artifacts through their environment-tagged
158//! version pointers, and `deployment` (R0.12 wave 3) binds the revision
159//! an environment serves — each resolved at admission (a resolution
160//! failure stops the run before it starts) and journaled ahead of the
161//! run's own events, so the evidence names exactly what served.
162
163mod a2a;
164mod agents;
165mod artifacts;
166mod assistants;
167mod auth;
168mod broker;
169pub mod capsule_policy;
170mod capsules;
171mod coordination;
172mod crons;
173mod deploy;
174mod error;
175mod gate;
176mod journals;
177mod learn;
178mod mcp_bridge;
179mod memory;
180mod outbox;
181mod policy;
182mod receipts;
183mod registry;
184mod replay;
185mod routes;
186mod runs;
187mod server_store;
188mod sse;
189mod store;
190mod supervision;
191mod tasks;
192mod threads;
193mod triggers;
194
195use std::collections::HashMap;
196use std::net::SocketAddr;
197use std::path::PathBuf;
198use std::sync::Arc;
199
200use axum::Router;
201use rusty_agent_runtime::capsule::ResourceBudget;
202use rusty_agent_runtime::deploy::RevisionGateEvaluator;
203use rusty_agent_runtime::graph::Graph;
204use rusty_agent_runtime::learn::{CandidateEvaluator, EnvironmentTag, PromotionEnvelope};
205use rusty_agent_runtime::state::StateSpec;
206use tokio_util::sync::CancellationToken;
207
208pub use error::ApiError;
209pub use gate::{
210 DirectoryGatePolicySource, EvalRevisionGateEvaluator, GatePolicySource, RevisionEvaluationAgent,
211};
212pub use learn::{DatasetSource, DirectoryDatasetSource, EvalCandidateEvaluator, EvaluationAgent};
213pub use runs::RunStatus;
214
215/// Default bound on graceful shutdown (25 s): how long
216/// [`serve_with_shutdown`] lets in-flight requests, runs, and the outbox
217/// relay finish after the shutdown signal before the server stops anyway.
218///
219/// Chosen to fit under two outer backstops: Kubernetes' default 30 s
220/// pod-termination grace (so the drain finishes before SIGKILL would
221/// arrive) and the task queue's default 30 s lease (so a drained server has
222/// released its work well before lease expiry would reassign it). The
223/// durable machinery — checkpoint resume, lease expiry — remains the
224/// correctness net; the grace bound only decides how *fast* the common
225/// case is. Override with [`ServerConfig::with_shutdown_grace`].
226pub const DEFAULT_SHUTDOWN_GRACE: std::time::Duration = std::time::Duration::from_secs(25);
227
228/// Names the JSON-file layout already owns at the store root
229/// (`agent_leases/`, `agents/`, `artifacts/`, `assistants/`, `capsules/`,
230/// `capsule_policies/`, `coordinations/`, `crons/`, `deployments/`,
231/// `env-secrets/`, `journals/`, `learn/`,
232/// `memory/`, `memory_artifacts/`, `outbox/`, `policy/`, `registry/`,
233/// `store/`,
234/// `tasks/`, `threads/`, `trigger_events/`, `triggers/`, plus the
235/// `latest` pointer file inside each thread's checkpoint dir).
236/// Client-chosen ids and tenant ids claiming one of these would write
237/// checkpoints into platform directories (or platform records into
238/// checkpoint dirs), so both `validate_client_id` and
239/// [`ServerConfig::with_tenant_key`] reject them.
240pub(crate) const RESERVED_NAMES: &[&str] = &[
241 "agent_leases",
242 "agents",
243 "artifacts",
244 "assistants",
245 "capsules",
246 "capsule_policies",
247 "connections",
248 "coordinations",
249 "crons",
250 "deployments",
251 "env-secrets",
252 "journals",
253 "keys",
254 "learn",
255 "memory",
256 "memory_artifacts",
257 "outbox",
258 "policy",
259 "receipts",
260 "registry",
261 "store",
262 "tasks",
263 "threads",
264 "trigger_events",
265 "triggers",
266 "latest",
267];
268
269/// One registered graph: the compiled topology plus the state schema the
270/// executor needs to drive it.
271#[derive(Debug, Clone)]
272struct GraphEntry {
273 graph: Graph,
274 spec: StateSpec,
275}
276
277/// The set of graphs this server hosts — the Rust analog of the `graphs` map
278/// in LangGraph's `langgraph.json`. Registration is compile-checked in user
279/// code; a `GraphRegistry` is heterogeneous (each entry carries its own
280/// [`StateSpec`]), which is safe because `State` is a JSON map at runtime.
281#[derive(Debug, Default, Clone)]
282pub struct GraphRegistry {
283 entries: HashMap<String, GraphEntry>,
284}
285
286impl GraphRegistry {
287 pub fn new() -> Self {
288 Self::default()
289 }
290
291 /// Register a compiled graph under `name`, together with the state spec
292 /// the executor should merge its node updates through. Re-registering a
293 /// name replaces the previous entry.
294 pub fn register(
295 &mut self,
296 name: impl Into<String>,
297 graph: Graph,
298 spec: StateSpec,
299 ) -> &mut Self {
300 self.entries.insert(name.into(), GraphEntry { graph, spec });
301 self
302 }
303
304 /// `true` if a graph is registered under `name`.
305 pub fn contains(&self, name: &str) -> bool {
306 self.entries.contains_key(name)
307 }
308
309 /// All registered graph names, sorted for stable output.
310 pub fn names(&self) -> Vec<String> {
311 let mut names: Vec<String> = self.entries.keys().cloned().collect();
312 names.sort_unstable();
313 names
314 }
315
316 /// The declared channel names of a registered graph's spec, sorted.
317 pub fn channel_names(&self, name: &str) -> Vec<String> {
318 let mut channels: Vec<String> = self
319 .entries
320 .get(name)
321 .map(|entry| entry.spec.channel_names().map(str::to_owned).collect())
322 .unwrap_or_default();
323 channels.sort_unstable();
324 channels
325 }
326
327 pub(crate) fn get(&self, name: &str) -> Option<(Graph, StateSpec)> {
328 self.entries
329 .get(name)
330 .map(|entry| (entry.graph.clone(), entry.spec.clone()))
331 }
332}
333
334/// Tenant quota for the durable task queue (R0.6 wave 3): three gauges,
335/// each capped independently — tasks queued, tasks in flight, dead-letter
336/// depth — enforced at **submission** (`POST /tasks`, `POST /tasks/outbox`,
337/// and `update_state`'s atomic `enqueue` list). Over quota answers `429
338/// quota_exceeded`, the honest shape for "retry this submission later".
339///
340/// The gauges are the store's `TaskUsage` definitions: the backlog
341/// counts scheduled retries *and* outbox rows pending publication (a flood
342/// through the outbox must not bypass the quota), in flight counts every
343/// `leased` record, and the DLQ counts because an unbounded dead-letter
344/// queue is a quiet disk-full outage. `None` (the default for every field)
345/// means unlimited, preserving pre-wave-3 behavior.
346#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
347pub struct TaskQuota {
348 /// Cap on the tenant's backlog (queued + retry-scheduled + pending
349 /// outbox rows). A submission that would push the backlog past the cap
350 /// is rejected.
351 pub max_queued: Option<usize>,
352
353 /// Cap on tasks in flight (status `leased`). Pure backpressure: a
354 /// tenant already at the cap has its new submissions rejected until
355 /// workers settle.
356 pub max_in_flight: Option<usize>,
357
358 /// Cap on dead-letter depth. A tenant at the cap must inspect and
359 /// re-drive its DLQ before more work is accepted.
360 pub max_dlq: Option<usize>,
361}
362
363impl TaskQuota {
364 /// Every gauge capped: `TaskQuota { max_queued: Some(q),
365 /// max_in_flight: Some(f), max_dlq: Some(d) }`. Field-by-field
366 /// construction works too — the struct's fields are public and each
367 /// `None` means unlimited.
368 pub fn capped(max_queued: usize, max_in_flight: usize, max_dlq: usize) -> Self {
369 Self {
370 max_queued: Some(max_queued),
371 max_in_flight: Some(max_in_flight),
372 max_dlq: Some(max_dlq),
373 }
374 }
375
376 /// `true` when no gauge is capped — the quota gate short-circuits
377 /// without a store read (the default configuration's fast path).
378 pub(crate) fn is_unlimited(&self) -> bool {
379 self.max_queued.is_none() && self.max_in_flight.is_none() && self.max_dlq.is_none()
380 }
381}
382
383/// Server configuration.
384///
385/// Checkpointing is rooted at `store_path` via
386/// [`rusty_agent_runtime::checkpoint::JsonFileCheckpointer`]. Auth maps static API
387/// keys (checked against the `X-Api-Key` header) to tenants: the legacy
388/// single [`ServerConfig::with_api_key`] maps its key to the `default`
389/// tenant, while [`ServerConfig::with_tenant_key`] adds `(tenant, key)`
390/// pairs for multi-tenant deployments. With no keys configured the server
391/// runs in open (dev) mode — no header required, everything lives in the
392/// `default` tenant. Every tenant-scoped resource (threads + checkpoints,
393/// assistants, crons, KV namespaces) is isolated per tenant; cross-tenant
394/// access answers `404`. With the `postgres` feature,
395/// `ServerConfig::with_postgres` switches **both** the run checkpointer
396/// (core's `PostgresCheckpointer`) and the server store (assistants /
397/// crons / threads / KV) to Postgres.
398#[derive(Debug, Clone)]
399pub struct ServerConfig {
400 /// Address to bind when using [`serve`] (default `0.0.0.0:8080`).
401 pub bind_addr: SocketAddr,
402
403 /// Root directory for checkpoint files
404 /// (`{store_path}/{thread_id}/{checkpoint_id}.json`). Also roots the
405 /// JSON-file assistants/crons/threads/KV persistence. Unused for
406 /// checkpointing when `database_url` is set (still used as the
407 /// `store_path` reported by `GET /info`).
408 pub store_path: PathBuf,
409
410 /// Postgres connection URL. When set (requires the `postgres` feature —
411 /// see `ServerConfig::with_postgres`), checkpoints live in core's
412 /// `rusty_checkpoints` table and the platform surface in the
413 /// `server_assistants` / `server_crons` / `server_threads` /
414 /// `server_kv` / `server_journals` tables, all auto-migrated on
415 /// connect. Connections are established lazily on first use.
416 pub database_url: Option<String>,
417
418 /// Per-thread in-flight run cap used as the **enqueue queue depth**
419 /// (default 1). There is always at most one *active* run per thread.
420 pub max_concurrent_runs_per_thread: usize,
421
422 /// Static API key required via the `X-Api-Key` header, mapped to the
423 /// `default` tenant (legacy single-key mode). `None` (the default) with
424 /// an empty [`ServerConfig::api_keys`] is dev mode: no authentication.
425 pub api_key: Option<String>,
426
427 /// Additional `(api_key, tenant)` pairs for multi-tenant deployments
428 /// (see [`ServerConfig::with_tenant_key`]). Each key maps to exactly one
429 /// tenant; every tenant's threads, assistants, crons, and KV namespaces
430 /// are isolated from all others (cross-tenant access answers `404`).
431 pub api_keys: Vec<(String, String)>,
432
433 /// Per-run SSE event-log capacity (frames retained for replay, default
434 /// 1000).
435 pub event_log_capacity: usize,
436
437 /// How often the transactional-outbox relay publishes pending rows into
438 /// the task queue (R0.6 wave 2b, default 250 ms). The relay is
439 /// crash-safe at any interval — pending rows survive restarts and
440 /// publishing dedupes on the task's idempotency key — so this only
441 /// tunes outbox-to-queue latency, not correctness.
442 pub outbox_relay_interval: std::time::Duration,
443
444 /// Bound on graceful shutdown (R0.6 wave 2c, default
445 /// [`DEFAULT_SHUTDOWN_GRACE`]): after the shutdown signal, in-flight
446 /// requests and runs get this long to finish before the server stops
447 /// regardless. Runs stopped by the drain are resumable from their last
448 /// checkpoint, so a short grace costs a resume, never work.
449 pub shutdown_grace: std::time::Duration,
450
451 /// Per-pool in-flight caps for the durable task queue (R0.6 wave 3):
452 /// pool name → maximum live leases. The claim path counts a pool's
453 /// unexpired leases and stops handing out new ones at the cap, so
454 /// pools coexist without starving each other (a saturated GPU pool
455 /// never blocks IO-pool claims). Pools without an entry are
456 /// uncapped — the pre-wave-3 behavior. See
457 /// [`ServerConfig::with_pool_limit`].
458 pub task_pool_limits: HashMap<String, usize>,
459
460 /// The default tenant quota (R0.6 wave 3), applied to every tenant
461 /// without an entry in [`ServerConfig::tenant_task_quotas`]. All
462 /// gauges uncapped by default. See [`TaskQuota`].
463 pub task_quota: TaskQuota,
464
465 /// Per-tenant quota overrides (R0.6 wave 3), replacing
466 /// [`ServerConfig::task_quota`] wholesale for the named tenant (an
467 /// override *is* the tenant's quota, not a patch on the default).
468 /// See [`ServerConfig::with_tenant_quota`].
469 pub tenant_task_quotas: HashMap<String, TaskQuota>,
470
471 /// The promotion envelope (R0.8 Rusty Learn, wave 3): the declared,
472 /// per-deployment standing approval the promotion gate evaluates
473 /// every promotion against. Defaults to
474 /// [`PromotionEnvelope::r08_default`] — memory-set candidates at
475 /// run/agent scope auto-promote on cleared evidence; everything else
476 /// requires an approval token. See
477 /// [`ServerConfig::with_promotion_envelope`].
478 pub promotion_envelope: PromotionEnvelope,
479
480 /// The candidate evaluator (R0.8 wave 3): the evaluation
481 /// composition `POST /learn/candidates/{id}/evaluate` drives. `None`
482 /// (the default) answers `409` — a deployment without an evaluator
483 /// can hold and inspect candidates, but promotion is gated on
484 /// evidence, and evidence requires an evaluator. See
485 /// [`ServerConfig::with_candidate_evaluator`].
486 pub candidate_evaluator: Option<Arc<dyn CandidateEvaluator>>,
487
488 /// The revision gate evaluator (R0.12 wave 4): the release-gate
489 /// composition promotions into a gated environment drive. `None`
490 /// (the default) fails closed — a promotion into an environment
491 /// declaring a gate answers `409`, because a gate that cannot run
492 /// is a gate that did not pass. See
493 /// [`ServerConfig::with_revision_gate_evaluator`].
494 pub revision_gate_evaluator: Option<Arc<dyn RevisionGateEvaluator>>,
495
496 /// The deployment's default environment tag (R0.11 Extension Plane,
497 /// wave 2): the promotion target a run resolves against when its
498 /// registry binding declares no environment of its own. `None` (the
499 /// default) resolves the untagged surface — the pre-R0.11 behavior.
500 /// This is *declared configuration*: a run's environment is its
501 /// binding's tag or this default, never an invented per-run guess.
502 /// See [`ServerConfig::with_default_environment_tag`].
503 pub default_environment_tag: Option<EnvironmentTag>,
504
505 /// Operator-authored Cedar policy files (R0.9 Rusty Capsules, wave
506 /// 2), loaded for the `default` tenant at startup — the deployment's
507 /// standing authorization, the way static API keys are its standing
508 /// authentication. Requires the `capsules` feature; each file is
509 /// registered under its content-derived version
510 /// (`cedar-{sha256[..12]}`), never activated — activation is an
511 /// explicit operator move through `POST /capsule_policies/active`. See
512 /// [`ServerConfig::with_capsule_policy_file`].
513 pub capsule_policy_files: Vec<PathBuf>,
514
515 /// The tenant-wide budget ceiling (R0.9 wave 2): the outermost bound
516 /// every capsule budget composes under. Fuel, memory, wall time, and
517 /// output bytes clamp down to it; a run declaring more `max_tokens`
518 /// or `max_cost_usd` than the ceiling (or than the run's own budget)
519 /// is refused `422` — token and cost bounds cannot be retrofitted
520 /// mid-run the way fuel can, so admission refuses the overspend. See
521 /// [`ServerConfig::with_capsule_budget_ceiling`].
522 pub capsule_budget_ceiling: Option<ResourceBudget>,
523
524 /// The deployment's network-egress seam for capsules the A2A bridge
525 /// executes in-process (R0.9 wave 4): the [`NetworkConnector`] every
526 /// bridge-built capsule host is constructed with. Without one, A2A
527 /// tasks carrying a capsule payload fail closed with
528 /// `capsule_execution_unavailable` — the bridge never executes guest
529 /// code against an egress path the operator did not explicitly wire.
530 /// Requires the `capsules` feature. See
531 /// [`ServerConfig::with_capsule_connector`].
532 #[cfg(feature = "capsules")]
533 pub capsule_connector: Option<Arc<dyn rusty_agent_runtime::capsule_host::NetworkConnector>>,
534
535 /// The credential handle TTL (R0.11 Extension Plane, wave 3): how
536 /// long one issued handle stays valid, default 300 seconds — "handles
537 /// live for minutes": short enough that expiry is routine, long
538 /// enough that a run's burst of provider calls reuses one issuance.
539 /// Revocation never waits on this — resolution reads the connection's
540 /// live state, so a revoked connection fails at the next call
541 /// regardless. See [`ServerConfig::with_broker_handle_ttl`].
542 pub broker_handle_ttl: std::time::Duration,
543
544 /// The OAuth provider (R0.11 Extension Plane, wave 4): the
545 /// deployment's token-endpoint seam. Plugging one in enables the
546 /// whole lifecycle — authorization-code exchange on
547 /// `POST /connections` and `POST /connections/{id}/consent`,
548 /// refresh-at-resolution for due tokens, and the sweeper. `None`
549 /// (the default) means no lifecycle: the exchange endpoints answer
550 /// `409` (the evaluator-absent precedent) and OAuth connections
551 /// serve their sealed material unchanged, exactly as static kinds
552 /// do. See [`ServerConfig::with_oauth_provider`].
553 pub oauth_provider: Option<Arc<dyn rusty_agent_runtime::broker::OAuthProvider>>,
554
555 /// The OAuth refresh window (R0.11 wave 4), default 300 seconds: a
556 /// token expiring within this horizon — scaled per connection by a
557 /// deterministic jitter in [0.5, 1.0), so a fleet registered
558 /// together does not stampede the provider together — is rotated
559 /// before it is served. See
560 /// [`ServerConfig::with_broker_refresh_window`].
561 pub broker_refresh_window: std::time::Duration,
562
563 /// The broker sweep interval (R0.11 wave 4), default `None` (off):
564 /// how often the background sweeper runs the refresh lifecycle over
565 /// every connection. Off is safe — resolution runs the same
566 /// lifecycle, so no sweeper degrades to refresh-at-use, never to
567 /// expiry; the sweeper exists so idle connections are fresh before
568 /// the next call asks. See [`ServerConfig::with_broker_sweep_interval`].
569 pub broker_sweep_interval: Option<std::time::Duration>,
570
571 /// The artifact retention sweep interval (R0.12 wave 2), default
572 /// `None` (off): how often the background sweeper evaluates every
573 /// artifact address in the deployment against its declared
574 /// retention. Off is safe — `POST /artifacts/sweep` and the release
575 /// act's prune tail run the same evaluation, so no sweeper degrades
576 /// to operator-triggered passes, never to unprotected pruning. See
577 /// [`ServerConfig::with_artifact_sweep_interval`].
578 pub artifact_sweep_interval: Option<std::time::Duration>,
579}
580
581impl Default for ServerConfig {
582 fn default() -> Self {
583 Self {
584 bind_addr: SocketAddr::from(([0, 0, 0, 0], 8080)),
585 store_path: PathBuf::from("./data/checkpoints"),
586 database_url: None,
587 max_concurrent_runs_per_thread: 1,
588 api_key: None,
589 api_keys: Vec::new(),
590 event_log_capacity: 1000,
591 outbox_relay_interval: crate::outbox::DEFAULT_RELAY_INTERVAL,
592 shutdown_grace: DEFAULT_SHUTDOWN_GRACE,
593 task_pool_limits: HashMap::new(),
594 task_quota: TaskQuota::default(),
595 tenant_task_quotas: HashMap::new(),
596 promotion_envelope: PromotionEnvelope::r08_default(),
597 candidate_evaluator: None,
598 revision_gate_evaluator: None,
599 default_environment_tag: None,
600 capsule_policy_files: Vec::new(),
601 capsule_budget_ceiling: None,
602 #[cfg(feature = "capsules")]
603 capsule_connector: None,
604 broker_handle_ttl: broker::DEFAULT_HANDLE_TTL,
605 oauth_provider: None,
606 broker_refresh_window: broker::DEFAULT_REFRESH_WINDOW,
607 broker_sweep_interval: None,
608 artifact_sweep_interval: None,
609 }
610 }
611}
612
613impl ServerConfig {
614 /// A config with the given bind address and checkpoint store root;
615 /// everything else at its default.
616 pub fn new(bind_addr: SocketAddr, store_path: impl Into<PathBuf>) -> Self {
617 Self {
618 bind_addr,
619 store_path: store_path.into(),
620 ..Self::default()
621 }
622 }
623
624 /// Builder-style: require an API key on every request. The key maps to
625 /// the `default` tenant (legacy single-tenant mode).
626 pub fn with_api_key(mut self, key: impl Into<String>) -> Self {
627 self.api_key = Some(key.into());
628 self
629 }
630
631 /// Builder-style: map an API key to a tenant (multi-tenant mode). Every
632 /// request presenting `key` via `X-Api-Key` runs as `tenant`, fully
633 /// isolated from all other tenants. Tenant ids must match
634 /// `[A-Za-z0-9._-]` (1–64 chars) and must not be a reserved layout name
635 /// (`assistants`, `crons`, `store`, `threads`, `latest`) — they become a
636 /// path segment in the JSON-file layout and a `{tenant}/` id prefix
637 /// everywhere else.
638 ///
639 /// # Panics
640 ///
641 /// Panics on an empty key or an invalid tenant id (configuration is a
642 /// programmer error, caught at startup).
643 pub fn with_tenant_key(mut self, tenant: impl Into<String>, key: impl Into<String>) -> Self {
644 let tenant = tenant.into();
645 let key = key.into();
646 let valid = !tenant.is_empty()
647 && tenant.len() <= 64
648 && tenant
649 .chars()
650 .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.'))
651 && !RESERVED_NAMES.contains(&tenant.as_str());
652 assert!(
653 valid,
654 "invalid tenant id `{tenant}` (allowed: [A-Za-z0-9._-], 1..=64 chars, not a reserved name)"
655 );
656 assert!(
657 !key.is_empty(),
658 "API key for tenant `{tenant}` must not be empty"
659 );
660 self.api_keys.push((key, tenant));
661 self
662 }
663
664 /// `true` when at least one API key is configured (legacy `api_key` or
665 /// any tenant key), i.e. requests must authenticate.
666 pub fn auth_enabled(&self) -> bool {
667 self.api_key.is_some() || !self.api_keys.is_empty()
668 }
669
670 /// The tenant a presented API key maps to, or `None` for unknown keys.
671 /// Tenant keys are checked first (last registration wins on duplicate
672 /// keys); the legacy `api_key` maps to the `default` tenant.
673 pub fn tenant_for_key(&self, key: &str) -> Option<&str> {
674 if let Some((_, tenant)) = self.api_keys.iter().rev().find(|(k, _)| k == key) {
675 return Some(tenant.as_str());
676 }
677 if self.api_key.as_deref() == Some(key) {
678 return Some("default");
679 }
680 None
681 }
682
683 /// Builder-style: persist everything in Postgres at `url` (e.g.
684 /// `postgres://user:pass@localhost/rusty`). Switches the run
685 /// checkpointer to [`rusty_agent_runtime::checkpoint_postgres::PostgresCheckpointer`]
686 /// **and** the assistants/crons/threads/KV/journals server store to the
687 /// `server_*` tables. Schemas auto-migrate on (lazy) connect.
688 #[cfg(feature = "postgres")]
689 pub fn with_postgres(mut self, url: impl Into<String>) -> Self {
690 self.database_url = Some(url.into());
691 self
692 }
693
694 /// Builder-style: set the per-thread enqueue queue depth cap. Values
695 /// below 1 are clamped to 1 (a zero-deep queue would reject every
696 /// `enqueue` run).
697 pub fn with_max_concurrent_runs_per_thread(mut self, cap: usize) -> Self {
698 self.max_concurrent_runs_per_thread = cap;
699 self
700 }
701
702 /// Builder-style: set the per-run SSE event-log capacity. Values below
703 /// 16 are clamped to 16 (replay needs room for at least the
704 /// metadata/updates/end frames of a minimal run).
705 pub fn with_event_log_capacity(mut self, capacity: usize) -> Self {
706 self.event_log_capacity = capacity;
707 self
708 }
709
710 /// Builder-style: set the transactional-outbox relay's poll interval
711 /// (R0.6 wave 2b). Tests can set a long interval to drive publishing
712 /// deterministically; correctness never depends on the interval.
713 pub fn with_outbox_relay_interval(mut self, interval: std::time::Duration) -> Self {
714 self.outbox_relay_interval = interval;
715 self
716 }
717
718 /// Builder-style: set the graceful-shutdown bound (R0.6 wave 2c; see
719 /// [`DEFAULT_SHUTDOWN_GRACE`] for the default's rationale). Past this
720 /// window the server stops even if requests or runs are still in flight
721 /// — durability (checkpoint resume, lease expiry) is the backstop.
722 pub fn with_shutdown_grace(mut self, grace: std::time::Duration) -> Self {
723 self.shutdown_grace = grace;
724 self
725 }
726
727 /// Builder-style: cap the named pool's live leases (R0.6 wave 3). The
728 /// claim path counts a pool's unexpired leases and stops handing out
729 /// new ones at `max_in_flight`, so a saturated pool (GPU-bound work)
730 /// never starves the others (IO-bound work). A cap of `0` pauses the
731 /// pool: nothing is leased from it until the cap is raised. Pools
732 /// without an entry stay uncapped.
733 ///
734 /// # Panics
735 ///
736 /// Panics on an invalid pool name (configuration is a programmer error,
737 /// caught at startup); names must match `[A-Za-z0-9._-]` (1–128 chars),
738 /// the same rule the enqueue and claim paths validate.
739 pub fn with_pool_limit(mut self, pool: impl Into<String>, max_in_flight: usize) -> Self {
740 let pool = pool.into();
741 assert!(
742 crate::tasks::validate_pool(&pool).is_ok(),
743 "invalid pool name `{pool}` (allowed: [A-Za-z0-9._-], 1..=128 chars)"
744 );
745 self.task_pool_limits.insert(pool, max_in_flight);
746 self
747 }
748
749 /// Builder-style: set the default tenant quota for the durable task
750 /// queue (R0.6 wave 3), applied to every tenant without a
751 /// [`ServerConfig::with_tenant_quota`] override. Over quota, task
752 /// submissions answer `429 quota_exceeded`. The default is uncapped on
753 /// every gauge. See [`TaskQuota`] for the gauge definitions.
754 pub fn with_task_quota(mut self, quota: TaskQuota) -> Self {
755 self.task_quota = quota;
756 self
757 }
758
759 /// Builder-style: override the task quota for one tenant (R0.6 wave 3).
760 /// The override replaces [`ServerConfig::task_quota`] wholesale for
761 /// that tenant — an uncapped gauge in the override is unlimited even if
762 /// the default caps it.
763 ///
764 /// # Panics
765 ///
766 /// Panics on an invalid tenant id (same rule as
767 /// [`ServerConfig::with_tenant_key`]).
768 pub fn with_tenant_quota(mut self, tenant: impl Into<String>, quota: TaskQuota) -> Self {
769 let tenant = tenant.into();
770 let valid = !tenant.is_empty()
771 && tenant.len() <= 64
772 && tenant
773 .chars()
774 .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.'))
775 && !RESERVED_NAMES.contains(&tenant.as_str());
776 assert!(
777 valid,
778 "invalid tenant id `{tenant}` (allowed: [A-Za-z0-9._-], 1..=64 chars, not a reserved name)"
779 );
780 self.tenant_task_quotas.insert(tenant, quota);
781 self
782 }
783
784 /// The quota governing `tenant`: its override when one is configured,
785 /// else the server-wide default ([`ServerConfig::task_quota`]).
786 pub(crate) fn quota_for(&self, tenant: &str) -> &TaskQuota {
787 self.tenant_task_quotas
788 .get(tenant)
789 .unwrap_or(&self.task_quota)
790 }
791
792 /// Builder-style: set the promotion envelope (R0.8 Rusty Learn, wave
793 /// 3) — the declared, per-deployment standing approval the promotion
794 /// gate evaluates every promotion against. Replaces
795 /// [`PromotionEnvelope::r08_default`].
796 ///
797 /// # Panics
798 ///
799 /// Panics when the envelope fails [`PromotionEnvelope::validate`]
800 /// (a canary fraction outside `(0, 1]`, a negative improvement bar —
801 /// configuration is a programmer error, caught at startup).
802 pub fn with_promotion_envelope(mut self, envelope: PromotionEnvelope) -> Self {
803 if let Err(error) = envelope.validate() {
804 panic!("invalid promotion envelope: {error}");
805 }
806 self.promotion_envelope = envelope;
807 self
808 }
809
810 /// Builder-style: register the candidate evaluator (R0.8 wave 3) —
811 /// the evaluation composition `POST /learn/candidates/{id}/evaluate`
812 /// drives. Without one the route answers `409`: promotion is gated
813 /// on evidence, and evidence requires an evaluator.
814 pub fn with_candidate_evaluator(mut self, evaluator: Arc<dyn CandidateEvaluator>) -> Self {
815 self.candidate_evaluator = Some(evaluator);
816 self
817 }
818
819 /// Builder-style: register the revision gate evaluator (R0.12 wave
820 /// 4) — the release-gate composition promotions into gated
821 /// environments drive. Without one a gated promotion answers `409`:
822 /// the gate fails closed, so an unavailable evaluator is a refusal,
823 /// never a skip.
824 pub fn with_revision_gate_evaluator(
825 mut self,
826 evaluator: Arc<dyn RevisionGateEvaluator>,
827 ) -> Self {
828 self.revision_gate_evaluator = Some(evaluator);
829 self
830 }
831
832 /// Builder-style: declare the deployment's default environment tag
833 /// (R0.11 wave 2) — the promotion target a registry-bound run
834 /// resolves against when its binding names no environment. `None`
835 /// (the default) resolves the untagged surface. The tag is validated
836 /// at construction (the same rules the wire applies), so no
837 /// deployment can declare a default its own pointers could never
838 /// carry.
839 pub fn with_default_environment_tag(mut self, tag: impl Into<String>) -> Self {
840 self.default_environment_tag = Some(
841 EnvironmentTag::new(tag.into())
842 .unwrap_or_else(|e| panic!("invalid default environment tag: {e}")),
843 );
844 self
845 }
846
847 /// Builder-style: add one operator-authored Cedar policy file (R0.9
848 /// Rusty Capsules, wave 2), loaded for the `default` tenant at
849 /// startup. Files are *registered*, never activated — moving a
850 /// tenant's active pointer is an explicit operator move through
851 /// `POST /capsule_policies/active`, so a restart can never silently
852 /// change what serves. Without the `capsules` feature the files are
853 /// ignored (the plane's routes answer `503` either way).
854 pub fn with_capsule_policy_file(mut self, path: impl Into<PathBuf>) -> Self {
855 self.capsule_policy_files.push(path.into());
856 self
857 }
858
859 /// Builder-style: set the tenant-wide capsule budget ceiling (R0.9
860 /// wave 2) — the outermost bound every capsule budget composes
861 /// under. `None` (the default) leaves budgets bounded only by the
862 /// run's own declaration and each capsule's manifest.
863 pub fn with_capsule_budget_ceiling(mut self, ceiling: ResourceBudget) -> Self {
864 self.capsule_budget_ceiling = Some(ceiling);
865 self
866 }
867
868 /// Builder-style: set the network connector for capsules the A2A
869 /// bridge executes in-process (R0.9 wave 4) — the deployment's
870 /// explicit egress seam. `None` (the default) refuses capsule
871 /// execution over A2A with `capsule_execution_unavailable`: guest
872 /// code never runs against an egress path the operator did not wire.
873 #[cfg(feature = "capsules")]
874 pub fn with_capsule_connector(
875 mut self,
876 connector: Arc<dyn rusty_agent_runtime::capsule_host::NetworkConnector>,
877 ) -> Self {
878 self.capsule_connector = Some(connector);
879 self
880 }
881
882 /// Builder-style: set the credential handle TTL (R0.11 wave 3). The
883 /// TTL bounds how long one issuance is reused, not how long a
884 /// revocation takes to bite — resolution reads live state, so a
885 /// revoked connection fails at the very next call.
886 pub fn with_broker_handle_ttl(mut self, ttl: std::time::Duration) -> Self {
887 self.broker_handle_ttl = ttl;
888 self
889 }
890
891 /// Builder-style: plug the OAuth provider (R0.11 wave 4) — the
892 /// deployment's token-endpoint seam. One plug enables the whole
893 /// lifecycle: authorization-code exchange on the connection routes,
894 /// refresh-at-resolution, and the sweeper. Without it the exchange
895 /// endpoints answer `409` and OAuth connections serve static
896 /// material.
897 pub fn with_oauth_provider(
898 mut self,
899 provider: Arc<dyn rusty_agent_runtime::broker::OAuthProvider>,
900 ) -> Self {
901 self.oauth_provider = Some(provider);
902 self
903 }
904
905 /// Builder-style: the OAuth refresh window (R0.11 wave 4). Tokens
906 /// expiring within this horizon (deterministically jittered per
907 /// connection) rotate before they serve.
908 pub fn with_broker_refresh_window(mut self, window: std::time::Duration) -> Self {
909 self.broker_refresh_window = window;
910 self
911 }
912
913 /// Builder-style: run the broker sweeper on this interval (R0.11
914 /// wave 4). Off by default; meaningful only with an OAuth provider
915 /// plugged in (without one there is nothing to sweep).
916 pub fn with_broker_sweep_interval(mut self, interval: std::time::Duration) -> Self {
917 self.broker_sweep_interval = Some(interval);
918 self
919 }
920
921 /// Builder-style: run the artifact retention sweeper on this interval
922 /// (R0.12 wave 2). Off by default; `POST /artifacts/sweep` and the
923 /// release act's prune tail run the same evaluation, so a stopped
924 /// sweeper degrades to operator-triggered passes, never to
925 /// unprotected pruning.
926 pub fn with_artifact_sweep_interval(mut self, interval: std::time::Duration) -> Self {
927 self.artifact_sweep_interval = Some(interval);
928 self
929 }
930}
931
932/// Build the axum [`Router`] for a registry and config. Use this to embed the
933/// rusty-agent-server routes into a larger application, or to drive the API in tests
934/// via `tower::ServiceExt::oneshot`.
935///
936/// The router's drain control is internal and never fires (embedders who
937/// want cooperative drain should use [`router_with_shutdown`]).
938pub fn router(registry: GraphRegistry, config: ServerConfig) -> Router {
939 router_with_shutdown(registry, config, CancellationToken::new())
940}
941
942/// Build the axum [`Router`] with an explicit drain control (R0.6 wave 2c).
943///
944/// Cancelling `shutdown` starts the cooperative drain **inside** the
945/// application — in-flight runs stop at their next checkpoint boundary, the
946/// outbox relay finishes its current pass and stops, the cron scheduler
947/// stops firing, and new run submissions are rejected with `503` — while
948/// the HTTP surface keeps serving so in-flight requests can complete.
949/// Stopping HTTP itself is the embedder's job (axum's
950/// `with_graceful_shutdown`); [`serve_with_shutdown`] wires both halves
951/// together.
952pub fn router_with_shutdown(
953 registry: GraphRegistry,
954 config: ServerConfig,
955 shutdown: CancellationToken,
956) -> Router {
957 routes::router_with_shutdown(registry, config, shutdown)
958}
959
960/// Build the axum [`Router`] and hand back the credential broker it wired
961/// (R0.11 wave 4) — the app's own broker, so in-process mediators (graph
962/// nodes carrying a
963/// [`CredentialMediator`](rusty_agent_runtime::broker::CredentialMediator),
964/// capsule hosts) issue and resolve against the same custody and the
965/// same evidence chain the HTTP connection routes drive. The release
966/// proof's runs connector-resolve exactly this way. Drain control is
967/// internal and never fires, as for [`router`]; embedders wanting
968/// cooperative drain compose [`router_with_shutdown`] and read the
969/// broker off app state instead.
970pub fn router_with_broker(
971 registry: GraphRegistry,
972 config: ServerConfig,
973) -> (
974 Router,
975 Arc<dyn rusty_agent_runtime::broker::CredentialBroker>,
976) {
977 let (router, broker) = routes::build_router(registry, config, CancellationToken::new());
978 (router, broker)
979}
980
981/// Build the router and bind it to `config.bind_addr`. Blocks until the
982/// server shuts down: SIGINT or SIGTERM starts the graceful drain described
983/// on [`serve_with_shutdown`].
984pub async fn serve(registry: GraphRegistry, config: ServerConfig) -> std::io::Result<()> {
985 serve_with_shutdown(registry, config, shutdown_signal()).await
986}
987
988/// Build the router, bind it to `config.bind_addr`, and serve until
989/// `shutdown` resolves, then drain gracefully (R0.6 wave 2c). The drain is
990/// ordered so the common case of a rolling deploy strands nothing:
991///
992/// 1. **Stop taking new work.** axum stops accepting connections; the
993/// server's shared drain token fires, so new run submissions answer
994/// `503` and the cron scheduler stops firing.
995/// 2. **Let in-flight work land.** In-flight requests complete (axum waits
996/// for them); in-flight runs are cooperatively cancelled at their next
997/// super-step boundary — where a checkpoint was just persisted — and
998/// end terminal-`cancelled`, resumable by re-running the thread; the
999/// outbox relay finishes its current publish pass and stops (pending
1000/// rows are durable and publish on the next process's first pass).
1001/// 3. **Stop, bounded.** The whole drain is capped at
1002/// [`ServerConfig::shutdown_grace`]. Past it the server returns anyway:
1003/// anything still in flight is abandoned mid-step, which is exactly the
1004/// crash case the durable machinery already covers — runs resume from
1005/// their last checkpoint and leased tasks return to visibility within
1006/// one lease period. Grace makes the common case fast; it is never the
1007/// correctness mechanism.
1008pub async fn serve_with_shutdown(
1009 registry: GraphRegistry,
1010 config: ServerConfig,
1011 shutdown: impl std::future::Future<Output = ()> + Send + 'static,
1012) -> std::io::Result<()> {
1013 let addr = config.bind_addr;
1014 let grace = config.shutdown_grace;
1015 // Open (dev) mode on a non-loopback address exposes the full API — run
1016 // creation, KV writes, checkpoint deletion — to the network. That's a
1017 // legitimate dev choice, but it must never be a quiet one.
1018 if !config.auth_enabled() && !addr.ip().is_loopback() {
1019 tracing::warn!(
1020 %addr,
1021 "serving WITHOUT authentication on a non-loopback address; \
1022 configure `with_api_key`/`with_tenant_key` or bind 127.0.0.1"
1023 );
1024 }
1025 let draining = CancellationToken::new();
1026 let app = router_with_shutdown(registry, config, draining.clone());
1027 let listener = tokio::net::TcpListener::bind(addr).await?;
1028 tracing::info!(%addr, "rusty-server listening");
1029 let server = axum::serve(listener, app).with_graceful_shutdown({
1030 let draining = draining.clone();
1031 async move {
1032 shutdown.await;
1033 tracing::info!(
1034 grace_ms = grace.as_millis() as u64,
1035 "shutdown requested; draining"
1036 );
1037 // One token drives the whole cooperative drain: runs stop at
1038 // their next checkpoint boundary, the relay finishes its pass,
1039 // the cron scheduler stops, new runs are rejected.
1040 draining.cancel();
1041 }
1042 });
1043 tokio::select! {
1044 // `WithGracefulShutdown` is `IntoFuture`, not `Future`, so it needs
1045 // this async wrapper to sit in a `select!` arm.
1046 result = async { server.await } => result,
1047 () = async {
1048 // The grace clock starts when the drain does. axum's graceful
1049 // shutdown waits for in-flight connections without a bound of
1050 // its own, so this arm is the bound.
1051 draining.cancelled().await;
1052 tokio::time::sleep(grace).await;
1053 } => {
1054 tracing::warn!(
1055 grace_ms = grace.as_millis() as u64,
1056 "drain grace expired; forcing shutdown — in-flight runs resume \
1057 from their last checkpoint and leased tasks return at lease expiry"
1058 );
1059 Ok(())
1060 }
1061 }
1062}
1063
1064/// The default shutdown signal for [`serve`]: resolves on SIGINT or SIGTERM
1065/// (the two signals orchestrators and terminals actually send). Exposed so
1066/// binaries that compose their own shutdown logic can reuse it.
1067pub async fn shutdown_signal() {
1068 let ctrl_c = tokio::signal::ctrl_c();
1069 #[cfg(unix)]
1070 {
1071 // Signal-handler installation only fails when the fd limit is
1072 // exhausted; a server that cannot hear SIGTERM must not start
1073 // pretending it drains gracefully.
1074 let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
1075 .expect("installing the SIGTERM handler must succeed");
1076 tokio::select! {
1077 _ = ctrl_c => {},
1078 _ = sigterm.recv() => {},
1079 }
1080 }
1081 #[cfg(not(unix))]
1082 {
1083 let _ = ctrl_c.await;
1084 }
1085}