inferd-daemon 0.2.0

The inferd daemon: NDJSON-over-IPC server, admission queue, single-instance lock, router, activity log.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
//! v2 connection lifecycle — Phase 2A wired.
//!
//! Per ADR 0015, v2 lives on a *separate* socket from v1. This module
//! mirrors `lifecycle.rs` but for the v2 wire types
//! (`inferd_proto::v2::RequestV2` / `ResponseV2`).
//!
//! Per request:
//!   1. Read one NDJSON frame, parse as `RequestV2`.
//!   2. `RequestV2::resolve()` — structural validation.
//!   3. Admission gate (same Admission shared with v1; one slot
//!      is one slot regardless of wire version).
//!   4. Dispatch through the router; check the chosen backend's
//!      `capabilities().v2` flag — backends that don't support v2
//!      yield `Error{Internal, "v2 not supported by this backend"}`.
//!   5. `backend.generate_v2(resolved)` — pre-stream errors map to
//!      v2 error codes; mid-stream backend failure (no Done) maps to
//!      `BackendUnavailable`.
//!   6. Stream `TokenEventV2`s, translating each to the
//!      corresponding `ResponseV2::Frame` / `Done`.

use crate::auth::{AuthFrame, key_matches};
use crate::endpoint::Connection;
use crate::peercred::PeerIdentity;
use crate::queue::SubmitError;
use crate::router::{Router, RouterError};
use inferd_engine::{GenerateError, TokenEventV2};
use inferd_proto::ProtoError;
use inferd_proto::v2::{ErrorCodeV2, RequestV2, ResponseBlock, ResponseV2};
use inferd_proto::write_frame;
use std::io;
use std::sync::Arc;
use tokio::io::{AsyncWrite, AsyncWriteExt, BufReader};
use tokio::sync::Mutex;
use tokio_stream::StreamExt;
use tracing::{debug, info, warn};

/// Per-accept context for v2 connections. v2 reuses v1's
/// `AcceptContext` shape — same TCP API key, same admission
/// gate. The admission gate, in particular, sees v1 and v2
/// requests as equivalent: one slot is one slot.
pub use crate::lifecycle::AcceptContext;

/// Handle one accepted v2 client connection.
pub async fn handle_v2_connection<C: Connection + 'static>(
    mut conn: C,
    router: Arc<Router>,
    peer: PeerIdentity,
    ctx: AcceptContext,
) -> Result<(), io::Error> {
    let transport = conn.transport();
    info!(
        target: "inferd_daemon::activity",
        transport = transport,
        wire_version = "v2",
        peer = %peer,
        peer_uid = peer.uid,
        peer_pid = peer.pid,
        peer_sid = peer.sid.as_deref(),
        "v2_connection_accepted"
    );

    let (read_half, write_half) = tokio::io::split(&mut conn);
    let mut reader = BufReader::with_capacity(64 * 1024, read_half);
    let writer = Arc::new(Mutex::new(write_half));

    // F-8 first-frame auth on TCP, identical to v1.
    if transport == "tcp"
        && let Some(expected) = ctx.expected_api_key.as_deref()
    {
        match read_auth_frame(&mut reader).await {
            Some(frame) if key_matches(&frame.key, expected) => {
                debug!(transport, "v2 tcp auth ok");
            }
            _ => {
                warn!(
                    target: "inferd_daemon::activity",
                    peer = %peer,
                    wire_version = "v2",
                    "v2_tcp_auth_rejected"
                );
                return Ok(());
            }
        }
    }

    loop {
        let request: RequestV2 = match read_request_v2(&mut reader).await {
            Ok(Some(r)) => r,
            Ok(None) => return Ok(()),
            Err(ProtoError::Io(e)) => return Err(e),
            Err(e) => {
                let resp = ResponseV2::Error {
                    id: String::new(),
                    code: error_code_for(&e),
                    message: e.to_string(),
                };
                write_response_v2(&writer, &resp).await?;
                return Ok(());
            }
        };

        let id = request.id.clone();
        let resolved = match request.resolve() {
            Ok(r) => r,
            Err(e) => {
                let resp = ResponseV2::Error {
                    id,
                    code: ErrorCodeV2::InvalidRequest,
                    message: e.to_string(),
                };
                write_response_v2(&writer, &resp).await?;
                continue;
            }
        };

        // Admission gate. v1 and v2 share one Admission instance; a
        // v2 in-flight request occupies the same slot a v1 one would.
        let _admit_permit = match ctx.admission.as_ref().map(|a| a.try_admit()) {
            None => None,
            Some(Ok(p)) => Some(p),
            Some(Err(SubmitError::QueueFull)) => {
                let resp = ResponseV2::Error {
                    id: resolved.id.clone(),
                    code: ErrorCodeV2::QueueFull,
                    message: "queue full".into(),
                };
                write_response_v2(&writer, &resp).await?;
                continue;
            }
            Some(Err(SubmitError::Closed)) => {
                let resp = ResponseV2::Error {
                    id: resolved.id.clone(),
                    code: ErrorCodeV2::BackendUnavailable,
                    message: "admission closed".into(),
                };
                write_response_v2(&writer, &resp).await?;
                return Ok(());
            }
        };

        // Dispatch through the router.
        let dispatch = match router.dispatch() {
            Ok(d) => d,
            Err(RouterError::NoBackends) | Err(RouterError::NoneAvailable) => {
                let resp = ResponseV2::Error {
                    id: resolved.id.clone(),
                    code: ErrorCodeV2::BackendUnavailable,
                    message: "no backend available".into(),
                };
                write_response_v2(&writer, &resp).await?;
                continue;
            }
        };
        let backend_name = dispatch.name.clone();
        let backend = dispatch.backend;

        // Backends that don't advertise v2 capability are not given a
        // generate_v2 call — the trait's default impl would also
        // refuse, but checking up front lets us emit a clearer error
        // and avoids paying for a method-dispatch round-trip just to
        // surface "not supported".
        if !backend.capabilities().v2 {
            let resp = ResponseV2::Error {
                id: resolved.id.clone(),
                code: ErrorCodeV2::Internal,
                message: format!("backend {backend_name:?} does not advertise v2 capability"),
            };
            write_response_v2(&writer, &resp).await?;
            continue;
        }

        let req_id = resolved.id.clone();
        let n_attachments = resolved.attachments.len();
        let n_tools = resolved.tools.len();

        let mut stream = match backend.generate_v2(resolved).await {
            Ok(s) => s,
            Err(e) => {
                let (code, message, is_backend_failure) = match e {
                    GenerateError::InvalidRequest(m) => (ErrorCodeV2::InvalidRequest, m, false),
                    GenerateError::NotReady => (
                        ErrorCodeV2::BackendUnavailable,
                        "backend not ready".into(),
                        true,
                    ),
                    GenerateError::Unavailable(m) => (ErrorCodeV2::BackendUnavailable, m, true),
                    GenerateError::Internal(m) => (ErrorCodeV2::Internal, m, true),
                };
                if is_backend_failure {
                    router.record_failure(&backend_name);
                }
                let resp = ResponseV2::Error {
                    id: req_id,
                    code,
                    message,
                };
                write_response_v2(&writer, &resp).await?;
                continue;
            }
        };

        let mut terminal_emitted = false;
        while let Some(ev) = stream.next().await {
            match ev {
                TokenEventV2::Text(delta) => {
                    let frame = ResponseV2::Frame {
                        id: req_id.clone(),
                        block: ResponseBlock::Text { delta },
                    };
                    write_response_v2(&writer, &frame).await?;
                }
                TokenEventV2::Thinking(delta) => {
                    let frame = ResponseV2::Frame {
                        id: req_id.clone(),
                        block: ResponseBlock::Thinking { delta },
                    };
                    write_response_v2(&writer, &frame).await?;
                }
                TokenEventV2::ToolUse {
                    tool_call_id,
                    name,
                    input,
                } => {
                    let frame = ResponseV2::Frame {
                        id: req_id.clone(),
                        block: ResponseBlock::ToolUse {
                            tool_call_id,
                            name,
                            input,
                        },
                    };
                    write_response_v2(&writer, &frame).await?;
                }
                TokenEventV2::Done { stop_reason, usage } => {
                    let frame = ResponseV2::Done {
                        id: req_id.clone(),
                        usage,
                        stop_reason,
                        backend: backend_name.clone(),
                    };
                    write_response_v2(&writer, &frame).await?;
                    router.record_success(&backend_name);
                    info!(
                        target: "inferd_daemon::activity",
                        req_id = %req_id,
                        backend = %backend_name,
                        wire_version = "v2",
                        stop_reason = ?stop_reason,
                        input_tokens = usage.input_tokens,
                        output_tokens = usage.output_tokens,
                        n_attachments = n_attachments,
                        n_tools = n_tools,
                        "v2_request_done"
                    );
                    terminal_emitted = true;
                    break;
                }
            }
        }

        if !terminal_emitted {
            router.record_failure(&backend_name);
            warn!(
                target: "inferd_daemon::activity",
                req_id = %req_id,
                backend = %backend_name,
                wire_version = "v2",
                "v2_request_error_mid_stream"
            );
            let frame = ResponseV2::Error {
                id: req_id,
                code: ErrorCodeV2::BackendUnavailable,
                message: "backend ended stream without terminal frame".into(),
            };
            write_response_v2(&writer, &frame).await?;
        }
    }
}

fn error_code_for(e: &ProtoError) -> ErrorCodeV2 {
    match e {
        ProtoError::FrameTooLarge => ErrorCodeV2::FrameTooLarge,
        ProtoError::Decode(_) | ProtoError::InvalidRequest(_) => ErrorCodeV2::InvalidRequest,
        ProtoError::Io(_) => ErrorCodeV2::Internal,
    }
}

async fn read_auth_frame<R>(reader: &mut R) -> Option<AuthFrame>
where
    R: tokio::io::AsyncBufRead + Unpin,
{
    use tokio::io::AsyncBufReadExt;
    let mut line = Vec::with_capacity(256);
    let limit = inferd_proto::MAX_FRAME_BYTES;
    loop {
        let buf = reader.fill_buf().await.ok()?;
        if buf.is_empty() {
            return None;
        }
        if let Some(idx) = buf.iter().position(|&b| b == b'\n') {
            if line.len() + idx > limit {
                return None;
            }
            line.extend_from_slice(&buf[..idx]);
            reader.consume(idx + 1);
            return AuthFrame::from_json(&line);
        }
        if line.len() + buf.len() > limit {
            return None;
        }
        line.extend_from_slice(buf);
        let n = buf.len();
        reader.consume(n);
    }
}

async fn read_request_v2<R>(reader: &mut R) -> Result<Option<RequestV2>, ProtoError>
where
    R: tokio::io::AsyncBufRead + Unpin,
{
    use tokio::io::AsyncBufReadExt;
    let mut line = Vec::with_capacity(512);
    let limit = inferd_proto::MAX_FRAME_BYTES;
    loop {
        let buf = reader.fill_buf().await?;
        if buf.is_empty() {
            if line.is_empty() {
                return Ok(None);
            }
            return inferd_proto::read_frame::<&[u8], RequestV2>(&mut &line[..]);
        }
        if let Some(idx) = buf.iter().position(|&b| b == b'\n') {
            if line.len() + idx > limit {
                return Err(ProtoError::FrameTooLarge);
            }
            line.extend_from_slice(&buf[..=idx]);
            reader.consume(idx + 1);
            return inferd_proto::read_frame::<&[u8], RequestV2>(&mut &line[..]);
        }
        if line.len() + buf.len() > limit {
            return Err(ProtoError::FrameTooLarge);
        }
        line.extend_from_slice(buf);
        let n = buf.len();
        reader.consume(n);
    }
}

async fn write_response_v2<W: AsyncWrite + Unpin>(
    writer: &Mutex<W>,
    resp: &ResponseV2,
) -> io::Result<()> {
    let mut buf = Vec::with_capacity(512);
    write_frame(&mut buf, resp)
        .map_err(|e| io::Error::other(format!("serialise v2 response: {e}")))?;
    let mut guard = writer.lock().await;
    guard.write_all(&buf).await?;
    guard.flush().await?;
    Ok(())
}

/// Serve a v2 TCP listener.
pub async fn serve_tcp_v2(
    listener: tokio::net::TcpListener,
    router: Arc<Router>,
    ctx: AcceptContext,
    mut shutdown: tokio::sync::oneshot::Receiver<()>,
) -> io::Result<()> {
    info!(addr = ?listener.local_addr()?, "v2 tcp listener accepting");
    loop {
        tokio::select! {
            _ = &mut shutdown => {
                info!("v2 tcp shutdown signalled");
                return Ok(());
            }
            accept = listener.accept() => {
                let (stream, peer_addr) = accept?;
                let peer = PeerIdentity::from_tcp(peer_addr);
                let r = Arc::clone(&router);
                let ctx = ctx.clone();
                debug!(?peer_addr, "v2 tcp accept");
                tokio::spawn(async move {
                    if let Err(e) = handle_v2_connection(stream, r, peer, ctx).await {
                        warn!(error = ?e, "v2 connection terminated with error");
                    }
                });
            }
        }
    }
}

/// Serve a v2 Unix domain socket listener.
#[cfg(unix)]
pub async fn serve_uds_v2(
    listener: tokio::net::UnixListener,
    router: Arc<Router>,
    ctx: AcceptContext,
    mut shutdown: tokio::sync::oneshot::Receiver<()>,
) -> io::Result<()> {
    info!("v2 uds listener accepting");
    loop {
        tokio::select! {
            _ = &mut shutdown => {
                info!("v2 uds shutdown signalled");
                return Ok(());
            }
            accept = listener.accept() => {
                let (stream, _) = accept?;
                let r = Arc::clone(&router);
                let peer = crate::peercred::unix::from_stream(&stream)
                    .unwrap_or_else(|e| {
                        warn!(error = %e, "v2 SO_PEERCRED failed; recording empty unix identity");
                        crate::peercred::PeerIdentity {
                            uid: None, gid: None, pid: None,
                            sid: None, remote_addr: None,
                            transport: "unix",
                        }
                    });
                let ctx = ctx.clone();
                debug!(?peer, "v2 uds accept");
                tokio::spawn(async move {
                    if let Err(e) = handle_v2_connection(stream, r, peer, ctx).await {
                        warn!(error = ?e, "v2 connection terminated with error");
                    }
                });
            }
        }
    }
}

/// Serve a v2 Windows named pipe listener.
#[cfg(windows)]
pub async fn serve_named_pipe_v2(
    path: &str,
    first_instance: tokio::net::windows::named_pipe::NamedPipeServer,
    router: Arc<Router>,
    ctx: AcceptContext,
    mut shutdown: tokio::sync::oneshot::Receiver<()>,
) -> io::Result<()> {
    use crate::endpoint::bind_named_pipe;

    info!(path = %path, "v2 named pipe listener accepting");
    let mut server = first_instance;
    loop {
        tokio::select! {
            _ = &mut shutdown => {
                info!("v2 named pipe shutdown signalled");
                return Ok(());
            }
            connect_result = server.connect() => {
                connect_result?;
                let connected = server;
                server = bind_named_pipe(path, false)?;

                let peer = crate::peercred::windows::from_stream(&connected)
                    .unwrap_or_else(|e| {
                        warn!(error = %e, "v2 GetNamedPipeClientProcessId failed; empty pipe identity");
                        crate::peercred::PeerIdentity {
                            uid: None, gid: None, pid: None,
                            sid: None, remote_addr: None,
                            transport: "pipe",
                        }
                    });
                let r = Arc::clone(&router);
                let ctx = ctx.clone();
                debug!(?peer, "v2 named pipe accept");
                tokio::spawn(async move {
                    if let Err(e) = handle_v2_connection(connected, r, peer, ctx).await {
                        warn!(error = ?e, "v2 connection terminated with error");
                    }
                });
            }
        }
    }
}