Skip to main content

xs/
api.rs

1use std::collections::HashMap;
2use std::error::Error;
3use std::str::FromStr;
4
5use chrono::{DateTime, Utc};
6use scru128::Scru128Id;
7
8use base64::Engine;
9
10use tokio::io::AsyncWriteExt;
11use tokio_stream::wrappers::ReceiverStream;
12use tokio_stream::StreamExt;
13use tokio_util::io::ReaderStream;
14
15use http_body_util::StreamBody;
16use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full};
17use hyper::body::Bytes;
18use hyper::header::ACCEPT;
19use hyper::server::conn::http1;
20use hyper::service::service_fn;
21use hyper::{Method, Request, Response, StatusCode};
22use hyper_util::rt::TokioIo;
23
24use crate::listener::Listener;
25use crate::nu;
26use crate::store::{self, FollowOption, Frame, ReadOptions, Store, TTL};
27
28type HTTPResult = Result<Response<BoxBody<Bytes, crate::Error>>, crate::Error>;
29
30/// Serialize a frame to JSON, optionally including a timestamp extracted from the scru128 ID
31fn serialize_frame(frame: &Frame, with_timestamp: bool) -> String {
32    if with_timestamp {
33        let mut value = serde_json::to_value(frame).unwrap();
34        if let serde_json::Value::Object(ref mut map) = value {
35            let millis = frame.id.timestamp() as i64;
36            let dt: DateTime<Utc> = DateTime::from_timestamp_millis(millis)
37                .unwrap_or_else(|| DateTime::from_timestamp(0, 0).unwrap());
38            map.insert("timestamp".to_string(), serde_json::json!(dt.to_rfc3339()));
39        }
40        serde_json::to_string(&value).unwrap()
41    } else {
42        serde_json::to_string(frame).unwrap()
43    }
44}
45
46#[derive(Debug, PartialEq, Clone)]
47enum AcceptType {
48    Ndjson,
49    EventStream,
50}
51
52enum Routes {
53    StreamCat {
54        accept_type: AcceptType,
55        options: ReadOptions,
56        with_timestamp: bool,
57    },
58    StreamAppend {
59        topic: String,
60        ttl: Option<TTL>,
61        with_timestamp: bool,
62    },
63    LastGet {
64        topic: Option<String>,
65        last: usize,
66        follow: bool,
67        with_timestamp: bool,
68    },
69    StreamItemGet {
70        id: Scru128Id,
71        with_timestamp: bool,
72    },
73    StreamItemRemove(Scru128Id),
74    CasGet(ssri::Integrity),
75    CasPost,
76    Import,
77    Eval,
78    Version,
79    Metrics,
80    NotFound,
81    BadRequest(String),
82}
83
84/// Validates an Integrity object to ensure all its hashes are properly formatted
85fn validate_integrity(integrity: &ssri::Integrity) -> bool {
86    // Check if there are any hashes
87    if integrity.hashes.is_empty() {
88        return false;
89    }
90
91    // For each hash, check if it has a valid base64-encoded digest
92    for hash in &integrity.hashes {
93        // Check if digest is valid base64 using the modern API
94        if base64::engine::general_purpose::STANDARD
95            .decode(&hash.digest)
96            .is_err()
97        {
98            return false;
99        }
100    }
101
102    true
103}
104
105fn match_route(
106    method: &Method,
107    path: &str,
108    headers: &hyper::HeaderMap,
109    query: Option<&str>,
110) -> Routes {
111    let params: HashMap<String, String> =
112        url::form_urlencoded::parse(query.unwrap_or("").as_bytes())
113            .into_owned()
114            .collect();
115
116    match (method, path) {
117        (&Method::GET, "/version") => Routes::Version,
118        (&Method::GET, "/metrics") => Routes::Metrics,
119
120        (&Method::GET, "/") => {
121            let accept_type = match headers.get(ACCEPT) {
122                Some(accept) if accept == "text/event-stream" => AcceptType::EventStream,
123                _ => AcceptType::Ndjson,
124            };
125
126            let options = ReadOptions::from_query(query);
127            let with_timestamp = params.contains_key("with-timestamp");
128
129            match options {
130                Ok(options) => Routes::StreamCat {
131                    accept_type,
132                    options,
133                    with_timestamp,
134                },
135                Err(e) => Routes::BadRequest(e.to_string()),
136            }
137        }
138
139        (&Method::GET, "/last") => {
140            let follow = params.contains_key("follow");
141            let last = params.get("last").and_then(|v| v.parse().ok()).unwrap_or(1);
142            let with_timestamp = params.contains_key("with-timestamp");
143            Routes::LastGet {
144                topic: None,
145                last,
146                follow,
147                with_timestamp,
148            }
149        }
150
151        (&Method::GET, p) if p.starts_with("/last/") => {
152            let topic = p.strip_prefix("/last/").unwrap().to_string();
153            let follow = params.contains_key("follow");
154            let last = params.get("last").and_then(|v| v.parse().ok()).unwrap_or(1);
155            let with_timestamp = params.contains_key("with-timestamp");
156            Routes::LastGet {
157                topic: Some(topic),
158                last,
159                follow,
160                with_timestamp,
161            }
162        }
163
164        (&Method::GET, p) if p.starts_with("/cas/") => {
165            if let Some(hash) = p.strip_prefix("/cas/") {
166                match ssri::Integrity::from_str(hash) {
167                    Ok(integrity) => {
168                        if validate_integrity(&integrity) {
169                            Routes::CasGet(integrity)
170                        } else {
171                            Routes::BadRequest(format!("Invalid CAS hash format: {hash}"))
172                        }
173                    }
174                    Err(e) => Routes::BadRequest(format!("Invalid CAS hash: {e}")),
175                }
176            } else {
177                Routes::NotFound
178            }
179        }
180
181        (&Method::POST, "/cas") => Routes::CasPost,
182        (&Method::POST, "/import") => Routes::Import,
183        (&Method::POST, "/eval") => Routes::Eval,
184
185        (&Method::GET, p) => match Scru128Id::from_str(p.trim_start_matches('/')) {
186            Ok(id) => {
187                let with_timestamp = params.contains_key("with-timestamp");
188                Routes::StreamItemGet { id, with_timestamp }
189            }
190            Err(e) => Routes::BadRequest(format!("Invalid frame ID: {e}")),
191        },
192
193        (&Method::DELETE, p) => match Scru128Id::from_str(p.trim_start_matches('/')) {
194            Ok(id) => Routes::StreamItemRemove(id),
195            Err(e) => Routes::BadRequest(format!("Invalid frame ID: {e}")),
196        },
197
198        (&Method::POST, path) if path.starts_with("/append/") => {
199            let topic = path.strip_prefix("/append/").unwrap().to_string();
200            let with_timestamp = params.contains_key("with-timestamp");
201
202            match TTL::from_query(query) {
203                Ok(ttl) => Routes::StreamAppend {
204                    topic,
205                    ttl: Some(ttl),
206                    with_timestamp,
207                },
208                Err(e) => Routes::BadRequest(e.to_string()),
209            }
210        }
211
212        _ => Routes::NotFound,
213    }
214}
215
216async fn handle(
217    mut store: Store,
218    _engine: nu::Engine, // TODO: potentially vestigial, will .process come back?
219    req: Request<hyper::body::Incoming>,
220) -> HTTPResult {
221    let method = req.method();
222    let path = req.uri().path();
223    let headers = req.headers().clone();
224    let query = req.uri().query();
225
226    let res = match match_route(method, path, &headers, query) {
227        Routes::Version => handle_version().await,
228        Routes::Metrics => handle_metrics(&store).await,
229
230        Routes::StreamCat {
231            accept_type,
232            options,
233            with_timestamp,
234        } => handle_stream_cat(&mut store, options, accept_type, with_timestamp).await,
235
236        Routes::StreamAppend {
237            topic,
238            ttl,
239            with_timestamp,
240        } => handle_stream_append(&mut store, req, topic, ttl, with_timestamp).await,
241
242        Routes::CasGet(hash) => {
243            let reader = store.cas_reader(hash).await?;
244            let stream = ReaderStream::new(reader);
245
246            let stream = stream.map(|frame| {
247                let frame = frame.unwrap();
248                Ok(hyper::body::Frame::data(frame))
249            });
250
251            let body = StreamBody::new(stream).boxed();
252            Ok(Response::new(body))
253        }
254
255        Routes::CasPost => handle_cas_post(&mut store, req.into_body()).await,
256
257        Routes::StreamItemGet { id, with_timestamp } => {
258            response_frame_or_404(store.get(&id), with_timestamp)
259        }
260
261        Routes::StreamItemRemove(id) => handle_stream_item_remove(&mut store, id).await,
262
263        Routes::LastGet {
264            topic,
265            last,
266            follow,
267            with_timestamp,
268        } => handle_last_get(&store, topic.as_deref(), last, follow, with_timestamp).await,
269
270        Routes::Import => handle_import(&mut store, req.into_body()).await,
271
272        Routes::Eval => handle_eval(&store, req.into_body()).await,
273
274        Routes::NotFound => response_404(),
275        Routes::BadRequest(msg) => response_400(msg),
276    };
277
278    res.or_else(|e| response_500(e.to_string()))
279}
280
281/// The trailer that says a read ended because the reader fell behind the
282/// live stream: `xs-lagged: <frames skipped>` (ADR 0008).
283pub const LAGGED_TRAILER: &str = "xs-lagged";
284
285/// The trailer that carries an `eval` script's error.
286pub const ERROR_TRAILER: &str = "xs-error";
287
288/// A nushell error's own words: the innermost error, past the wrappers
289/// (`each` wraps what its closure raised), with a generic error's details.
290fn error_text(error: &nu_protocol::ShellError) -> String {
291    match error {
292        nu_protocol::ShellError::Generic(g) if g.msg.is_empty() => g.error.to_string(),
293        nu_protocol::ShellError::Generic(g) => format!("{}: {}", g.error, g.msg),
294        nu_protocol::ShellError::EvalBlockWithInput { sources, .. } if !sources.is_empty() => {
295            error_text(&sources[0])
296        }
297        other => other.to_string(),
298    }
299}
300
301/// Text as a header value: visible ASCII only, anything else as a space.
302fn header_text(text: &str) -> hyper::header::HeaderValue {
303    let clean: String = text
304        .chars()
305        .map(|c| if c.is_ascii_graphic() { c } else { ' ' })
306        .collect();
307    hyper::header::HeaderValue::from_str(clean.trim())
308        .unwrap_or(hyper::header::HeaderValue::from_static("error"))
309}
310
311/// A read's item as body frames. A read that fell behind ends with the
312/// `xs.lagged` line, which line readers (curl, nu) see, then the
313/// `xs-lagged` trailer, which the CLI reads instead of parsing lines.
314fn body_frames(
315    item: crate::store::ReadItem,
316    encode: &impl Fn(&Frame) -> Vec<u8>,
317) -> Vec<hyper::body::Frame<Bytes>> {
318    match item {
319        Ok(frame) => vec![hyper::body::Frame::data(Bytes::from(encode(&frame)))],
320        Err(err) => {
321            let crate::store::ReadError::Lagged { skipped } = &err;
322            let mut trailers = hyper::HeaderMap::new();
323            trailers.insert(LAGGED_TRAILER, (*skipped).into());
324            vec![
325                hyper::body::Frame::data(Bytes::from(encode(&err.to_frame()))),
326                hyper::body::Frame::trailers(trailers),
327            ]
328        }
329    }
330}
331
332async fn handle_stream_cat(
333    store: &mut Store,
334    options: ReadOptions,
335    accept_type: AcceptType,
336    with_timestamp: bool,
337) -> HTTPResult {
338    let rx = store.read(options);
339
340    let accept_type_clone = accept_type.clone();
341    let encode = move |frame: &Frame| match accept_type_clone {
342        AcceptType::Ndjson => {
343            let mut encoded = serialize_frame(frame, with_timestamp).into_bytes();
344            encoded.push(b'\n');
345            encoded
346        }
347        AcceptType::EventStream => format!(
348            "id: {id}\ndata: {data}\n\n",
349            id = frame.id,
350            data = serialize_frame(frame, with_timestamp)
351        )
352        .into_bytes(),
353    };
354    let stream = futures::StreamExt::flat_map(ReceiverStream::new(rx), move |item| {
355        futures::stream::iter(body_frames(item, &encode))
356    })
357    .map(Ok::<_, crate::Error>);
358
359    let body = StreamBody::new(stream).boxed();
360
361    let content_type = match accept_type {
362        AcceptType::Ndjson => "application/x-ndjson",
363        AcceptType::EventStream => "text/event-stream",
364    };
365
366    Ok(Response::builder()
367        .status(StatusCode::OK)
368        .header("Content-Type", content_type)
369        .header("Trailer", LAGGED_TRAILER)
370        .body(body)?)
371}
372
373async fn handle_stream_append<B>(
374    store: &mut Store,
375    req: Request<B>,
376    topic: String,
377    ttl: Option<TTL>,
378    with_timestamp: bool,
379) -> HTTPResult
380where
381    B: hyper::body::Body<Data = Bytes> + Unpin,
382    B::Error: Into<crate::Error>,
383{
384    let (parts, mut body) = req.into_parts();
385
386    let hash = {
387        // Opened on the first non-empty chunk: a meta-only append never
388        // touches the CAS, and opening a writer creates a temp file.
389        let mut writer = None;
390
391        while let Some(frame) = body.frame().await {
392            if let Ok(data) = frame.map_err(Into::into)?.into_data() {
393                if data.is_empty() {
394                    continue;
395                }
396                if writer.is_none() {
397                    writer = Some(store.cas_writer().await?);
398                }
399                writer.as_mut().unwrap().write_all(&data).await?;
400            }
401        }
402
403        match writer {
404            Some(writer) => Some(writer.commit().await?),
405            None => None,
406        }
407    };
408
409    let meta = match parts
410        .headers
411        .get("xs-meta")
412        .map(|x| x.to_str())
413        .transpose()
414        .unwrap()
415        .map(|s| {
416            // First decode the Base64-encoded string
417            base64::prelude::BASE64_STANDARD
418                .decode(s)
419                .map_err(|e| format!("xs-meta isn't valid Base64: {e}"))
420                .and_then(|decoded| {
421                    // Then parse the decoded bytes as UTF-8 string
422                    String::from_utf8(decoded)
423                        .map_err(|e| format!("xs-meta isn't valid UTF-8: {e}"))
424                        .and_then(|json_str| {
425                            // Finally parse the UTF-8 string as JSON
426                            serde_json::from_str(&json_str)
427                                .map_err(|e| format!("xs-meta isn't valid JSON: {e}"))
428                        })
429                })
430        })
431        .transpose()
432    {
433        Ok(meta) => meta,
434        Err(e) => return response_400(e.to_string()),
435    };
436
437    let frame = store.append(
438        Frame::builder(topic)
439            .maybe_hash(hash)
440            .maybe_meta(meta)
441            .maybe_ttl(ttl)
442            .build(),
443    )?;
444
445    Ok(Response::builder()
446        .status(StatusCode::OK)
447        .header("Content-Type", "application/json")
448        .body(full(serialize_frame(&frame, with_timestamp)))?)
449}
450
451async fn handle_cas_post(store: &mut Store, mut body: hyper::body::Incoming) -> HTTPResult {
452    let hash = {
453        let mut writer = store.cas_writer().await?;
454        let mut bytes_written = 0;
455
456        while let Some(frame) = body.frame().await {
457            if let Ok(data) = frame?.into_data() {
458                writer.write_all(&data).await?;
459                bytes_written += data.len();
460            }
461        }
462
463        if bytes_written == 0 {
464            return response_400("Empty body".to_string());
465        }
466
467        writer.commit().await?
468    };
469
470    Ok(Response::builder()
471        .status(StatusCode::OK)
472        .header("Content-Type", "text/plain")
473        .body(full(hash.to_string()))?)
474}
475
476async fn handle_version() -> HTTPResult {
477    let version = env!("CARGO_PKG_VERSION");
478    let version_info = serde_json::json!({ "version": version });
479    Ok(Response::builder()
480        .status(StatusCode::OK)
481        .header("Content-Type", "application/json")
482        .body(full(serde_json::to_string(&version_info).unwrap()))?)
483}
484
485/// `GET /metrics`: counters for this process, as JSON.
486///
487/// Everything here is cumulative since the process started and cannot be
488/// reset, so a rate is the difference between two reads. The `keyspaces`
489/// section comes from fjall and describes cache and filter behaviour; the
490/// `store` section is xs' own view of what it spent that behaviour on.
491///
492/// Nothing here is a stream frame on purpose: metrics on the stream would be
493/// caught by the reader's own filters and TTLs.
494async fn handle_metrics(store: &Store) -> HTTPResult {
495    let body = serde_json::json!({
496        "store": store.stats(),
497        "keyspaces": store.metrics(),
498    });
499    Ok(Response::builder()
500        .status(StatusCode::OK)
501        .header("Content-Type", "application/json")
502        .body(full(serde_json::to_string(&body).unwrap()))?)
503}
504
505pub async fn serve(
506    store: Store,
507    engine: nu::Engine,
508    expose: Option<String>,
509) -> Result<(), crate::Error> {
510    let path = store.path.join("sock").to_string_lossy().to_string();
511    let listener = Listener::bind(&path).await?;
512
513    let mut listeners = vec![listener];
514    let mut expose_meta = None;
515
516    if let Some(expose) = expose {
517        let expose_listener = Listener::bind(&expose).await?;
518
519        // Check if this is an iroh listener and get the ticket
520        if let Some(ticket) = expose_listener.get_ticket() {
521            expose_meta = Some(serde_json::json!({"expose": format!("iroh://{}", ticket)}));
522        } else {
523            expose_meta = Some(serde_json::json!({"expose": expose}));
524        }
525
526        listeners.push(expose_listener);
527    }
528
529    if let Err(e) = store.append(Frame::builder("xs.start").maybe_meta(expose_meta).build()) {
530        tracing::error!("Failed to append xs.start frame: {}", e);
531    }
532
533    let mut tasks = Vec::new();
534    for listener in listeners {
535        let store = store.clone();
536        let engine = engine.clone();
537        let task = tokio::spawn(async move { listener_loop(listener, store, engine).await });
538        tasks.push(task);
539    }
540
541    // TODO: graceful shutdown and error handling
542    // Wait for all listener tasks to complete (or until the first error)
543    for task in tasks {
544        task.await??;
545    }
546
547    Ok(())
548}
549
550async fn listener_loop(
551    mut listener: Listener,
552    store: Store,
553    engine: nu::Engine,
554) -> Result<(), crate::Error> {
555    loop {
556        let (stream, _) = listener.accept().await?;
557        let io = TokioIo::new(stream);
558        let store = store.clone();
559        let engine = engine.clone();
560        tokio::task::spawn(async move {
561            if let Err(err) = http1::Builder::new()
562                .serve_connection(
563                    io,
564                    service_fn(move |req| handle(store.clone(), engine.clone(), req)),
565                )
566                .await
567            {
568                // Match against the error kind to selectively ignore `NotConnected` errors
569                if let Some(std::io::ErrorKind::NotConnected) = err.source().and_then(|source| {
570                    source
571                        .downcast_ref::<std::io::Error>()
572                        .map(|io_err| io_err.kind())
573                }) {
574                    // ignore the NotConnected error, hyper's way of saying the client disconnected
575                } else {
576                    // todo, Handle or log other errors
577                    tracing::error!("TBD: {:?}", err);
578                }
579            }
580        });
581    }
582}
583
584fn response_frame_or_404(frame: Option<store::Frame>, with_timestamp: bool) -> HTTPResult {
585    if let Some(frame) = frame {
586        Ok(Response::builder()
587            .status(StatusCode::OK)
588            .header("Content-Type", "application/json")
589            .body(full(serialize_frame(&frame, with_timestamp)))?)
590    } else {
591        response_404()
592    }
593}
594
595async fn handle_stream_item_remove(store: &mut Store, id: Scru128Id) -> HTTPResult {
596    match store.remove(&id) {
597        Ok(()) => Ok(Response::builder()
598            .status(StatusCode::NO_CONTENT)
599            .body(empty())?),
600        Err(e) => {
601            tracing::error!("Failed to remove item {}: {:?}", id, e);
602
603            Ok(Response::builder()
604                .status(StatusCode::INTERNAL_SERVER_ERROR)
605                .body(full("internal-error"))?)
606        }
607    }
608}
609
610async fn handle_last_get(
611    store: &Store,
612    topic: Option<&str>,
613    last: usize,
614    follow: bool,
615    with_timestamp: bool,
616) -> HTTPResult {
617    if !follow {
618        let options = ReadOptions::builder()
619            .last(last)
620            .maybe_topic(topic.map(|t| t.to_string()))
621            .build();
622
623        let frames: Vec<Frame> = store.read_sync(options).collect();
624
625        if frames.is_empty() {
626            return response_404();
627        }
628
629        // Format based on request, not result count:
630        // - last == 1 (default): JSON object
631        // - last > 1: NDJSON (even if fewer frames returned)
632        if last == 1 {
633            return response_frame_or_404(Some(frames.into_iter().next().unwrap()), with_timestamp);
634        }
635
636        let mut body = Vec::new();
637        for frame in frames {
638            body.extend(serialize_frame(&frame, with_timestamp).into_bytes());
639            body.push(b'\n');
640        }
641
642        return Ok(Response::builder()
643            .status(StatusCode::OK)
644            .header("Content-Type", "application/x-ndjson")
645            .body(full(body))?);
646    }
647
648    // Follow mode: use ReadOptions::last to get historical + live frames
649    // This emits xs.threshold after historical frames
650    let rx = store.read(
651        ReadOptions::builder()
652            .last(last)
653            .maybe_topic(topic.map(|t| t.to_string()))
654            .follow(FollowOption::On)
655            .build(),
656    );
657
658    let encode = move |frame: &Frame| {
659        let mut bytes = serialize_frame(frame, with_timestamp).into_bytes();
660        bytes.push(b'\n');
661        bytes
662    };
663    let stream = futures::StreamExt::flat_map(ReceiverStream::new(rx), move |item| {
664        futures::stream::iter(body_frames(item, &encode))
665    })
666    .map(Ok::<_, crate::Error>);
667
668    Ok(Response::builder()
669        .status(StatusCode::OK)
670        .header("Content-Type", "application/x-ndjson")
671        .header("Trailer", LAGGED_TRAILER)
672        .body(StreamBody::new(stream).boxed())?)
673}
674
675async fn handle_import(store: &mut Store, body: hyper::body::Incoming) -> HTTPResult {
676    let bytes = body.collect().await?.to_bytes();
677    let frame: Frame = match serde_json::from_slice(&bytes) {
678        Ok(frame) => frame,
679        Err(e) => return response_400(format!("Invalid frame JSON: {e}")),
680    };
681
682    store.insert_frame(&frame)?;
683
684    Ok(Response::builder()
685        .status(StatusCode::OK)
686        .header("Content-Type", "application/json")
687        .body(full(serde_json::to_string(&frame).unwrap()))?)
688}
689
690fn response_404() -> HTTPResult {
691    Ok(Response::builder()
692        .status(StatusCode::NOT_FOUND)
693        .body(empty())?)
694}
695
696fn response_400(message: String) -> HTTPResult {
697    let body = full(message);
698    Ok(Response::builder()
699        .status(StatusCode::BAD_REQUEST)
700        .body(body)?)
701}
702
703fn response_500(message: String) -> HTTPResult {
704    let body = full(message);
705    Ok(Response::builder()
706        .status(StatusCode::INTERNAL_SERVER_ERROR)
707        .body(body)?)
708}
709
710fn full<T: Into<Bytes>>(chunk: T) -> BoxBody<Bytes, crate::Error> {
711    Full::new(chunk.into())
712        .map_err(|never| match never {})
713        .boxed()
714}
715
716fn empty() -> BoxBody<Bytes, crate::Error> {
717    Empty::<Bytes>::new()
718        .map_err(|never| match never {})
719        .boxed()
720}
721
722/// Engine for an ad-hoc `.eval` script: the prepared base (Stream reads +
723/// Direct `.append`) plus the VFS modules registered so far, so eval scripts
724/// can `use` them, the same builtins the runners get.
725fn eval_engine(store: &Store) -> Result<nu::Engine, String> {
726    let mut engine = nu::prepared_base(store, true)
727        .map_err(|e| format!("Failed to build nushell engine: {e}"))?;
728    // Modules registered up to now: a fresh time-ordered id exceeds every
729    // already-appended frame.
730    let modules = store.nu_modules_at(&scru128::new());
731    nu::load_modules(&mut engine.state, store, &modules)
732        .map_err(|e| format!("Failed to load modules: {e}"))?;
733    Ok(engine)
734}
735
736async fn handle_eval(store: &Store, body: hyper::body::Incoming) -> HTTPResult {
737    // Read the script from the request body
738    let bytes = body.collect().await?.to_bytes();
739    let script =
740        String::from_utf8(bytes.to_vec()).map_err(|e| format!("Invalid UTF-8 in script: {e}"))?;
741
742    let engine = eval_engine(store)?;
743
744    // Execute the script on a dedicated thread, not on this tokio runtime
745    // thread. handle_eval is an async handler, so engine.eval would otherwise
746    // run on a runtime thread. The script can call .cat/.last, whose historical
747    // path parks on tokio's blocking_recv, which panics ("Cannot block the
748    // current thread from within a runtime") on a runtime thread. Running the
749    // eval on a plain std::thread avoids that.
750    let result = std::thread::scope(|scope| {
751        scope
752            .spawn(|| engine.eval(nu_protocol::PipelineData::empty(), script))
753            .join()
754            .expect("eval thread panicked")
755    })
756    .map_err(|e| format!("Script evaluation failed:\n{e}"))?;
757
758    // Format output based on PipelineData type according to spec
759    match result {
760        nu_protocol::PipelineData::ByteStream(stream, ..) => {
761            // ByteStream → raw bytes with proper streaming using channel pattern
762            if let Some(mut reader) = stream.reader() {
763                use std::io::Read;
764
765                let (tx, rx) = tokio::sync::mpsc::channel(16);
766
767                // Spawn sync task to read from nushell Reader and send to channel
768                std::thread::spawn(move || {
769                    let mut buffer = [0u8; 8192];
770                    loop {
771                        match reader.read(&mut buffer) {
772                            Ok(0) => break, // EOF
773                            Ok(n) => {
774                                let chunk = Bytes::copy_from_slice(&buffer[..n]);
775                                if tx
776                                    .blocking_send(Ok(hyper::body::Frame::data(chunk)))
777                                    .is_err()
778                                {
779                                    break;
780                                }
781                            }
782                            Err(e) => {
783                                let _ = tx.blocking_send(Err(Box::new(e) as crate::Error));
784                                break;
785                            }
786                        }
787                    }
788                });
789
790                let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
791                let body = StreamBody::new(stream).boxed();
792                Ok(Response::builder()
793                    .status(StatusCode::OK)
794                    .header("Content-Type", "application/octet-stream")
795                    .body(body)?)
796            } else {
797                // No reader available, return empty response
798                Ok(Response::builder()
799                    .status(StatusCode::OK)
800                    .header("Content-Type", "application/octet-stream")
801                    .body(empty())?)
802            }
803        }
804        nu_protocol::PipelineData::ListStream(stream, ..) => {
805            // ListStream → JSONL stream with proper streaming using channel pattern
806            let (tx, rx) = tokio::sync::mpsc::channel(16);
807
808            // Spawn sync task to iterate stream and send JSONL to channel
809            std::thread::spawn(move || {
810                for value in stream.into_iter() {
811                    // An error in the stream (a lagging `.cat -f`, say) ends
812                    // the response with its message as a trailer; turned
813                    // into JSON it would be `null` (util.rs value_to_json).
814                    if let nu_protocol::Value::Error { error, .. } = &value {
815                        let mut trailers = hyper::HeaderMap::new();
816                        let text = error_text(error);
817                        trailers.insert(ERROR_TRAILER, header_text(&text));
818                        let _ = tx.blocking_send(Ok(hyper::body::Frame::trailers(trailers)));
819                        break;
820                    }
821                    let json = nu::value_to_json(&value);
822                    match serde_json::to_vec(&json) {
823                        Ok(mut json_bytes) => {
824                            json_bytes.push(b'\n'); // Add newline for JSONL
825                            let chunk = Bytes::from(json_bytes);
826                            if tx
827                                .blocking_send(Ok(hyper::body::Frame::data(chunk)))
828                                .is_err()
829                            {
830                                break;
831                            }
832                        }
833                        Err(e) => {
834                            let _ = tx.blocking_send(Err(Box::new(e) as crate::Error));
835                            break;
836                        }
837                    }
838                }
839            });
840
841            let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
842            let body = StreamBody::new(stream).boxed();
843            Ok(Response::builder()
844                .status(StatusCode::OK)
845                .header("Content-Type", "application/x-ndjson")
846                .header("Trailer", ERROR_TRAILER)
847                .body(body)?)
848        }
849        nu_protocol::PipelineData::Value(value, ..) => {
850            match &value {
851                nu_protocol::Value::String { .. }
852                | nu_protocol::Value::Int { .. }
853                | nu_protocol::Value::Float { .. }
854                | nu_protocol::Value::Bool { .. } => {
855                    // Single primitive value → raw text
856                    let text = match value {
857                        nu_protocol::Value::String { val, .. } => val.clone(),
858                        nu_protocol::Value::Int { val, .. } => val.to_string(),
859                        nu_protocol::Value::Float { val, .. } => val.to_string(),
860                        nu_protocol::Value::Bool { val, .. } => val.to_string(),
861                        _ => value.into_string().unwrap_or_else(|_| "".to_string()),
862                    };
863                    Ok(Response::builder()
864                        .status(StatusCode::OK)
865                        .header("Content-Type", "text/plain")
866                        .body(full(text))?)
867                }
868                _ => {
869                    // Single structured value → JSON
870                    let json = nu::value_to_json(&value);
871                    let json_string = serde_json::to_string(&json)
872                        .map_err(|e| format!("Failed to serialize JSON: {e}"))?;
873                    Ok(Response::builder()
874                        .status(StatusCode::OK)
875                        .header("Content-Type", "application/json")
876                        .body(full(json_string))?)
877                }
878            }
879        }
880        nu_protocol::PipelineData::Empty => {
881            // Empty → nothing
882            Ok(Response::builder()
883                .status(StatusCode::NO_CONTENT)
884                .body(empty())?)
885        }
886    }
887}
888
889#[cfg(test)]
890mod tests {
891    use super::*;
892
893    #[test]
894    fn a_lag_ends_with_the_line_then_the_trailer() {
895        let encode = |frame: &Frame| frame.topic.clone().into_bytes();
896        let frames = body_frames(
897            Err(crate::store::ReadError::Lagged { skipped: 506 }),
898            &encode,
899        );
900        assert_eq!(frames.len(), 2);
901        assert_eq!(frames[0].data_ref().unwrap().as_ref(), b"xs.lagged");
902        let trailers = frames[1].trailers_ref().unwrap();
903        assert_eq!(trailers.get(LAGGED_TRAILER).unwrap(), "506");
904    }
905
906    #[tokio::test]
907    async fn test_handle_stream_append_body() {
908        let temp_dir = tempfile::tempdir().unwrap();
909        let mut store = Store::new(temp_dir.path().to_path_buf()).unwrap();
910
911        async fn append(
912            store: &mut Store,
913            meta: Option<&str>,
914            chunks: Vec<&'static [u8]>,
915        ) -> serde_json::Value {
916            let stream = tokio_stream::iter(chunks.into_iter().map(|chunk| {
917                Ok::<_, crate::Error>(hyper::body::Frame::data(Bytes::from_static(chunk)))
918            }));
919            let mut req = Request::post("/append/note");
920            if let Some(meta) = meta {
921                req = req.header("xs-meta", base64::prelude::BASE64_STANDARD.encode(meta));
922            }
923            let req = req.body(StreamBody::new(stream)).unwrap();
924            let response = handle_stream_append(store, req, "note".into(), None, false)
925                .await
926                .unwrap();
927            assert_eq!(StatusCode::OK, response.status());
928            let body = response.into_body().collect().await.unwrap().to_bytes();
929            serde_json::from_slice(&body).unwrap()
930        }
931
932        // No body, or only empty chunks: the frame is stored with no hash.
933        let frame = append(&mut store, None, vec![]).await;
934        assert!(frame["hash"].is_null());
935        let frame = append(&mut store, None, vec![b"", b""]).await;
936        assert!(frame["hash"].is_null());
937
938        // A body split across chunks, with empty chunks around it, lands
939        // whole in the CAS.
940        let frame = append(&mut store, None, vec![b"", b"hello ", b"", b"world"]).await;
941        let hash: ssri::Integrity = frame["hash"].as_str().unwrap().parse().unwrap();
942        assert_eq!(
943            b"hello world".to_vec(),
944            store.cas_read(&hash).await.unwrap()
945        );
946
947        // Meta and a body together.
948        let frame = append(&mut store, Some(r#"{"n":1}"#), vec![b"x"]).await;
949        assert_eq!(serde_json::json!({"n": 1}), frame["meta"]);
950        let hash: ssri::Integrity = frame["hash"].as_str().unwrap().parse().unwrap();
951        assert_eq!(b"x".to_vec(), store.cas_read(&hash).await.unwrap());
952    }
953
954    #[test]
955    fn test_match_route_last() {
956        let headers = hyper::HeaderMap::new();
957
958        // /last/topic without follow
959        assert!(matches!(
960            match_route(&Method::GET, "/last/test", &headers, None),
961            Routes::LastGet { topic: Some(t), last: 1, follow: false, .. } if t == "test"
962        ));
963
964        // /last/topic with follow
965        assert!(matches!(
966            match_route(&Method::GET, "/last/test", &headers, Some("follow=true")),
967            Routes::LastGet { topic: Some(t), last: 1, follow: true, .. } if t == "test"
968        ));
969
970        // /last without topic
971        assert!(matches!(
972            match_route(&Method::GET, "/last", &headers, None),
973            Routes::LastGet {
974                topic: None,
975                last: 1,
976                follow: false,
977                ..
978            }
979        ));
980
981        // /last with last=5
982        assert!(matches!(
983            match_route(&Method::GET, "/last", &headers, Some("last=5")),
984            Routes::LastGet {
985                topic: None,
986                last: 5,
987                follow: false,
988                ..
989            }
990        ));
991
992        // /last/topic with last=3 and follow
993        assert!(matches!(
994            match_route(&Method::GET, "/last/test", &headers, Some("last=3&follow=true")),
995            Routes::LastGet { topic: Some(t), last: 3, follow: true, .. } if t == "test"
996        ));
997    }
998
999    #[tokio::test]
1000    async fn test_handle_last_get_ndjson_format() {
1001        use crate::store::Store;
1002        use http_body_util::BodyExt;
1003
1004        let temp_dir = tempfile::tempdir().unwrap();
1005        let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1006
1007        // Add one frame
1008        store
1009            .append(crate::store::Frame::builder("test").build())
1010            .unwrap();
1011
1012        // Request last=1 (default) - should return JSON object (no trailing newline)
1013        let response = handle_last_get(&store, Some("test"), 1, false, false)
1014            .await
1015            .unwrap();
1016        let body = response.into_body().collect().await.unwrap().to_bytes();
1017        let body_str = String::from_utf8(body.to_vec()).unwrap();
1018        assert!(
1019            !body_str.ends_with('\n'),
1020            "Response should be JSON (no trailing newline) when last=1, got: {:?}",
1021            body_str
1022        );
1023
1024        // Request last=3 but only 1 frame exists - should still return NDJSON
1025        let response = handle_last_get(&store, Some("test"), 3, false, false)
1026            .await
1027            .unwrap();
1028        let body = response.into_body().collect().await.unwrap().to_bytes();
1029        let body_str = String::from_utf8(body.to_vec()).unwrap();
1030        assert!(
1031            body_str.ends_with('\n'),
1032            "Response should be NDJSON (end with newline) when last > 1, got: {:?}",
1033            body_str
1034        );
1035
1036        // Verify full chain: match_route -> dispatch -> handle_last_get
1037        let headers = hyper::HeaderMap::new();
1038        let route = match_route(&Method::GET, "/last/test", &headers, Some("last=3"));
1039        if let Routes::LastGet {
1040            topic,
1041            last,
1042            follow,
1043            with_timestamp,
1044        } = route
1045        {
1046            assert_eq!(last, 3, "Route should parse last=3");
1047            let response = handle_last_get(&store, topic.as_deref(), last, follow, with_timestamp)
1048                .await
1049                .unwrap();
1050            let body = response.into_body().collect().await.unwrap().to_bytes();
1051            let body_str = String::from_utf8(body.to_vec()).unwrap();
1052            assert!(
1053                body_str.ends_with('\n'),
1054                "Full chain should return NDJSON when last=3, got: {:?}",
1055                body_str
1056            );
1057        } else {
1058            panic!("Expected Routes::LastGet");
1059        }
1060    }
1061
1062    #[tokio::test]
1063    async fn test_handle_eval_logic() {
1064        // Test the core nushell execution logic by testing the engine directly
1065        use crate::nu::Engine;
1066        use crate::store::Store;
1067        use nu_protocol::PipelineData;
1068
1069        // Create a temporary store for testing
1070        let temp_dir = tempfile::tempdir().unwrap();
1071        let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1072
1073        // Create nushell engine with store helper commands
1074        let mut engine = Engine::new().unwrap();
1075
1076        // Add core commands
1077        crate::nu::add_core_commands(&mut engine, &store).unwrap();
1078
1079        // Add streaming commands
1080        engine
1081            .add_commands(vec![
1082                Box::new(crate::nu::commands::cat_command::CatCommand::new(
1083                    store.clone(),
1084                )),
1085                Box::new(crate::nu::commands::last_command::LastCommand::new(
1086                    store.clone(),
1087                )),
1088                Box::new(crate::nu::commands::append_command::AppendCommand::new(
1089                    store.clone(),
1090                )),
1091            ])
1092            .unwrap();
1093
1094        // Test simple string expression
1095        let result = engine
1096            .eval(PipelineData::empty(), r#""hello world""#.to_string())
1097            .unwrap();
1098
1099        match result {
1100            PipelineData::Value(value, ..) => {
1101                let text = value.into_string().unwrap();
1102                assert_eq!(text, "hello world");
1103            }
1104            _ => panic!("Expected Value, got {:?}", result),
1105        }
1106
1107        // Test simple math expression - result should be an integer
1108        let result = engine
1109            .eval(PipelineData::empty(), "2 + 3".to_string())
1110            .unwrap();
1111
1112        match result {
1113            PipelineData::Value(nu_protocol::Value::Int { val, .. }, ..) => {
1114                assert_eq!(val, 5);
1115            }
1116            _ => panic!("Expected Int Value, got {:?}", result),
1117        }
1118    }
1119
1120    // An `.eval` script must be able to `use` a registered VFS module.
1121    #[tokio::test]
1122    async fn test_eval_uses_modules() {
1123        use crate::store::{Frame, Store};
1124        use nu_protocol::PipelineData;
1125
1126        let temp_dir = tempfile::tempdir().unwrap();
1127        let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1128
1129        let hash = store
1130            .cas_insert_sync(r#"export def add_nums [x, y] { $"sum is ($x + $y)" }"#)
1131            .unwrap();
1132        store
1133            .append(Frame::builder("xs.module.mymod").hash(hash).build())
1134            .unwrap();
1135
1136        let engine = super::eval_engine(&store).unwrap();
1137        let result = engine
1138            .eval(
1139                PipelineData::empty(),
1140                "use mymod; mymod add_nums 40 2".to_string(),
1141            )
1142            .unwrap()
1143            .into_value(nu_protocol::Span::test_data())
1144            .unwrap();
1145        assert_eq!(result.into_string().unwrap(), "sum is 42");
1146    }
1147
1148    // A module's exported function must be able to call the store builtins (here
1149    // `.append`), not just pure nushell. (rokf94's report; the engine carries
1150    // the builtins before modules load.)
1151    #[tokio::test]
1152    async fn test_module_can_use_append_builtin() {
1153        use crate::store::{Frame, Store};
1154        use nu_protocol::{PipelineData, Span};
1155
1156        let temp_dir = tempfile::tempdir().unwrap();
1157        let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1158
1159        let hash = store
1160            .cas_insert_sync(r#"export def emit [] { {} | .append out --meta {via: "module"} }"#)
1161            .unwrap();
1162        store
1163            .append(Frame::builder("xs.module.emitter").hash(hash).build())
1164            .unwrap();
1165
1166        let engine = super::eval_engine(&store).unwrap();
1167        let value = engine
1168            .eval(
1169                PipelineData::empty(),
1170                "use emitter; emitter emit".to_string(),
1171            )
1172            .unwrap()
1173            .into_value(Span::test_data())
1174            .unwrap();
1175
1176        // `.append` ran inside the module and returned the appended frame.
1177        let json = crate::nu::value_to_json(&value);
1178        assert_eq!(json["topic"], "out");
1179        assert_eq!(json["meta"]["via"], "module");
1180    }
1181}