Skip to main content

xs/
api.rs

1use std::collections::HashMap;
2use std::error::Error;
3use std::str::FromStr;
4
5use chrono::{DateTime, Utc};
6use scru128::Scru128Id;
7
8use base64::Engine;
9
10use tokio::io::AsyncWriteExt;
11use tokio_stream::wrappers::ReceiverStream;
12use tokio_stream::StreamExt;
13use tokio_util::io::ReaderStream;
14
15use http_body_util::StreamBody;
16use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full};
17use hyper::body::Bytes;
18use hyper::header::ACCEPT;
19use hyper::server::conn::http1;
20use hyper::service::service_fn;
21use hyper::{Method, Request, Response, StatusCode};
22use hyper_util::rt::TokioIo;
23
24use crate::listener::Listener;
25use crate::nu;
26use crate::store::{self, FollowOption, Frame, ReadOptions, Store, TTL};
27
28type BoxError = Box<dyn std::error::Error + Send + Sync>;
29type HTTPResult = Result<Response<BoxBody<Bytes, BoxError>>, BoxError>;
30
31/// Serialize a frame to JSON, optionally including a timestamp extracted from the scru128 ID
32fn serialize_frame(frame: &Frame, with_timestamp: bool) -> String {
33    if with_timestamp {
34        let mut value = serde_json::to_value(frame).unwrap();
35        if let serde_json::Value::Object(ref mut map) = value {
36            let millis = frame.id.timestamp() as i64;
37            let dt: DateTime<Utc> = DateTime::from_timestamp_millis(millis)
38                .unwrap_or_else(|| DateTime::from_timestamp(0, 0).unwrap());
39            map.insert("timestamp".to_string(), serde_json::json!(dt.to_rfc3339()));
40        }
41        serde_json::to_string(&value).unwrap()
42    } else {
43        serde_json::to_string(frame).unwrap()
44    }
45}
46
47#[derive(Debug, PartialEq, Clone)]
48enum AcceptType {
49    Ndjson,
50    EventStream,
51}
52
53enum Routes {
54    StreamCat {
55        accept_type: AcceptType,
56        options: ReadOptions,
57        with_timestamp: bool,
58    },
59    StreamAppend {
60        topic: String,
61        ttl: Option<TTL>,
62        with_timestamp: bool,
63    },
64    LastGet {
65        topic: Option<String>,
66        last: usize,
67        follow: bool,
68        with_timestamp: bool,
69    },
70    StreamItemGet {
71        id: Scru128Id,
72        with_timestamp: bool,
73    },
74    StreamItemRemove(Scru128Id),
75    CasGet(ssri::Integrity),
76    CasPost,
77    Import,
78    Eval,
79    Version,
80    Metrics,
81    NotFound,
82    BadRequest(String),
83}
84
85/// Validates an Integrity object to ensure all its hashes are properly formatted
86fn validate_integrity(integrity: &ssri::Integrity) -> bool {
87    // Check if there are any hashes
88    if integrity.hashes.is_empty() {
89        return false;
90    }
91
92    // For each hash, check if it has a valid base64-encoded digest
93    for hash in &integrity.hashes {
94        // Check if digest is valid base64 using the modern API
95        if base64::engine::general_purpose::STANDARD
96            .decode(&hash.digest)
97            .is_err()
98        {
99            return false;
100        }
101    }
102
103    true
104}
105
106fn match_route(
107    method: &Method,
108    path: &str,
109    headers: &hyper::HeaderMap,
110    query: Option<&str>,
111) -> Routes {
112    let params: HashMap<String, String> =
113        url::form_urlencoded::parse(query.unwrap_or("").as_bytes())
114            .into_owned()
115            .collect();
116
117    match (method, path) {
118        (&Method::GET, "/version") => Routes::Version,
119        (&Method::GET, "/metrics") => Routes::Metrics,
120
121        (&Method::GET, "/") => {
122            let accept_type = match headers.get(ACCEPT) {
123                Some(accept) if accept == "text/event-stream" => AcceptType::EventStream,
124                _ => AcceptType::Ndjson,
125            };
126
127            let options = ReadOptions::from_query(query);
128            let with_timestamp = params.contains_key("with-timestamp");
129
130            match options {
131                Ok(options) => Routes::StreamCat {
132                    accept_type,
133                    options,
134                    with_timestamp,
135                },
136                Err(e) => Routes::BadRequest(e.to_string()),
137            }
138        }
139
140        (&Method::GET, "/last") => {
141            let follow = params.contains_key("follow");
142            let last = params.get("last").and_then(|v| v.parse().ok()).unwrap_or(1);
143            let with_timestamp = params.contains_key("with-timestamp");
144            Routes::LastGet {
145                topic: None,
146                last,
147                follow,
148                with_timestamp,
149            }
150        }
151
152        (&Method::GET, p) if p.starts_with("/last/") => {
153            let topic = p.strip_prefix("/last/").unwrap().to_string();
154            let follow = params.contains_key("follow");
155            let last = params.get("last").and_then(|v| v.parse().ok()).unwrap_or(1);
156            let with_timestamp = params.contains_key("with-timestamp");
157            Routes::LastGet {
158                topic: Some(topic),
159                last,
160                follow,
161                with_timestamp,
162            }
163        }
164
165        (&Method::GET, p) if p.starts_with("/cas/") => {
166            if let Some(hash) = p.strip_prefix("/cas/") {
167                match ssri::Integrity::from_str(hash) {
168                    Ok(integrity) => {
169                        if validate_integrity(&integrity) {
170                            Routes::CasGet(integrity)
171                        } else {
172                            Routes::BadRequest(format!("Invalid CAS hash format: {hash}"))
173                        }
174                    }
175                    Err(e) => Routes::BadRequest(format!("Invalid CAS hash: {e}")),
176                }
177            } else {
178                Routes::NotFound
179            }
180        }
181
182        (&Method::POST, "/cas") => Routes::CasPost,
183        (&Method::POST, "/import") => Routes::Import,
184        (&Method::POST, "/eval") => Routes::Eval,
185
186        (&Method::GET, p) => match Scru128Id::from_str(p.trim_start_matches('/')) {
187            Ok(id) => {
188                let with_timestamp = params.contains_key("with-timestamp");
189                Routes::StreamItemGet { id, with_timestamp }
190            }
191            Err(e) => Routes::BadRequest(format!("Invalid frame ID: {e}")),
192        },
193
194        (&Method::DELETE, p) => match Scru128Id::from_str(p.trim_start_matches('/')) {
195            Ok(id) => Routes::StreamItemRemove(id),
196            Err(e) => Routes::BadRequest(format!("Invalid frame ID: {e}")),
197        },
198
199        (&Method::POST, path) if path.starts_with("/append/") => {
200            let topic = path.strip_prefix("/append/").unwrap().to_string();
201            let with_timestamp = params.contains_key("with-timestamp");
202
203            match TTL::from_query(query) {
204                Ok(ttl) => Routes::StreamAppend {
205                    topic,
206                    ttl: Some(ttl),
207                    with_timestamp,
208                },
209                Err(e) => Routes::BadRequest(e.to_string()),
210            }
211        }
212
213        _ => Routes::NotFound,
214    }
215}
216
217async fn handle(
218    mut store: Store,
219    _engine: nu::Engine, // TODO: potentially vestigial, will .process come back?
220    req: Request<hyper::body::Incoming>,
221) -> HTTPResult {
222    let method = req.method();
223    let path = req.uri().path();
224    let headers = req.headers().clone();
225    let query = req.uri().query();
226
227    let res = match match_route(method, path, &headers, query) {
228        Routes::Version => handle_version().await,
229        Routes::Metrics => handle_metrics(&store).await,
230
231        Routes::StreamCat {
232            accept_type,
233            options,
234            with_timestamp,
235        } => handle_stream_cat(&mut store, options, accept_type, with_timestamp).await,
236
237        Routes::StreamAppend {
238            topic,
239            ttl,
240            with_timestamp,
241        } => handle_stream_append(&mut store, req, topic, ttl, with_timestamp).await,
242
243        Routes::CasGet(hash) => {
244            let reader = store.cas_reader(hash).await?;
245            let stream = ReaderStream::new(reader);
246
247            let stream = stream.map(|frame| {
248                let frame = frame.unwrap();
249                Ok(hyper::body::Frame::data(frame))
250            });
251
252            let body = StreamBody::new(stream).boxed();
253            Ok(Response::new(body))
254        }
255
256        Routes::CasPost => handle_cas_post(&mut store, req.into_body()).await,
257
258        Routes::StreamItemGet { id, with_timestamp } => {
259            response_frame_or_404(store.get(&id), with_timestamp)
260        }
261
262        Routes::StreamItemRemove(id) => handle_stream_item_remove(&mut store, id).await,
263
264        Routes::LastGet {
265            topic,
266            last,
267            follow,
268            with_timestamp,
269        } => handle_last_get(&store, topic.as_deref(), last, follow, with_timestamp).await,
270
271        Routes::Import => handle_import(&mut store, req.into_body()).await,
272
273        Routes::Eval => handle_eval(&store, req.into_body()).await,
274
275        Routes::NotFound => response_404(),
276        Routes::BadRequest(msg) => response_400(msg),
277    };
278
279    res.or_else(|e| response_500(e.to_string()))
280}
281
282async fn handle_stream_cat(
283    store: &mut Store,
284    options: ReadOptions,
285    accept_type: AcceptType,
286    with_timestamp: bool,
287) -> HTTPResult {
288    let rx = store.read(options);
289    let stream = ReceiverStream::new(rx);
290
291    let accept_type_clone = accept_type.clone();
292    let stream = stream.map(move |frame| {
293        let bytes = match accept_type_clone {
294            AcceptType::Ndjson => {
295                let mut encoded = serialize_frame(&frame, with_timestamp).into_bytes();
296                encoded.push(b'\n');
297                encoded
298            }
299            AcceptType::EventStream => format!(
300                "id: {id}\ndata: {data}\n\n",
301                id = frame.id,
302                data = serialize_frame(&frame, with_timestamp)
303            )
304            .into_bytes(),
305        };
306        Ok(hyper::body::Frame::data(Bytes::from(bytes)))
307    });
308
309    let body = StreamBody::new(stream).boxed();
310
311    let content_type = match accept_type {
312        AcceptType::Ndjson => "application/x-ndjson",
313        AcceptType::EventStream => "text/event-stream",
314    };
315
316    Ok(Response::builder()
317        .status(StatusCode::OK)
318        .header("Content-Type", content_type)
319        .body(body)?)
320}
321
322async fn handle_stream_append<B>(
323    store: &mut Store,
324    req: Request<B>,
325    topic: String,
326    ttl: Option<TTL>,
327    with_timestamp: bool,
328) -> HTTPResult
329where
330    B: hyper::body::Body<Data = Bytes> + Unpin,
331    B::Error: Into<BoxError>,
332{
333    let (parts, mut body) = req.into_parts();
334
335    let hash = {
336        // Opened on the first non-empty chunk: a meta-only append never
337        // touches the CAS, and opening a writer creates a temp file.
338        let mut writer = None;
339
340        while let Some(frame) = body.frame().await {
341            if let Ok(data) = frame.map_err(Into::into)?.into_data() {
342                if data.is_empty() {
343                    continue;
344                }
345                if writer.is_none() {
346                    writer = Some(store.cas_writer().await?);
347                }
348                writer.as_mut().unwrap().write_all(&data).await?;
349            }
350        }
351
352        match writer {
353            Some(writer) => Some(writer.commit().await?),
354            None => None,
355        }
356    };
357
358    let meta = match parts
359        .headers
360        .get("xs-meta")
361        .map(|x| x.to_str())
362        .transpose()
363        .unwrap()
364        .map(|s| {
365            // First decode the Base64-encoded string
366            base64::prelude::BASE64_STANDARD
367                .decode(s)
368                .map_err(|e| format!("xs-meta isn't valid Base64: {e}"))
369                .and_then(|decoded| {
370                    // Then parse the decoded bytes as UTF-8 string
371                    String::from_utf8(decoded)
372                        .map_err(|e| format!("xs-meta isn't valid UTF-8: {e}"))
373                        .and_then(|json_str| {
374                            // Finally parse the UTF-8 string as JSON
375                            serde_json::from_str(&json_str)
376                                .map_err(|e| format!("xs-meta isn't valid JSON: {e}"))
377                        })
378                })
379        })
380        .transpose()
381    {
382        Ok(meta) => meta,
383        Err(e) => return response_400(e.to_string()),
384    };
385
386    let frame = store.append(
387        Frame::builder(topic)
388            .maybe_hash(hash)
389            .maybe_meta(meta)
390            .maybe_ttl(ttl)
391            .build(),
392    )?;
393
394    Ok(Response::builder()
395        .status(StatusCode::OK)
396        .header("Content-Type", "application/json")
397        .body(full(serialize_frame(&frame, with_timestamp)))?)
398}
399
400async fn handle_cas_post(store: &mut Store, mut body: hyper::body::Incoming) -> HTTPResult {
401    let hash = {
402        let mut writer = store.cas_writer().await?;
403        let mut bytes_written = 0;
404
405        while let Some(frame) = body.frame().await {
406            if let Ok(data) = frame?.into_data() {
407                writer.write_all(&data).await?;
408                bytes_written += data.len();
409            }
410        }
411
412        if bytes_written == 0 {
413            return response_400("Empty body".to_string());
414        }
415
416        writer.commit().await?
417    };
418
419    Ok(Response::builder()
420        .status(StatusCode::OK)
421        .header("Content-Type", "text/plain")
422        .body(full(hash.to_string()))?)
423}
424
425async fn handle_version() -> HTTPResult {
426    let version = env!("CARGO_PKG_VERSION");
427    let version_info = serde_json::json!({ "version": version });
428    Ok(Response::builder()
429        .status(StatusCode::OK)
430        .header("Content-Type", "application/json")
431        .body(full(serde_json::to_string(&version_info).unwrap()))?)
432}
433
434/// `GET /metrics`: counters for this process, as JSON.
435///
436/// Everything here is cumulative since the process started and cannot be
437/// reset, so a rate is the difference between two reads. The `keyspaces`
438/// section comes from fjall and describes cache and filter behaviour; the
439/// `store` section is xs' own view of what it spent that behaviour on.
440///
441/// Nothing here is a stream frame on purpose: metrics on the stream would be
442/// caught by the reader's own filters and TTLs.
443async fn handle_metrics(store: &Store) -> HTTPResult {
444    let body = serde_json::json!({
445        "store": store.stats(),
446        "keyspaces": store.metrics(),
447    });
448    Ok(Response::builder()
449        .status(StatusCode::OK)
450        .header("Content-Type", "application/json")
451        .body(full(serde_json::to_string(&body).unwrap()))?)
452}
453
454pub async fn serve(
455    store: Store,
456    engine: nu::Engine,
457    expose: Option<String>,
458) -> Result<(), BoxError> {
459    let path = store.path.join("sock").to_string_lossy().to_string();
460    let listener = Listener::bind(&path).await?;
461
462    let mut listeners = vec![listener];
463    let mut expose_meta = None;
464
465    if let Some(expose) = expose {
466        let expose_listener = Listener::bind(&expose).await?;
467
468        // Check if this is an iroh listener and get the ticket
469        if let Some(ticket) = expose_listener.get_ticket() {
470            expose_meta = Some(serde_json::json!({"expose": format!("iroh://{}", ticket)}));
471        } else {
472            expose_meta = Some(serde_json::json!({"expose": expose}));
473        }
474
475        listeners.push(expose_listener);
476    }
477
478    if let Err(e) = store.append(Frame::builder("xs.start").maybe_meta(expose_meta).build()) {
479        tracing::error!("Failed to append xs.start frame: {}", e);
480    }
481
482    let mut tasks = Vec::new();
483    for listener in listeners {
484        let store = store.clone();
485        let engine = engine.clone();
486        let task = tokio::spawn(async move { listener_loop(listener, store, engine).await });
487        tasks.push(task);
488    }
489
490    // TODO: graceful shutdown and error handling
491    // Wait for all listener tasks to complete (or until the first error)
492    for task in tasks {
493        task.await??;
494    }
495
496    Ok(())
497}
498
499async fn listener_loop(
500    mut listener: Listener,
501    store: Store,
502    engine: nu::Engine,
503) -> Result<(), BoxError> {
504    loop {
505        let (stream, _) = listener.accept().await?;
506        let io = TokioIo::new(stream);
507        let store = store.clone();
508        let engine = engine.clone();
509        tokio::task::spawn(async move {
510            if let Err(err) = http1::Builder::new()
511                .serve_connection(
512                    io,
513                    service_fn(move |req| handle(store.clone(), engine.clone(), req)),
514                )
515                .await
516            {
517                // Match against the error kind to selectively ignore `NotConnected` errors
518                if let Some(std::io::ErrorKind::NotConnected) = err.source().and_then(|source| {
519                    source
520                        .downcast_ref::<std::io::Error>()
521                        .map(|io_err| io_err.kind())
522                }) {
523                    // ignore the NotConnected error, hyper's way of saying the client disconnected
524                } else {
525                    // todo, Handle or log other errors
526                    tracing::error!("TBD: {:?}", err);
527                }
528            }
529        });
530    }
531}
532
533fn response_frame_or_404(frame: Option<store::Frame>, with_timestamp: bool) -> HTTPResult {
534    if let Some(frame) = frame {
535        Ok(Response::builder()
536            .status(StatusCode::OK)
537            .header("Content-Type", "application/json")
538            .body(full(serialize_frame(&frame, with_timestamp)))?)
539    } else {
540        response_404()
541    }
542}
543
544async fn handle_stream_item_remove(store: &mut Store, id: Scru128Id) -> HTTPResult {
545    match store.remove(&id) {
546        Ok(()) => Ok(Response::builder()
547            .status(StatusCode::NO_CONTENT)
548            .body(empty())?),
549        Err(e) => {
550            tracing::error!("Failed to remove item {}: {:?}", id, e);
551
552            Ok(Response::builder()
553                .status(StatusCode::INTERNAL_SERVER_ERROR)
554                .body(full("internal-error"))?)
555        }
556    }
557}
558
559async fn handle_last_get(
560    store: &Store,
561    topic: Option<&str>,
562    last: usize,
563    follow: bool,
564    with_timestamp: bool,
565) -> HTTPResult {
566    if !follow {
567        let options = ReadOptions::builder()
568            .last(last)
569            .maybe_topic(topic.map(|t| t.to_string()))
570            .build();
571
572        let frames: Vec<Frame> = store.read_sync(options).collect();
573
574        if frames.is_empty() {
575            return response_404();
576        }
577
578        // Format based on request, not result count:
579        // - last == 1 (default): JSON object
580        // - last > 1: NDJSON (even if fewer frames returned)
581        if last == 1 {
582            return response_frame_or_404(Some(frames.into_iter().next().unwrap()), with_timestamp);
583        }
584
585        let mut body = Vec::new();
586        for frame in frames {
587            body.extend(serialize_frame(&frame, with_timestamp).into_bytes());
588            body.push(b'\n');
589        }
590
591        return Ok(Response::builder()
592            .status(StatusCode::OK)
593            .header("Content-Type", "application/x-ndjson")
594            .body(full(body))?);
595    }
596
597    // Follow mode: use ReadOptions::last to get historical + live frames
598    // This emits xs.threshold after historical frames
599    let rx = store.read(
600        ReadOptions::builder()
601            .last(last)
602            .maybe_topic(topic.map(|t| t.to_string()))
603            .follow(FollowOption::On)
604            .build(),
605    );
606
607    let stream = tokio_stream::wrappers::ReceiverStream::new(rx).map(move |frame| {
608        let mut bytes = serialize_frame(&frame, with_timestamp).into_bytes();
609        bytes.push(b'\n');
610        Ok::<_, BoxError>(hyper::body::Frame::data(Bytes::from(bytes)))
611    });
612
613    Ok(Response::builder()
614        .status(StatusCode::OK)
615        .header("Content-Type", "application/x-ndjson")
616        .body(StreamBody::new(stream).boxed())?)
617}
618
619async fn handle_import(store: &mut Store, body: hyper::body::Incoming) -> HTTPResult {
620    let bytes = body.collect().await?.to_bytes();
621    let frame: Frame = match serde_json::from_slice(&bytes) {
622        Ok(frame) => frame,
623        Err(e) => return response_400(format!("Invalid frame JSON: {e}")),
624    };
625
626    store.insert_frame(&frame)?;
627
628    Ok(Response::builder()
629        .status(StatusCode::OK)
630        .header("Content-Type", "application/json")
631        .body(full(serde_json::to_string(&frame).unwrap()))?)
632}
633
634fn response_404() -> HTTPResult {
635    Ok(Response::builder()
636        .status(StatusCode::NOT_FOUND)
637        .body(empty())?)
638}
639
640fn response_400(message: String) -> HTTPResult {
641    let body = full(message);
642    Ok(Response::builder()
643        .status(StatusCode::BAD_REQUEST)
644        .body(body)?)
645}
646
647fn response_500(message: String) -> HTTPResult {
648    let body = full(message);
649    Ok(Response::builder()
650        .status(StatusCode::INTERNAL_SERVER_ERROR)
651        .body(body)?)
652}
653
654fn full<T: Into<Bytes>>(chunk: T) -> BoxBody<Bytes, BoxError> {
655    Full::new(chunk.into())
656        .map_err(|never| match never {})
657        .boxed()
658}
659
660fn empty() -> BoxBody<Bytes, BoxError> {
661    Empty::<Bytes>::new()
662        .map_err(|never| match never {})
663        .boxed()
664}
665
666/// Engine for an ad-hoc `.eval` script: the prepared base (Stream reads +
667/// Direct `.append`) plus the VFS modules registered so far, so eval scripts
668/// can `use` them, the same builtins the runners get.
669fn eval_engine(store: &Store) -> Result<nu::Engine, String> {
670    let mut engine = nu::prepared_base(store, true)
671        .map_err(|e| format!("Failed to build nushell engine: {e}"))?;
672    // Modules registered up to now: a fresh time-ordered id exceeds every
673    // already-appended frame.
674    let modules = store.nu_modules_at(&scru128::new());
675    nu::load_modules(&mut engine.state, store, &modules)
676        .map_err(|e| format!("Failed to load modules: {e}"))?;
677    Ok(engine)
678}
679
680async fn handle_eval(store: &Store, body: hyper::body::Incoming) -> HTTPResult {
681    // Read the script from the request body
682    let bytes = body.collect().await?.to_bytes();
683    let script =
684        String::from_utf8(bytes.to_vec()).map_err(|e| format!("Invalid UTF-8 in script: {e}"))?;
685
686    let engine = eval_engine(store)?;
687
688    // Execute the script on a dedicated thread, not on this tokio runtime
689    // thread. handle_eval is an async handler, so engine.eval would otherwise
690    // run on a runtime thread. The script can call .cat/.last, whose historical
691    // path parks on tokio's blocking_recv, which panics ("Cannot block the
692    // current thread from within a runtime") on a runtime thread. Running the
693    // eval on a plain std::thread avoids that.
694    let result = std::thread::scope(|scope| {
695        scope
696            .spawn(|| engine.eval(nu_protocol::PipelineData::empty(), script))
697            .join()
698            .expect("eval thread panicked")
699    })
700    .map_err(|e| format!("Script evaluation failed:\n{e}"))?;
701
702    // Format output based on PipelineData type according to spec
703    match result {
704        nu_protocol::PipelineData::ByteStream(stream, ..) => {
705            // ByteStream → raw bytes with proper streaming using channel pattern
706            if let Some(mut reader) = stream.reader() {
707                use std::io::Read;
708
709                let (tx, rx) = tokio::sync::mpsc::channel(16);
710
711                // Spawn sync task to read from nushell Reader and send to channel
712                std::thread::spawn(move || {
713                    let mut buffer = [0u8; 8192];
714                    loop {
715                        match reader.read(&mut buffer) {
716                            Ok(0) => break, // EOF
717                            Ok(n) => {
718                                let chunk = Bytes::copy_from_slice(&buffer[..n]);
719                                if tx
720                                    .blocking_send(Ok(hyper::body::Frame::data(chunk)))
721                                    .is_err()
722                                {
723                                    break;
724                                }
725                            }
726                            Err(e) => {
727                                let _ = tx.blocking_send(Err(
728                                    Box::new(e) as Box<dyn std::error::Error + Send + Sync>
729                                ));
730                                break;
731                            }
732                        }
733                    }
734                });
735
736                let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
737                let body = StreamBody::new(stream).boxed();
738                Ok(Response::builder()
739                    .status(StatusCode::OK)
740                    .header("Content-Type", "application/octet-stream")
741                    .body(body)?)
742            } else {
743                // No reader available, return empty response
744                Ok(Response::builder()
745                    .status(StatusCode::OK)
746                    .header("Content-Type", "application/octet-stream")
747                    .body(empty())?)
748            }
749        }
750        nu_protocol::PipelineData::ListStream(stream, ..) => {
751            // ListStream → JSONL stream with proper streaming using channel pattern
752            let (tx, rx) = tokio::sync::mpsc::channel(16);
753
754            // Spawn sync task to iterate stream and send JSONL to channel
755            std::thread::spawn(move || {
756                for value in stream.into_iter() {
757                    let json = nu::value_to_json(&value);
758                    match serde_json::to_vec(&json) {
759                        Ok(mut json_bytes) => {
760                            json_bytes.push(b'\n'); // Add newline for JSONL
761                            let chunk = Bytes::from(json_bytes);
762                            if tx
763                                .blocking_send(Ok(hyper::body::Frame::data(chunk)))
764                                .is_err()
765                            {
766                                break;
767                            }
768                        }
769                        Err(e) => {
770                            let _ = tx.blocking_send(Err(
771                                Box::new(e) as Box<dyn std::error::Error + Send + Sync>
772                            ));
773                            break;
774                        }
775                    }
776                }
777            });
778
779            let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
780            let body = StreamBody::new(stream).boxed();
781            Ok(Response::builder()
782                .status(StatusCode::OK)
783                .header("Content-Type", "application/x-ndjson")
784                .body(body)?)
785        }
786        nu_protocol::PipelineData::Value(value, ..) => {
787            match &value {
788                nu_protocol::Value::String { .. }
789                | nu_protocol::Value::Int { .. }
790                | nu_protocol::Value::Float { .. }
791                | nu_protocol::Value::Bool { .. } => {
792                    // Single primitive value → raw text
793                    let text = match value {
794                        nu_protocol::Value::String { val, .. } => val.clone(),
795                        nu_protocol::Value::Int { val, .. } => val.to_string(),
796                        nu_protocol::Value::Float { val, .. } => val.to_string(),
797                        nu_protocol::Value::Bool { val, .. } => val.to_string(),
798                        _ => value.into_string().unwrap_or_else(|_| "".to_string()),
799                    };
800                    Ok(Response::builder()
801                        .status(StatusCode::OK)
802                        .header("Content-Type", "text/plain")
803                        .body(full(text))?)
804                }
805                _ => {
806                    // Single structured value → JSON
807                    let json = nu::value_to_json(&value);
808                    let json_string = serde_json::to_string(&json)
809                        .map_err(|e| format!("Failed to serialize JSON: {e}"))?;
810                    Ok(Response::builder()
811                        .status(StatusCode::OK)
812                        .header("Content-Type", "application/json")
813                        .body(full(json_string))?)
814                }
815            }
816        }
817        nu_protocol::PipelineData::Empty => {
818            // Empty → nothing
819            Ok(Response::builder()
820                .status(StatusCode::NO_CONTENT)
821                .body(empty())?)
822        }
823    }
824}
825
826#[cfg(test)]
827mod tests {
828    use super::*;
829
830    #[tokio::test]
831    async fn test_handle_stream_append_body() {
832        let temp_dir = tempfile::tempdir().unwrap();
833        let mut store = Store::new(temp_dir.path().to_path_buf()).unwrap();
834
835        async fn append(
836            store: &mut Store,
837            meta: Option<&str>,
838            chunks: Vec<&'static [u8]>,
839        ) -> serde_json::Value {
840            let stream = tokio_stream::iter(chunks.into_iter().map(|chunk| {
841                Ok::<_, BoxError>(hyper::body::Frame::data(Bytes::from_static(chunk)))
842            }));
843            let mut req = Request::post("/append/note");
844            if let Some(meta) = meta {
845                req = req.header("xs-meta", base64::prelude::BASE64_STANDARD.encode(meta));
846            }
847            let req = req.body(StreamBody::new(stream)).unwrap();
848            let response = handle_stream_append(store, req, "note".into(), None, false)
849                .await
850                .unwrap();
851            assert_eq!(StatusCode::OK, response.status());
852            let body = response.into_body().collect().await.unwrap().to_bytes();
853            serde_json::from_slice(&body).unwrap()
854        }
855
856        // No body, or only empty chunks: the frame is stored with no hash.
857        let frame = append(&mut store, None, vec![]).await;
858        assert!(frame["hash"].is_null());
859        let frame = append(&mut store, None, vec![b"", b""]).await;
860        assert!(frame["hash"].is_null());
861
862        // A body split across chunks, with empty chunks around it, lands
863        // whole in the CAS.
864        let frame = append(&mut store, None, vec![b"", b"hello ", b"", b"world"]).await;
865        let hash: ssri::Integrity = frame["hash"].as_str().unwrap().parse().unwrap();
866        assert_eq!(
867            b"hello world".to_vec(),
868            store.cas_read(&hash).await.unwrap()
869        );
870
871        // Meta and a body together.
872        let frame = append(&mut store, Some(r#"{"n":1}"#), vec![b"x"]).await;
873        assert_eq!(serde_json::json!({"n": 1}), frame["meta"]);
874        let hash: ssri::Integrity = frame["hash"].as_str().unwrap().parse().unwrap();
875        assert_eq!(b"x".to_vec(), store.cas_read(&hash).await.unwrap());
876    }
877
878    #[test]
879    fn test_match_route_last() {
880        let headers = hyper::HeaderMap::new();
881
882        // /last/topic without follow
883        assert!(matches!(
884            match_route(&Method::GET, "/last/test", &headers, None),
885            Routes::LastGet { topic: Some(t), last: 1, follow: false, .. } if t == "test"
886        ));
887
888        // /last/topic with follow
889        assert!(matches!(
890            match_route(&Method::GET, "/last/test", &headers, Some("follow=true")),
891            Routes::LastGet { topic: Some(t), last: 1, follow: true, .. } if t == "test"
892        ));
893
894        // /last without topic
895        assert!(matches!(
896            match_route(&Method::GET, "/last", &headers, None),
897            Routes::LastGet {
898                topic: None,
899                last: 1,
900                follow: false,
901                ..
902            }
903        ));
904
905        // /last with last=5
906        assert!(matches!(
907            match_route(&Method::GET, "/last", &headers, Some("last=5")),
908            Routes::LastGet {
909                topic: None,
910                last: 5,
911                follow: false,
912                ..
913            }
914        ));
915
916        // /last/topic with last=3 and follow
917        assert!(matches!(
918            match_route(&Method::GET, "/last/test", &headers, Some("last=3&follow=true")),
919            Routes::LastGet { topic: Some(t), last: 3, follow: true, .. } if t == "test"
920        ));
921    }
922
923    #[tokio::test]
924    async fn test_handle_last_get_ndjson_format() {
925        use crate::store::Store;
926        use http_body_util::BodyExt;
927
928        let temp_dir = tempfile::tempdir().unwrap();
929        let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
930
931        // Add one frame
932        store
933            .append(crate::store::Frame::builder("test").build())
934            .unwrap();
935
936        // Request last=1 (default) - should return JSON object (no trailing newline)
937        let response = handle_last_get(&store, Some("test"), 1, false, false)
938            .await
939            .unwrap();
940        let body = response.into_body().collect().await.unwrap().to_bytes();
941        let body_str = String::from_utf8(body.to_vec()).unwrap();
942        assert!(
943            !body_str.ends_with('\n'),
944            "Response should be JSON (no trailing newline) when last=1, got: {:?}",
945            body_str
946        );
947
948        // Request last=3 but only 1 frame exists - should still return NDJSON
949        let response = handle_last_get(&store, Some("test"), 3, false, false)
950            .await
951            .unwrap();
952        let body = response.into_body().collect().await.unwrap().to_bytes();
953        let body_str = String::from_utf8(body.to_vec()).unwrap();
954        assert!(
955            body_str.ends_with('\n'),
956            "Response should be NDJSON (end with newline) when last > 1, got: {:?}",
957            body_str
958        );
959
960        // Verify full chain: match_route -> dispatch -> handle_last_get
961        let headers = hyper::HeaderMap::new();
962        let route = match_route(&Method::GET, "/last/test", &headers, Some("last=3"));
963        if let Routes::LastGet {
964            topic,
965            last,
966            follow,
967            with_timestamp,
968        } = route
969        {
970            assert_eq!(last, 3, "Route should parse last=3");
971            let response = handle_last_get(&store, topic.as_deref(), last, follow, with_timestamp)
972                .await
973                .unwrap();
974            let body = response.into_body().collect().await.unwrap().to_bytes();
975            let body_str = String::from_utf8(body.to_vec()).unwrap();
976            assert!(
977                body_str.ends_with('\n'),
978                "Full chain should return NDJSON when last=3, got: {:?}",
979                body_str
980            );
981        } else {
982            panic!("Expected Routes::LastGet");
983        }
984    }
985
986    #[tokio::test]
987    async fn test_handle_eval_logic() {
988        // Test the core nushell execution logic by testing the engine directly
989        use crate::nu::Engine;
990        use crate::store::Store;
991        use nu_protocol::PipelineData;
992
993        // Create a temporary store for testing
994        let temp_dir = tempfile::tempdir().unwrap();
995        let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
996
997        // Create nushell engine with store helper commands
998        let mut engine = Engine::new().unwrap();
999
1000        // Add core commands
1001        crate::nu::add_core_commands(&mut engine, &store).unwrap();
1002
1003        // Add streaming commands
1004        engine
1005            .add_commands(vec![
1006                Box::new(crate::nu::commands::cat_command::CatCommand::new(
1007                    store.clone(),
1008                )),
1009                Box::new(crate::nu::commands::last_command::LastCommand::new(
1010                    store.clone(),
1011                )),
1012                Box::new(crate::nu::commands::append_command::AppendCommand::new(
1013                    store.clone(),
1014                )),
1015            ])
1016            .unwrap();
1017
1018        // Test simple string expression
1019        let result = engine
1020            .eval(PipelineData::empty(), r#""hello world""#.to_string())
1021            .unwrap();
1022
1023        match result {
1024            PipelineData::Value(value, ..) => {
1025                let text = value.into_string().unwrap();
1026                assert_eq!(text, "hello world");
1027            }
1028            _ => panic!("Expected Value, got {:?}", result),
1029        }
1030
1031        // Test simple math expression - result should be an integer
1032        let result = engine
1033            .eval(PipelineData::empty(), "2 + 3".to_string())
1034            .unwrap();
1035
1036        match result {
1037            PipelineData::Value(nu_protocol::Value::Int { val, .. }, ..) => {
1038                assert_eq!(val, 5);
1039            }
1040            _ => panic!("Expected Int Value, got {:?}", result),
1041        }
1042    }
1043
1044    // An `.eval` script must be able to `use` a registered VFS module.
1045    #[tokio::test]
1046    async fn test_eval_uses_modules() {
1047        use crate::store::{Frame, Store};
1048        use nu_protocol::PipelineData;
1049
1050        let temp_dir = tempfile::tempdir().unwrap();
1051        let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1052
1053        let hash = store
1054            .cas_insert_sync(r#"export def add_nums [x, y] { $"sum is ($x + $y)" }"#)
1055            .unwrap();
1056        store
1057            .append(Frame::builder("xs.module.mymod").hash(hash).build())
1058            .unwrap();
1059
1060        let engine = super::eval_engine(&store).unwrap();
1061        let result = engine
1062            .eval(
1063                PipelineData::empty(),
1064                "use mymod; mymod add_nums 40 2".to_string(),
1065            )
1066            .unwrap()
1067            .into_value(nu_protocol::Span::test_data())
1068            .unwrap();
1069        assert_eq!(result.into_string().unwrap(), "sum is 42");
1070    }
1071
1072    // A module's exported function must be able to call the store builtins (here
1073    // `.append`), not just pure nushell. (rokf94's report; the engine carries
1074    // the builtins before modules load.)
1075    #[tokio::test]
1076    async fn test_module_can_use_append_builtin() {
1077        use crate::store::{Frame, Store};
1078        use nu_protocol::{PipelineData, Span};
1079
1080        let temp_dir = tempfile::tempdir().unwrap();
1081        let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1082
1083        let hash = store
1084            .cas_insert_sync(r#"export def emit [] { {} | .append out --meta {via: "module"} }"#)
1085            .unwrap();
1086        store
1087            .append(Frame::builder("xs.module.emitter").hash(hash).build())
1088            .unwrap();
1089
1090        let engine = super::eval_engine(&store).unwrap();
1091        let value = engine
1092            .eval(
1093                PipelineData::empty(),
1094                "use emitter; emitter emit".to_string(),
1095            )
1096            .unwrap()
1097            .into_value(Span::test_data())
1098            .unwrap();
1099
1100        // `.append` ran inside the module and returned the appended frame.
1101        let json = crate::nu::value_to_json(&value);
1102        assert_eq!(json["topic"], "out");
1103        assert_eq!(json["meta"]["via"], "module");
1104    }
1105}