1use std::collections::HashMap;
2use std::error::Error;
3use std::str::FromStr;
4
5use chrono::{DateTime, Utc};
6use scru128::Scru128Id;
7
8use base64::Engine;
9
10use tokio::io::AsyncWriteExt;
11use tokio_stream::wrappers::ReceiverStream;
12use tokio_stream::StreamExt;
13use tokio_util::io::ReaderStream;
14
15use http_body_util::StreamBody;
16use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full};
17use hyper::body::Bytes;
18use hyper::header::ACCEPT;
19use hyper::server::conn::http1;
20use hyper::service::service_fn;
21use hyper::{Method, Request, Response, StatusCode};
22use hyper_util::rt::TokioIo;
23
24use crate::listener::Listener;
25use crate::nu;
26use crate::store::{self, FollowOption, Frame, ReadOptions, Store, TTL};
27
28type HTTPResult = Result<Response<BoxBody<Bytes, crate::Error>>, crate::Error>;
29
30fn serialize_frame(frame: &Frame, with_timestamp: bool) -> String {
32 if with_timestamp {
33 let mut value = serde_json::to_value(frame).unwrap();
34 if let serde_json::Value::Object(ref mut map) = value {
35 let millis = frame.id.timestamp() as i64;
36 let dt: DateTime<Utc> = DateTime::from_timestamp_millis(millis)
37 .unwrap_or_else(|| DateTime::from_timestamp(0, 0).unwrap());
38 map.insert("timestamp".to_string(), serde_json::json!(dt.to_rfc3339()));
39 }
40 serde_json::to_string(&value).unwrap()
41 } else {
42 serde_json::to_string(frame).unwrap()
43 }
44}
45
46#[derive(Debug, PartialEq, Clone)]
47enum AcceptType {
48 Ndjson,
49 EventStream,
50}
51
52enum Routes {
53 StreamCat {
54 accept_type: AcceptType,
55 options: ReadOptions,
56 with_timestamp: bool,
57 },
58 StreamAppend {
59 topic: String,
60 ttl: Option<TTL>,
61 with_timestamp: bool,
62 },
63 LastGet {
64 topic: Option<String>,
65 last: usize,
66 follow: bool,
67 with_timestamp: bool,
68 },
69 StreamItemGet {
70 id: Scru128Id,
71 with_timestamp: bool,
72 },
73 StreamItemRemove(Scru128Id),
74 CasGet(ssri::Integrity),
75 CasPost,
76 Import,
77 Eval,
78 Version,
79 Metrics,
80 NotFound,
81 BadRequest(String),
82}
83
84fn validate_integrity(integrity: &ssri::Integrity) -> bool {
86 if integrity.hashes.is_empty() {
88 return false;
89 }
90
91 for hash in &integrity.hashes {
93 if base64::engine::general_purpose::STANDARD
95 .decode(&hash.digest)
96 .is_err()
97 {
98 return false;
99 }
100 }
101
102 true
103}
104
105fn match_route(
106 method: &Method,
107 path: &str,
108 headers: &hyper::HeaderMap,
109 query: Option<&str>,
110) -> Routes {
111 let params: HashMap<String, String> =
112 url::form_urlencoded::parse(query.unwrap_or("").as_bytes())
113 .into_owned()
114 .collect();
115
116 match (method, path) {
117 (&Method::GET, "/version") => Routes::Version,
118 (&Method::GET, "/metrics") => Routes::Metrics,
119
120 (&Method::GET, "/") => {
121 let accept_type = match headers.get(ACCEPT) {
122 Some(accept) if accept == "text/event-stream" => AcceptType::EventStream,
123 _ => AcceptType::Ndjson,
124 };
125
126 let options = ReadOptions::from_query(query);
127 let with_timestamp = params.contains_key("with-timestamp");
128
129 match options {
130 Ok(options) => Routes::StreamCat {
131 accept_type,
132 options,
133 with_timestamp,
134 },
135 Err(e) => Routes::BadRequest(e.to_string()),
136 }
137 }
138
139 (&Method::GET, "/last") => {
140 let follow = params.contains_key("follow");
141 let last = params.get("last").and_then(|v| v.parse().ok()).unwrap_or(1);
142 let with_timestamp = params.contains_key("with-timestamp");
143 Routes::LastGet {
144 topic: None,
145 last,
146 follow,
147 with_timestamp,
148 }
149 }
150
151 (&Method::GET, p) if p.starts_with("/last/") => {
152 let topic = p.strip_prefix("/last/").unwrap().to_string();
153 let follow = params.contains_key("follow");
154 let last = params.get("last").and_then(|v| v.parse().ok()).unwrap_or(1);
155 let with_timestamp = params.contains_key("with-timestamp");
156 Routes::LastGet {
157 topic: Some(topic),
158 last,
159 follow,
160 with_timestamp,
161 }
162 }
163
164 (&Method::GET, p) if p.starts_with("/cas/") => {
165 if let Some(hash) = p.strip_prefix("/cas/") {
166 match ssri::Integrity::from_str(hash) {
167 Ok(integrity) => {
168 if validate_integrity(&integrity) {
169 Routes::CasGet(integrity)
170 } else {
171 Routes::BadRequest(format!("Invalid CAS hash format: {hash}"))
172 }
173 }
174 Err(e) => Routes::BadRequest(format!("Invalid CAS hash: {e}")),
175 }
176 } else {
177 Routes::NotFound
178 }
179 }
180
181 (&Method::POST, "/cas") => Routes::CasPost,
182 (&Method::POST, "/import") => Routes::Import,
183 (&Method::POST, "/eval") => Routes::Eval,
184
185 (&Method::GET, p) => match Scru128Id::from_str(p.trim_start_matches('/')) {
186 Ok(id) => {
187 let with_timestamp = params.contains_key("with-timestamp");
188 Routes::StreamItemGet { id, with_timestamp }
189 }
190 Err(e) => Routes::BadRequest(format!("Invalid frame ID: {e}")),
191 },
192
193 (&Method::DELETE, p) => match Scru128Id::from_str(p.trim_start_matches('/')) {
194 Ok(id) => Routes::StreamItemRemove(id),
195 Err(e) => Routes::BadRequest(format!("Invalid frame ID: {e}")),
196 },
197
198 (&Method::POST, path) if path.starts_with("/append/") => {
199 let topic = path.strip_prefix("/append/").unwrap().to_string();
200 let with_timestamp = params.contains_key("with-timestamp");
201
202 match TTL::from_query(query) {
203 Ok(ttl) => Routes::StreamAppend {
204 topic,
205 ttl: Some(ttl),
206 with_timestamp,
207 },
208 Err(e) => Routes::BadRequest(e.to_string()),
209 }
210 }
211
212 _ => Routes::NotFound,
213 }
214}
215
216async fn handle(
217 mut store: Store,
218 _engine: nu::Engine, req: Request<hyper::body::Incoming>,
220) -> HTTPResult {
221 let method = req.method();
222 let path = req.uri().path();
223 let headers = req.headers().clone();
224 let query = req.uri().query();
225
226 let res = match match_route(method, path, &headers, query) {
227 Routes::Version => handle_version().await,
228 Routes::Metrics => handle_metrics(&store).await,
229
230 Routes::StreamCat {
231 accept_type,
232 options,
233 with_timestamp,
234 } => handle_stream_cat(&mut store, options, accept_type, with_timestamp).await,
235
236 Routes::StreamAppend {
237 topic,
238 ttl,
239 with_timestamp,
240 } => handle_stream_append(&mut store, req, topic, ttl, with_timestamp).await,
241
242 Routes::CasGet(hash) => {
243 let reader = store.cas_reader(hash).await?;
244 let stream = ReaderStream::new(reader);
245
246 let stream = stream.map(|frame| {
247 let frame = frame.unwrap();
248 Ok(hyper::body::Frame::data(frame))
249 });
250
251 let body = StreamBody::new(stream).boxed();
252 Ok(Response::new(body))
253 }
254
255 Routes::CasPost => handle_cas_post(&mut store, req.into_body()).await,
256
257 Routes::StreamItemGet { id, with_timestamp } => {
258 response_frame_or_404(store.get(&id), with_timestamp)
259 }
260
261 Routes::StreamItemRemove(id) => handle_stream_item_remove(&mut store, id).await,
262
263 Routes::LastGet {
264 topic,
265 last,
266 follow,
267 with_timestamp,
268 } => handle_last_get(&store, topic.as_deref(), last, follow, with_timestamp).await,
269
270 Routes::Import => handle_import(&mut store, req.into_body()).await,
271
272 Routes::Eval => handle_eval(&store, req.into_body()).await,
273
274 Routes::NotFound => response_404(),
275 Routes::BadRequest(msg) => response_400(msg),
276 };
277
278 res.or_else(|e| response_500(e.to_string()))
279}
280
281pub const LAGGED_TRAILER: &str = "xs-lagged";
284
285pub const ERROR_TRAILER: &str = "xs-error";
287
288fn error_text(error: &nu_protocol::ShellError) -> String {
291 match error {
292 nu_protocol::ShellError::Generic(g) if g.msg.is_empty() => g.error.to_string(),
293 nu_protocol::ShellError::Generic(g) => format!("{}: {}", g.error, g.msg),
294 nu_protocol::ShellError::EvalBlockWithInput { sources, .. } if !sources.is_empty() => {
295 error_text(&sources[0])
296 }
297 other => other.to_string(),
298 }
299}
300
301fn header_text(text: &str) -> hyper::header::HeaderValue {
303 let clean: String = text
304 .chars()
305 .map(|c| if c.is_ascii_graphic() { c } else { ' ' })
306 .collect();
307 hyper::header::HeaderValue::from_str(clean.trim())
308 .unwrap_or(hyper::header::HeaderValue::from_static("error"))
309}
310
311fn body_frames(
315 item: crate::store::ReadItem,
316 encode: &impl Fn(&Frame) -> Vec<u8>,
317) -> Vec<hyper::body::Frame<Bytes>> {
318 match item {
319 Ok(frame) => vec![hyper::body::Frame::data(Bytes::from(encode(&frame)))],
320 Err(err) => {
321 let crate::store::ReadError::Lagged { skipped } = &err;
322 let mut trailers = hyper::HeaderMap::new();
323 trailers.insert(LAGGED_TRAILER, (*skipped).into());
324 vec![
325 hyper::body::Frame::data(Bytes::from(encode(&err.to_frame()))),
326 hyper::body::Frame::trailers(trailers),
327 ]
328 }
329 }
330}
331
332async fn handle_stream_cat(
333 store: &mut Store,
334 options: ReadOptions,
335 accept_type: AcceptType,
336 with_timestamp: bool,
337) -> HTTPResult {
338 let rx = store.read(options);
339
340 let accept_type_clone = accept_type.clone();
341 let encode = move |frame: &Frame| match accept_type_clone {
342 AcceptType::Ndjson => {
343 let mut encoded = serialize_frame(frame, with_timestamp).into_bytes();
344 encoded.push(b'\n');
345 encoded
346 }
347 AcceptType::EventStream => format!(
348 "id: {id}\ndata: {data}\n\n",
349 id = frame.id,
350 data = serialize_frame(frame, with_timestamp)
351 )
352 .into_bytes(),
353 };
354 let stream = futures::StreamExt::flat_map(ReceiverStream::new(rx), move |item| {
355 futures::stream::iter(body_frames(item, &encode))
356 })
357 .map(Ok::<_, crate::Error>);
358
359 let body = StreamBody::new(stream).boxed();
360
361 let content_type = match accept_type {
362 AcceptType::Ndjson => "application/x-ndjson",
363 AcceptType::EventStream => "text/event-stream",
364 };
365
366 Ok(Response::builder()
367 .status(StatusCode::OK)
368 .header("Content-Type", content_type)
369 .header("Trailer", LAGGED_TRAILER)
370 .body(body)?)
371}
372
373async fn handle_stream_append<B>(
374 store: &mut Store,
375 req: Request<B>,
376 topic: String,
377 ttl: Option<TTL>,
378 with_timestamp: bool,
379) -> HTTPResult
380where
381 B: hyper::body::Body<Data = Bytes> + Unpin,
382 B::Error: Into<crate::Error>,
383{
384 let (parts, mut body) = req.into_parts();
385
386 let hash = {
387 let mut writer = None;
390
391 while let Some(frame) = body.frame().await {
392 if let Ok(data) = frame.map_err(Into::into)?.into_data() {
393 if data.is_empty() {
394 continue;
395 }
396 if writer.is_none() {
397 writer = Some(store.cas_writer().await?);
398 }
399 writer.as_mut().unwrap().write_all(&data).await?;
400 }
401 }
402
403 match writer {
404 Some(writer) => Some(writer.commit().await?),
405 None => None,
406 }
407 };
408
409 let meta = match parts
410 .headers
411 .get("xs-meta")
412 .map(|x| x.to_str())
413 .transpose()
414 .unwrap()
415 .map(|s| {
416 base64::prelude::BASE64_STANDARD
418 .decode(s)
419 .map_err(|e| format!("xs-meta isn't valid Base64: {e}"))
420 .and_then(|decoded| {
421 String::from_utf8(decoded)
423 .map_err(|e| format!("xs-meta isn't valid UTF-8: {e}"))
424 .and_then(|json_str| {
425 serde_json::from_str(&json_str)
427 .map_err(|e| format!("xs-meta isn't valid JSON: {e}"))
428 })
429 })
430 })
431 .transpose()
432 {
433 Ok(meta) => meta,
434 Err(e) => return response_400(e.to_string()),
435 };
436
437 let frame = store.append(
438 Frame::builder(topic)
439 .maybe_hash(hash)
440 .maybe_meta(meta)
441 .maybe_ttl(ttl)
442 .build(),
443 )?;
444
445 Ok(Response::builder()
446 .status(StatusCode::OK)
447 .header("Content-Type", "application/json")
448 .body(full(serialize_frame(&frame, with_timestamp)))?)
449}
450
451async fn handle_cas_post(store: &mut Store, mut body: hyper::body::Incoming) -> HTTPResult {
452 let hash = {
453 let mut writer = store.cas_writer().await?;
454 let mut bytes_written = 0;
455
456 while let Some(frame) = body.frame().await {
457 if let Ok(data) = frame?.into_data() {
458 writer.write_all(&data).await?;
459 bytes_written += data.len();
460 }
461 }
462
463 if bytes_written == 0 {
464 return response_400("Empty body".to_string());
465 }
466
467 writer.commit().await?
468 };
469
470 Ok(Response::builder()
471 .status(StatusCode::OK)
472 .header("Content-Type", "text/plain")
473 .body(full(hash.to_string()))?)
474}
475
476async fn handle_version() -> HTTPResult {
477 let version = env!("CARGO_PKG_VERSION");
478 let version_info = serde_json::json!({ "version": version });
479 Ok(Response::builder()
480 .status(StatusCode::OK)
481 .header("Content-Type", "application/json")
482 .body(full(serde_json::to_string(&version_info).unwrap()))?)
483}
484
485async fn handle_metrics(store: &Store) -> HTTPResult {
495 let body = serde_json::json!({
496 "store": store.stats(),
497 "keyspaces": store.metrics(),
498 });
499 Ok(Response::builder()
500 .status(StatusCode::OK)
501 .header("Content-Type", "application/json")
502 .body(full(serde_json::to_string(&body).unwrap()))?)
503}
504
505pub async fn serve(
506 store: Store,
507 engine: nu::Engine,
508 expose: Option<String>,
509) -> Result<(), crate::Error> {
510 let path = store.path.join("sock").to_string_lossy().to_string();
511 let listener = Listener::bind(&path).await?;
512
513 let mut listeners = vec![listener];
514 let mut expose_meta = None;
515
516 if let Some(expose) = expose {
517 let expose_listener = Listener::bind(&expose).await?;
518
519 if let Some(ticket) = expose_listener.get_ticket() {
521 expose_meta = Some(serde_json::json!({"expose": format!("iroh://{}", ticket)}));
522 } else {
523 expose_meta = Some(serde_json::json!({"expose": expose}));
524 }
525
526 listeners.push(expose_listener);
527 }
528
529 if let Err(e) = store.append(Frame::builder("xs.start").maybe_meta(expose_meta).build()) {
530 tracing::error!("Failed to append xs.start frame: {}", e);
531 }
532
533 let mut tasks = Vec::new();
534 for listener in listeners {
535 let store = store.clone();
536 let engine = engine.clone();
537 let task = tokio::spawn(async move { listener_loop(listener, store, engine).await });
538 tasks.push(task);
539 }
540
541 for task in tasks {
544 task.await??;
545 }
546
547 Ok(())
548}
549
550async fn listener_loop(
551 mut listener: Listener,
552 store: Store,
553 engine: nu::Engine,
554) -> Result<(), crate::Error> {
555 loop {
556 let (stream, _) = listener.accept().await?;
557 let io = TokioIo::new(stream);
558 let store = store.clone();
559 let engine = engine.clone();
560 tokio::task::spawn(async move {
561 if let Err(err) = http1::Builder::new()
562 .serve_connection(
563 io,
564 service_fn(move |req| handle(store.clone(), engine.clone(), req)),
565 )
566 .await
567 {
568 if let Some(std::io::ErrorKind::NotConnected) = err.source().and_then(|source| {
570 source
571 .downcast_ref::<std::io::Error>()
572 .map(|io_err| io_err.kind())
573 }) {
574 } else {
576 tracing::error!("TBD: {:?}", err);
578 }
579 }
580 });
581 }
582}
583
584fn response_frame_or_404(frame: Option<store::Frame>, with_timestamp: bool) -> HTTPResult {
585 if let Some(frame) = frame {
586 Ok(Response::builder()
587 .status(StatusCode::OK)
588 .header("Content-Type", "application/json")
589 .body(full(serialize_frame(&frame, with_timestamp)))?)
590 } else {
591 response_404()
592 }
593}
594
595async fn handle_stream_item_remove(store: &mut Store, id: Scru128Id) -> HTTPResult {
596 match store.remove(&id) {
597 Ok(()) => Ok(Response::builder()
598 .status(StatusCode::NO_CONTENT)
599 .body(empty())?),
600 Err(e) => {
601 tracing::error!("Failed to remove item {}: {:?}", id, e);
602
603 Ok(Response::builder()
604 .status(StatusCode::INTERNAL_SERVER_ERROR)
605 .body(full("internal-error"))?)
606 }
607 }
608}
609
610async fn handle_last_get(
611 store: &Store,
612 topic: Option<&str>,
613 last: usize,
614 follow: bool,
615 with_timestamp: bool,
616) -> HTTPResult {
617 if !follow {
618 let options = ReadOptions::builder()
619 .last(last)
620 .maybe_topic(topic.map(|t| t.to_string()))
621 .build();
622
623 let frames: Vec<Frame> = store.read_sync(options).collect();
624
625 if frames.is_empty() {
626 return response_404();
627 }
628
629 if last == 1 {
633 return response_frame_or_404(Some(frames.into_iter().next().unwrap()), with_timestamp);
634 }
635
636 let mut body = Vec::new();
637 for frame in frames {
638 body.extend(serialize_frame(&frame, with_timestamp).into_bytes());
639 body.push(b'\n');
640 }
641
642 return Ok(Response::builder()
643 .status(StatusCode::OK)
644 .header("Content-Type", "application/x-ndjson")
645 .body(full(body))?);
646 }
647
648 let rx = store.read(
651 ReadOptions::builder()
652 .last(last)
653 .maybe_topic(topic.map(|t| t.to_string()))
654 .follow(FollowOption::On)
655 .build(),
656 );
657
658 let encode = move |frame: &Frame| {
659 let mut bytes = serialize_frame(frame, with_timestamp).into_bytes();
660 bytes.push(b'\n');
661 bytes
662 };
663 let stream = futures::StreamExt::flat_map(ReceiverStream::new(rx), move |item| {
664 futures::stream::iter(body_frames(item, &encode))
665 })
666 .map(Ok::<_, crate::Error>);
667
668 Ok(Response::builder()
669 .status(StatusCode::OK)
670 .header("Content-Type", "application/x-ndjson")
671 .header("Trailer", LAGGED_TRAILER)
672 .body(StreamBody::new(stream).boxed())?)
673}
674
675async fn handle_import(store: &mut Store, body: hyper::body::Incoming) -> HTTPResult {
676 let bytes = body.collect().await?.to_bytes();
677 let frame: Frame = match serde_json::from_slice(&bytes) {
678 Ok(frame) => frame,
679 Err(e) => return response_400(format!("Invalid frame JSON: {e}")),
680 };
681
682 store.insert_frame(&frame)?;
683
684 Ok(Response::builder()
685 .status(StatusCode::OK)
686 .header("Content-Type", "application/json")
687 .body(full(serde_json::to_string(&frame).unwrap()))?)
688}
689
690fn response_404() -> HTTPResult {
691 Ok(Response::builder()
692 .status(StatusCode::NOT_FOUND)
693 .body(empty())?)
694}
695
696fn response_400(message: String) -> HTTPResult {
697 let body = full(message);
698 Ok(Response::builder()
699 .status(StatusCode::BAD_REQUEST)
700 .body(body)?)
701}
702
703fn response_500(message: String) -> HTTPResult {
704 let body = full(message);
705 Ok(Response::builder()
706 .status(StatusCode::INTERNAL_SERVER_ERROR)
707 .body(body)?)
708}
709
710fn full<T: Into<Bytes>>(chunk: T) -> BoxBody<Bytes, crate::Error> {
711 Full::new(chunk.into())
712 .map_err(|never| match never {})
713 .boxed()
714}
715
716fn empty() -> BoxBody<Bytes, crate::Error> {
717 Empty::<Bytes>::new()
718 .map_err(|never| match never {})
719 .boxed()
720}
721
722fn eval_engine(store: &Store) -> Result<nu::Engine, String> {
726 let mut engine = nu::prepared_base(store, true)
727 .map_err(|e| format!("Failed to build nushell engine: {e}"))?;
728 let modules = store.nu_modules_at(&scru128::new());
731 nu::load_modules(&mut engine.state, store, &modules)
732 .map_err(|e| format!("Failed to load modules: {e}"))?;
733 Ok(engine)
734}
735
736async fn handle_eval(store: &Store, body: hyper::body::Incoming) -> HTTPResult {
737 let bytes = body.collect().await?.to_bytes();
739 let script =
740 String::from_utf8(bytes.to_vec()).map_err(|e| format!("Invalid UTF-8 in script: {e}"))?;
741
742 let engine = eval_engine(store)?;
743
744 let result = std::thread::scope(|scope| {
751 scope
752 .spawn(|| engine.eval(nu_protocol::PipelineData::empty(), script))
753 .join()
754 .expect("eval thread panicked")
755 })
756 .map_err(|e| format!("Script evaluation failed:\n{e}"))?;
757
758 match result {
760 nu_protocol::PipelineData::ByteStream(stream, ..) => {
761 if let Some(mut reader) = stream.reader() {
763 use std::io::Read;
764
765 let (tx, rx) = tokio::sync::mpsc::channel(16);
766
767 std::thread::spawn(move || {
769 let mut buffer = [0u8; 8192];
770 loop {
771 match reader.read(&mut buffer) {
772 Ok(0) => break, Ok(n) => {
774 let chunk = Bytes::copy_from_slice(&buffer[..n]);
775 if tx
776 .blocking_send(Ok(hyper::body::Frame::data(chunk)))
777 .is_err()
778 {
779 break;
780 }
781 }
782 Err(e) => {
783 let _ = tx.blocking_send(Err(Box::new(e) as crate::Error));
784 break;
785 }
786 }
787 }
788 });
789
790 let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
791 let body = StreamBody::new(stream).boxed();
792 Ok(Response::builder()
793 .status(StatusCode::OK)
794 .header("Content-Type", "application/octet-stream")
795 .body(body)?)
796 } else {
797 Ok(Response::builder()
799 .status(StatusCode::OK)
800 .header("Content-Type", "application/octet-stream")
801 .body(empty())?)
802 }
803 }
804 nu_protocol::PipelineData::ListStream(stream, ..) => {
805 let (tx, rx) = tokio::sync::mpsc::channel(16);
807
808 std::thread::spawn(move || {
810 for value in stream.into_iter() {
811 if let nu_protocol::Value::Error { error, .. } = &value {
815 let mut trailers = hyper::HeaderMap::new();
816 let text = error_text(error);
817 trailers.insert(ERROR_TRAILER, header_text(&text));
818 let _ = tx.blocking_send(Ok(hyper::body::Frame::trailers(trailers)));
819 break;
820 }
821 let json = nu::value_to_json(&value);
822 match serde_json::to_vec(&json) {
823 Ok(mut json_bytes) => {
824 json_bytes.push(b'\n'); let chunk = Bytes::from(json_bytes);
826 if tx
827 .blocking_send(Ok(hyper::body::Frame::data(chunk)))
828 .is_err()
829 {
830 break;
831 }
832 }
833 Err(e) => {
834 let _ = tx.blocking_send(Err(Box::new(e) as crate::Error));
835 break;
836 }
837 }
838 }
839 });
840
841 let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
842 let body = StreamBody::new(stream).boxed();
843 Ok(Response::builder()
844 .status(StatusCode::OK)
845 .header("Content-Type", "application/x-ndjson")
846 .header("Trailer", ERROR_TRAILER)
847 .body(body)?)
848 }
849 nu_protocol::PipelineData::Value(value, ..) => {
850 match &value {
851 nu_protocol::Value::String { .. }
852 | nu_protocol::Value::Int { .. }
853 | nu_protocol::Value::Float { .. }
854 | nu_protocol::Value::Bool { .. } => {
855 let text = match value {
857 nu_protocol::Value::String { val, .. } => val.clone(),
858 nu_protocol::Value::Int { val, .. } => val.to_string(),
859 nu_protocol::Value::Float { val, .. } => val.to_string(),
860 nu_protocol::Value::Bool { val, .. } => val.to_string(),
861 _ => value.into_string().unwrap_or_else(|_| "".to_string()),
862 };
863 Ok(Response::builder()
864 .status(StatusCode::OK)
865 .header("Content-Type", "text/plain")
866 .body(full(text))?)
867 }
868 _ => {
869 let json = nu::value_to_json(&value);
871 let json_string = serde_json::to_string(&json)
872 .map_err(|e| format!("Failed to serialize JSON: {e}"))?;
873 Ok(Response::builder()
874 .status(StatusCode::OK)
875 .header("Content-Type", "application/json")
876 .body(full(json_string))?)
877 }
878 }
879 }
880 nu_protocol::PipelineData::Empty => {
881 Ok(Response::builder()
883 .status(StatusCode::NO_CONTENT)
884 .body(empty())?)
885 }
886 }
887}
888
889#[cfg(test)]
890mod tests {
891 use super::*;
892
893 #[test]
894 fn a_lag_ends_with_the_line_then_the_trailer() {
895 let encode = |frame: &Frame| frame.topic.clone().into_bytes();
896 let frames = body_frames(
897 Err(crate::store::ReadError::Lagged { skipped: 506 }),
898 &encode,
899 );
900 assert_eq!(frames.len(), 2);
901 assert_eq!(frames[0].data_ref().unwrap().as_ref(), b"xs.lagged");
902 let trailers = frames[1].trailers_ref().unwrap();
903 assert_eq!(trailers.get(LAGGED_TRAILER).unwrap(), "506");
904 }
905
906 #[tokio::test]
907 async fn test_handle_stream_append_body() {
908 let temp_dir = tempfile::tempdir().unwrap();
909 let mut store = Store::new(temp_dir.path().to_path_buf()).unwrap();
910
911 async fn append(
912 store: &mut Store,
913 meta: Option<&str>,
914 chunks: Vec<&'static [u8]>,
915 ) -> serde_json::Value {
916 let stream = tokio_stream::iter(chunks.into_iter().map(|chunk| {
917 Ok::<_, crate::Error>(hyper::body::Frame::data(Bytes::from_static(chunk)))
918 }));
919 let mut req = Request::post("/append/note");
920 if let Some(meta) = meta {
921 req = req.header("xs-meta", base64::prelude::BASE64_STANDARD.encode(meta));
922 }
923 let req = req.body(StreamBody::new(stream)).unwrap();
924 let response = handle_stream_append(store, req, "note".into(), None, false)
925 .await
926 .unwrap();
927 assert_eq!(StatusCode::OK, response.status());
928 let body = response.into_body().collect().await.unwrap().to_bytes();
929 serde_json::from_slice(&body).unwrap()
930 }
931
932 let frame = append(&mut store, None, vec![]).await;
934 assert!(frame["hash"].is_null());
935 let frame = append(&mut store, None, vec![b"", b""]).await;
936 assert!(frame["hash"].is_null());
937
938 let frame = append(&mut store, None, vec![b"", b"hello ", b"", b"world"]).await;
941 let hash: ssri::Integrity = frame["hash"].as_str().unwrap().parse().unwrap();
942 assert_eq!(
943 b"hello world".to_vec(),
944 store.cas_read(&hash).await.unwrap()
945 );
946
947 let frame = append(&mut store, Some(r#"{"n":1}"#), vec![b"x"]).await;
949 assert_eq!(serde_json::json!({"n": 1}), frame["meta"]);
950 let hash: ssri::Integrity = frame["hash"].as_str().unwrap().parse().unwrap();
951 assert_eq!(b"x".to_vec(), store.cas_read(&hash).await.unwrap());
952 }
953
954 #[test]
955 fn test_match_route_last() {
956 let headers = hyper::HeaderMap::new();
957
958 assert!(matches!(
960 match_route(&Method::GET, "/last/test", &headers, None),
961 Routes::LastGet { topic: Some(t), last: 1, follow: false, .. } if t == "test"
962 ));
963
964 assert!(matches!(
966 match_route(&Method::GET, "/last/test", &headers, Some("follow=true")),
967 Routes::LastGet { topic: Some(t), last: 1, follow: true, .. } if t == "test"
968 ));
969
970 assert!(matches!(
972 match_route(&Method::GET, "/last", &headers, None),
973 Routes::LastGet {
974 topic: None,
975 last: 1,
976 follow: false,
977 ..
978 }
979 ));
980
981 assert!(matches!(
983 match_route(&Method::GET, "/last", &headers, Some("last=5")),
984 Routes::LastGet {
985 topic: None,
986 last: 5,
987 follow: false,
988 ..
989 }
990 ));
991
992 assert!(matches!(
994 match_route(&Method::GET, "/last/test", &headers, Some("last=3&follow=true")),
995 Routes::LastGet { topic: Some(t), last: 3, follow: true, .. } if t == "test"
996 ));
997 }
998
999 #[tokio::test]
1000 async fn test_handle_last_get_ndjson_format() {
1001 use crate::store::Store;
1002 use http_body_util::BodyExt;
1003
1004 let temp_dir = tempfile::tempdir().unwrap();
1005 let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1006
1007 store
1009 .append(crate::store::Frame::builder("test").build())
1010 .unwrap();
1011
1012 let response = handle_last_get(&store, Some("test"), 1, false, false)
1014 .await
1015 .unwrap();
1016 let body = response.into_body().collect().await.unwrap().to_bytes();
1017 let body_str = String::from_utf8(body.to_vec()).unwrap();
1018 assert!(
1019 !body_str.ends_with('\n'),
1020 "Response should be JSON (no trailing newline) when last=1, got: {:?}",
1021 body_str
1022 );
1023
1024 let response = handle_last_get(&store, Some("test"), 3, false, false)
1026 .await
1027 .unwrap();
1028 let body = response.into_body().collect().await.unwrap().to_bytes();
1029 let body_str = String::from_utf8(body.to_vec()).unwrap();
1030 assert!(
1031 body_str.ends_with('\n'),
1032 "Response should be NDJSON (end with newline) when last > 1, got: {:?}",
1033 body_str
1034 );
1035
1036 let headers = hyper::HeaderMap::new();
1038 let route = match_route(&Method::GET, "/last/test", &headers, Some("last=3"));
1039 if let Routes::LastGet {
1040 topic,
1041 last,
1042 follow,
1043 with_timestamp,
1044 } = route
1045 {
1046 assert_eq!(last, 3, "Route should parse last=3");
1047 let response = handle_last_get(&store, topic.as_deref(), last, follow, with_timestamp)
1048 .await
1049 .unwrap();
1050 let body = response.into_body().collect().await.unwrap().to_bytes();
1051 let body_str = String::from_utf8(body.to_vec()).unwrap();
1052 assert!(
1053 body_str.ends_with('\n'),
1054 "Full chain should return NDJSON when last=3, got: {:?}",
1055 body_str
1056 );
1057 } else {
1058 panic!("Expected Routes::LastGet");
1059 }
1060 }
1061
1062 #[tokio::test]
1063 async fn test_handle_eval_logic() {
1064 use crate::nu::Engine;
1066 use crate::store::Store;
1067 use nu_protocol::PipelineData;
1068
1069 let temp_dir = tempfile::tempdir().unwrap();
1071 let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1072
1073 let mut engine = Engine::new().unwrap();
1075
1076 crate::nu::add_core_commands(&mut engine, &store).unwrap();
1078
1079 engine
1081 .add_commands(vec![
1082 Box::new(crate::nu::commands::cat_command::CatCommand::new(
1083 store.clone(),
1084 )),
1085 Box::new(crate::nu::commands::last_command::LastCommand::new(
1086 store.clone(),
1087 )),
1088 Box::new(crate::nu::commands::append_command::AppendCommand::new(
1089 store.clone(),
1090 )),
1091 ])
1092 .unwrap();
1093
1094 let result = engine
1096 .eval(PipelineData::empty(), r#""hello world""#.to_string())
1097 .unwrap();
1098
1099 match result {
1100 PipelineData::Value(value, ..) => {
1101 let text = value.into_string().unwrap();
1102 assert_eq!(text, "hello world");
1103 }
1104 _ => panic!("Expected Value, got {:?}", result),
1105 }
1106
1107 let result = engine
1109 .eval(PipelineData::empty(), "2 + 3".to_string())
1110 .unwrap();
1111
1112 match result {
1113 PipelineData::Value(nu_protocol::Value::Int { val, .. }, ..) => {
1114 assert_eq!(val, 5);
1115 }
1116 _ => panic!("Expected Int Value, got {:?}", result),
1117 }
1118 }
1119
1120 #[tokio::test]
1122 async fn test_eval_uses_modules() {
1123 use crate::store::{Frame, Store};
1124 use nu_protocol::PipelineData;
1125
1126 let temp_dir = tempfile::tempdir().unwrap();
1127 let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1128
1129 let hash = store
1130 .cas_insert_sync(r#"export def add_nums [x, y] { $"sum is ($x + $y)" }"#)
1131 .unwrap();
1132 store
1133 .append(Frame::builder("xs.module.mymod").hash(hash).build())
1134 .unwrap();
1135
1136 let engine = super::eval_engine(&store).unwrap();
1137 let result = engine
1138 .eval(
1139 PipelineData::empty(),
1140 "use mymod; mymod add_nums 40 2".to_string(),
1141 )
1142 .unwrap()
1143 .into_value(nu_protocol::Span::test_data())
1144 .unwrap();
1145 assert_eq!(result.into_string().unwrap(), "sum is 42");
1146 }
1147
1148 #[tokio::test]
1152 async fn test_module_can_use_append_builtin() {
1153 use crate::store::{Frame, Store};
1154 use nu_protocol::{PipelineData, Span};
1155
1156 let temp_dir = tempfile::tempdir().unwrap();
1157 let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1158
1159 let hash = store
1160 .cas_insert_sync(r#"export def emit [] { {} | .append out --meta {via: "module"} }"#)
1161 .unwrap();
1162 store
1163 .append(Frame::builder("xs.module.emitter").hash(hash).build())
1164 .unwrap();
1165
1166 let engine = super::eval_engine(&store).unwrap();
1167 let value = engine
1168 .eval(
1169 PipelineData::empty(),
1170 "use emitter; emitter emit".to_string(),
1171 )
1172 .unwrap()
1173 .into_value(Span::test_data())
1174 .unwrap();
1175
1176 let json = crate::nu::value_to_json(&value);
1178 assert_eq!(json["topic"], "out");
1179 assert_eq!(json["meta"]["via"], "module");
1180 }
1181}