1use std::collections::HashMap;
2use std::error::Error;
3use std::str::FromStr;
4
5use chrono::{DateTime, Utc};
6use scru128::Scru128Id;
7
8use base64::Engine;
9
10use tokio::io::AsyncWriteExt;
11use tokio_stream::wrappers::ReceiverStream;
12use tokio_stream::StreamExt;
13use tokio_util::io::ReaderStream;
14
15use http_body_util::StreamBody;
16use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full};
17use hyper::body::Bytes;
18use hyper::header::ACCEPT;
19use hyper::server::conn::http1;
20use hyper::service::service_fn;
21use hyper::{Method, Request, Response, StatusCode};
22use hyper_util::rt::TokioIo;
23
24use crate::listener::Listener;
25use crate::nu;
26use crate::store::{self, FollowOption, Frame, ReadOptions, Store, TTL};
27
28type HTTPResult = Result<Response<BoxBody<Bytes, crate::Error>>, crate::Error>;
29
30fn serialize_frame(frame: &Frame, with_timestamp: bool) -> String {
32 if with_timestamp {
33 let mut value = serde_json::to_value(frame).unwrap();
34 if let serde_json::Value::Object(ref mut map) = value {
35 let millis = frame.id.timestamp() as i64;
36 let dt: DateTime<Utc> = DateTime::from_timestamp_millis(millis)
37 .unwrap_or_else(|| DateTime::from_timestamp(0, 0).unwrap());
38 map.insert("timestamp".to_string(), serde_json::json!(dt.to_rfc3339()));
39 }
40 serde_json::to_string(&value).unwrap()
41 } else {
42 serde_json::to_string(frame).unwrap()
43 }
44}
45
46#[derive(Debug, PartialEq, Clone)]
47enum AcceptType {
48 Ndjson,
49 EventStream,
50}
51
52enum Routes {
53 StreamCat {
54 accept_type: AcceptType,
55 options: ReadOptions,
56 with_timestamp: bool,
57 },
58 StreamAppend {
59 topic: String,
60 ttl: Option<TTL>,
61 with_timestamp: bool,
62 },
63 StreamAppendBatch {
64 with_timestamp: bool,
65 },
66 LastGet {
67 topic: Option<String>,
68 last: usize,
69 follow: bool,
70 with_timestamp: bool,
71 },
72 StreamItemGet {
73 id: Scru128Id,
74 with_timestamp: bool,
75 },
76 StreamItemRemove(Scru128Id),
77 CasGet(ssri::Integrity),
78 CasPost,
79 CasGc,
80 Import,
81 Eval,
82 Version,
83 Metrics,
84 NotFound,
85 BadRequest(String),
86}
87
88fn validate_integrity(integrity: &ssri::Integrity) -> bool {
90 if integrity.hashes.is_empty() {
92 return false;
93 }
94
95 for hash in &integrity.hashes {
97 if base64::engine::general_purpose::STANDARD
99 .decode(&hash.digest)
100 .is_err()
101 {
102 return false;
103 }
104 }
105
106 true
107}
108
109fn match_route(
110 method: &Method,
111 path: &str,
112 headers: &hyper::HeaderMap,
113 query: Option<&str>,
114) -> Routes {
115 let params: HashMap<String, String> =
116 url::form_urlencoded::parse(query.unwrap_or("").as_bytes())
117 .into_owned()
118 .collect();
119
120 match (method, path) {
121 (&Method::GET, "/version") => Routes::Version,
122 (&Method::GET, "/metrics") => Routes::Metrics,
123
124 (&Method::GET, "/") => {
125 let accept_type = match headers.get(ACCEPT) {
126 Some(accept) if accept == "text/event-stream" => AcceptType::EventStream,
127 _ => AcceptType::Ndjson,
128 };
129
130 let options = ReadOptions::from_query(query);
131 let with_timestamp = params.contains_key("with-timestamp");
132
133 match options {
134 Ok(options) => Routes::StreamCat {
135 accept_type,
136 options,
137 with_timestamp,
138 },
139 Err(e) => Routes::BadRequest(e.to_string()),
140 }
141 }
142
143 (&Method::GET, "/last") => {
144 let follow = params.contains_key("follow");
145 let last = params.get("last").and_then(|v| v.parse().ok()).unwrap_or(1);
146 let with_timestamp = params.contains_key("with-timestamp");
147 Routes::LastGet {
148 topic: None,
149 last,
150 follow,
151 with_timestamp,
152 }
153 }
154
155 (&Method::GET, p) if p.starts_with("/last/") => {
156 let topic = p.strip_prefix("/last/").unwrap().to_string();
157 let follow = params.contains_key("follow");
158 let last = params.get("last").and_then(|v| v.parse().ok()).unwrap_or(1);
159 let with_timestamp = params.contains_key("with-timestamp");
160 Routes::LastGet {
161 topic: Some(topic),
162 last,
163 follow,
164 with_timestamp,
165 }
166 }
167
168 (&Method::GET, p) if p.starts_with("/cas/") => {
169 if let Some(hash) = p.strip_prefix("/cas/") {
170 match ssri::Integrity::from_str(hash) {
171 Ok(integrity) => {
172 if validate_integrity(&integrity) {
173 Routes::CasGet(integrity)
174 } else {
175 Routes::BadRequest(format!("Invalid CAS hash format: {hash}"))
176 }
177 }
178 Err(e) => Routes::BadRequest(format!("Invalid CAS hash: {e}")),
179 }
180 } else {
181 Routes::NotFound
182 }
183 }
184
185 (&Method::POST, "/cas") => Routes::CasPost,
186 (&Method::POST, "/gc") => Routes::CasGc,
187 (&Method::POST, "/import") => Routes::Import,
188 (&Method::POST, "/eval") => Routes::Eval,
189
190 (&Method::GET, p) => match Scru128Id::from_str(p.trim_start_matches('/')) {
191 Ok(id) => {
192 let with_timestamp = params.contains_key("with-timestamp");
193 Routes::StreamItemGet { id, with_timestamp }
194 }
195 Err(e) => Routes::BadRequest(format!("Invalid frame ID: {e}")),
196 },
197
198 (&Method::DELETE, p) => match Scru128Id::from_str(p.trim_start_matches('/')) {
199 Ok(id) => Routes::StreamItemRemove(id),
200 Err(e) => Routes::BadRequest(format!("Invalid frame ID: {e}")),
201 },
202
203 (&Method::POST, "/append-batch") => Routes::StreamAppendBatch {
204 with_timestamp: params.contains_key("with-timestamp"),
205 },
206
207 (&Method::POST, path) if path.starts_with("/append/") => {
208 let topic = path.strip_prefix("/append/").unwrap().to_string();
209 let with_timestamp = params.contains_key("with-timestamp");
210
211 match TTL::from_query(query) {
212 Ok(ttl) => Routes::StreamAppend {
213 topic,
214 ttl: Some(ttl),
215 with_timestamp,
216 },
217 Err(e) => Routes::BadRequest(e.to_string()),
218 }
219 }
220
221 _ => Routes::NotFound,
222 }
223}
224
225async fn handle(
226 mut store: Store,
227 _engine: nu::Engine, req: Request<hyper::body::Incoming>,
229) -> HTTPResult {
230 let method = req.method();
231 let path = req.uri().path();
232 let headers = req.headers().clone();
233 let query = req.uri().query();
234
235 let res = match match_route(method, path, &headers, query) {
236 Routes::Version => handle_version().await,
237 Routes::Metrics => handle_metrics(&store).await,
238
239 Routes::StreamCat {
240 accept_type,
241 options,
242 with_timestamp,
243 } => handle_stream_cat(&mut store, options, accept_type, with_timestamp).await,
244
245 Routes::StreamAppend {
246 topic,
247 ttl,
248 with_timestamp,
249 } => handle_stream_append(&mut store, req, topic, ttl, with_timestamp).await,
250
251 Routes::StreamAppendBatch { with_timestamp } => {
252 handle_stream_append_batch(&store, req.into_body(), with_timestamp).await
253 }
254
255 Routes::CasGet(hash) => {
256 let reader = store.cas_reader(hash).await?;
257 let stream = ReaderStream::new(reader);
258
259 let stream = stream.map(|frame| {
260 let frame = frame.unwrap();
261 Ok(hyper::body::Frame::data(frame))
262 });
263
264 let body = StreamBody::new(stream).boxed();
265 Ok(Response::new(body))
266 }
267
268 Routes::CasPost => handle_cas_post(&mut store, req.into_body()).await,
269
270 Routes::CasGc => handle_cas_gc(&store).await,
271
272 Routes::StreamItemGet { id, with_timestamp } => {
273 response_frame_or_404(store.get(&id), with_timestamp)
274 }
275
276 Routes::StreamItemRemove(id) => handle_stream_item_remove(&mut store, id).await,
277
278 Routes::LastGet {
279 topic,
280 last,
281 follow,
282 with_timestamp,
283 } => handle_last_get(&store, topic.as_deref(), last, follow, with_timestamp).await,
284
285 Routes::Import => handle_import(&mut store, req.into_body()).await,
286
287 Routes::Eval => handle_eval(&store, req.into_body()).await,
288
289 Routes::NotFound => response_404(),
290 Routes::BadRequest(msg) => response_400(msg),
291 };
292
293 res.or_else(|e| response_500(e.to_string()))
294}
295
296pub const LAGGED_TRAILER: &str = "xs-lagged";
299
300pub const ERROR_TRAILER: &str = "xs-error";
302
303fn error_text(error: &nu_protocol::ShellError) -> String {
306 match error {
307 nu_protocol::ShellError::Generic(g) if g.msg.is_empty() => g.error.to_string(),
308 nu_protocol::ShellError::Generic(g) => format!("{}: {}", g.error, g.msg),
309 nu_protocol::ShellError::EvalBlockWithInput(e) => error_text(&e.error),
310 other => other.to_string(),
311 }
312}
313
314fn header_text(text: &str) -> hyper::header::HeaderValue {
316 let clean: String = text
317 .chars()
318 .map(|c| if c.is_ascii_graphic() { c } else { ' ' })
319 .collect();
320 hyper::header::HeaderValue::from_str(clean.trim())
321 .unwrap_or(hyper::header::HeaderValue::from_static("error"))
322}
323
324fn body_frames(
328 item: crate::store::ReadItem,
329 encode: &impl Fn(&Frame) -> Vec<u8>,
330) -> Vec<hyper::body::Frame<Bytes>> {
331 match item {
332 Ok(frame) => vec![hyper::body::Frame::data(Bytes::from(encode(&frame)))],
333 Err(err) => {
334 let crate::store::ReadError::Lagged { skipped } = &err;
335 let mut trailers = hyper::HeaderMap::new();
336 trailers.insert(LAGGED_TRAILER, (*skipped).into());
337 vec![
338 hyper::body::Frame::data(Bytes::from(encode(&err.to_frame()))),
339 hyper::body::Frame::trailers(trailers),
340 ]
341 }
342 }
343}
344
345async fn handle_stream_cat(
346 store: &mut Store,
347 options: ReadOptions,
348 accept_type: AcceptType,
349 with_timestamp: bool,
350) -> HTTPResult {
351 let rx = store.read(options);
352
353 let accept_type_clone = accept_type.clone();
354 let encode = move |frame: &Frame| match accept_type_clone {
355 AcceptType::Ndjson => {
356 let mut encoded = serialize_frame(frame, with_timestamp).into_bytes();
357 encoded.push(b'\n');
358 encoded
359 }
360 AcceptType::EventStream => format!(
361 "id: {id}\ndata: {data}\n\n",
362 id = frame.id,
363 data = serialize_frame(frame, with_timestamp)
364 )
365 .into_bytes(),
366 };
367 let stream = futures::StreamExt::flat_map(ReceiverStream::new(rx), move |item| {
368 futures::stream::iter(body_frames(item, &encode))
369 })
370 .map(Ok::<_, crate::Error>);
371
372 let body = StreamBody::new(stream).boxed();
373
374 let content_type = match accept_type {
375 AcceptType::Ndjson => "application/x-ndjson",
376 AcceptType::EventStream => "text/event-stream",
377 };
378
379 Ok(Response::builder()
380 .status(StatusCode::OK)
381 .header("Content-Type", content_type)
382 .header("Trailer", LAGGED_TRAILER)
383 .body(body)?)
384}
385
386const MAX_BATCH: usize = 1024;
390
391#[derive(serde::Deserialize)]
393#[serde(deny_unknown_fields)]
394struct BatchItem {
395 topic: String,
396 #[serde(default)]
397 meta: Option<serde_json::Value>,
398 #[serde(default)]
399 ttl: Option<TTL>,
400 #[serde(default)]
402 hash: Option<ssri::Integrity>,
403}
404
405async fn handle_stream_append_batch<B>(store: &Store, body: B, with_timestamp: bool) -> HTTPResult
410where
411 B: hyper::body::Body<Data = Bytes>,
412 B::Error: Into<crate::Error>,
413{
414 let bytes = body.collect().await.map_err(Into::into)?.to_bytes();
415
416 let mut frames = Vec::new();
417 for (n, line) in bytes.split(|b| *b == b'\n').enumerate() {
418 if line.iter().all(u8::is_ascii_whitespace) {
419 continue;
420 }
421 if frames.len() == MAX_BATCH {
422 return response_400(format!(
423 "a batch can hold at most {MAX_BATCH} frames: split it"
424 ));
425 }
426 let item: BatchItem = match serde_json::from_slice(line) {
427 Ok(item) => item,
428 Err(e) => return response_400(format!("line {}: {e}", n + 1)),
429 };
430 frames.push(
431 Frame::builder(item.topic)
432 .maybe_meta(item.meta)
433 .ttl(item.ttl.unwrap_or_default())
435 .maybe_hash(item.hash)
436 .build(),
437 );
438 }
439
440 let frames = store.append_many(frames)?;
441
442 let mut out = Vec::with_capacity(frames.len() * 128);
443 for frame in &frames {
444 out.extend_from_slice(serialize_frame(frame, with_timestamp).as_bytes());
445 out.push(b'\n');
446 }
447
448 Ok(Response::builder()
449 .status(StatusCode::OK)
450 .header("Content-Type", "application/x-ndjson")
451 .body(full(out))?)
452}
453
454async fn handle_stream_append<B>(
455 store: &mut Store,
456 req: Request<B>,
457 topic: String,
458 ttl: Option<TTL>,
459 with_timestamp: bool,
460) -> HTTPResult
461where
462 B: hyper::body::Body<Data = Bytes> + Unpin,
463 B::Error: Into<crate::Error>,
464{
465 let (parts, mut body) = req.into_parts();
466
467 let hash = {
468 let mut writer = None;
471
472 while let Some(frame) = body.frame().await {
473 if let Ok(data) = frame.map_err(Into::into)?.into_data() {
474 if data.is_empty() {
475 continue;
476 }
477 if writer.is_none() {
478 writer = Some(store.cas_writer().await?);
479 }
480 writer.as_mut().unwrap().write_all(&data).await?;
481 }
482 }
483
484 match writer {
485 Some(writer) => Some(writer.commit().await?),
486 None => None,
487 }
488 };
489
490 let meta = match parts
491 .headers
492 .get("xs-meta")
493 .map(|x| x.to_str())
494 .transpose()
495 .unwrap()
496 .map(|s| {
497 base64::prelude::BASE64_STANDARD
499 .decode(s)
500 .map_err(|e| format!("xs-meta isn't valid Base64: {e}"))
501 .and_then(|decoded| {
502 String::from_utf8(decoded)
504 .map_err(|e| format!("xs-meta isn't valid UTF-8: {e}"))
505 .and_then(|json_str| {
506 serde_json::from_str(&json_str)
508 .map_err(|e| format!("xs-meta isn't valid JSON: {e}"))
509 })
510 })
511 })
512 .transpose()
513 {
514 Ok(meta) => meta,
515 Err(e) => return response_400(e.to_string()),
516 };
517
518 let frame = store.append(
519 Frame::builder(topic)
520 .maybe_hash(hash)
521 .maybe_meta(meta)
522 .maybe_ttl(ttl)
523 .build(),
524 )?;
525
526 Ok(Response::builder()
527 .status(StatusCode::OK)
528 .header("Content-Type", "application/json")
529 .body(full(serialize_frame(&frame, with_timestamp)))?)
530}
531
532async fn handle_cas_post(store: &mut Store, mut body: hyper::body::Incoming) -> HTTPResult {
533 let hash = {
534 let mut writer = store.cas_writer().await?;
535 let mut bytes_written = 0;
536
537 while let Some(frame) = body.frame().await {
538 if let Ok(data) = frame?.into_data() {
539 writer.write_all(&data).await?;
540 bytes_written += data.len();
541 }
542 }
543
544 if bytes_written == 0 {
545 return response_400("Empty body".to_string());
546 }
547
548 writer.commit().await?
549 };
550
551 Ok(Response::builder()
552 .status(StatusCode::OK)
553 .header("Content-Type", "text/plain")
554 .body(full(hash.to_string()))?)
555}
556
557async fn handle_version() -> HTTPResult {
558 let version = env!("CARGO_PKG_VERSION");
559 let version_info = serde_json::json!({ "version": version });
560 Ok(Response::builder()
561 .status(StatusCode::OK)
562 .header("Content-Type", "application/json")
563 .body(full(serde_json::to_string(&version_info).unwrap()))?)
564}
565
566async fn handle_metrics(store: &Store) -> HTTPResult {
576 let body = serde_json::json!({
577 "store": store.stats(),
578 "keyspaces": store.metrics(),
579 });
580 Ok(Response::builder()
581 .status(StatusCode::OK)
582 .header("Content-Type", "application/json")
583 .body(full(serde_json::to_string(&body).unwrap()))?)
584}
585
586const CAS_GC_GRACE: std::time::Duration = std::time::Duration::from_secs(60 * 60);
590
591async fn handle_cas_gc(store: &Store) -> HTTPResult {
594 let store = store.clone();
595 let report = tokio::task::spawn_blocking(move || store.cas_gc(CAS_GC_GRACE)).await??;
596 Ok(Response::builder()
597 .status(StatusCode::OK)
598 .header("Content-Type", "application/json")
599 .body(full(serde_json::to_string(&report)?))?)
600}
601
602pub async fn serve(
603 store: Store,
604 engine: nu::Engine,
605 expose: Option<String>,
606) -> Result<(), crate::Error> {
607 let path = store.path.join("sock").to_string_lossy().to_string();
608 let listener = Listener::bind(&path).await?;
609
610 let mut listeners = vec![listener];
611 let mut expose_meta = None;
612
613 if let Some(expose) = expose {
614 let expose_listener = Listener::bind(&expose).await?;
615
616 if let Some(ticket) = expose_listener.get_ticket() {
618 expose_meta = Some(serde_json::json!({"expose": format!("iroh://{}", ticket)}));
619 } else {
620 expose_meta = Some(serde_json::json!({"expose": expose}));
621 }
622
623 listeners.push(expose_listener);
624 }
625
626 if let Err(e) = store.append(Frame::builder("xs.start").maybe_meta(expose_meta).build()) {
627 tracing::error!("Failed to append xs.start frame: {}", e);
628 }
629
630 {
633 let store = store.clone();
634 tokio::task::spawn_blocking(move || match store.cas_gc(CAS_GC_GRACE) {
635 Ok(report) => tracing::info!(?report, "cas gc at startup"),
636 Err(e) => tracing::error!("cas gc at startup: {e}"),
637 });
638 }
639
640 let mut tasks = Vec::new();
641 for listener in listeners {
642 let store = store.clone();
643 let engine = engine.clone();
644 let task = tokio::spawn(async move { listener_loop(listener, store, engine).await });
645 tasks.push(task);
646 }
647
648 for task in tasks {
651 task.await??;
652 }
653
654 Ok(())
655}
656
657async fn listener_loop(
658 mut listener: Listener,
659 store: Store,
660 engine: nu::Engine,
661) -> Result<(), crate::Error> {
662 loop {
663 let (stream, _) = listener.accept().await?;
664 let io = TokioIo::new(stream);
665 let store = store.clone();
666 let engine = engine.clone();
667 tokio::task::spawn(async move {
668 if let Err(err) = http1::Builder::new()
669 .serve_connection(
670 io,
671 service_fn(move |req| handle(store.clone(), engine.clone(), req)),
672 )
673 .await
674 {
675 if let Some(std::io::ErrorKind::NotConnected) = err.source().and_then(|source| {
677 source
678 .downcast_ref::<std::io::Error>()
679 .map(|io_err| io_err.kind())
680 }) {
681 } else {
683 tracing::error!("TBD: {:?}", err);
685 }
686 }
687 });
688 }
689}
690
691fn response_frame_or_404(frame: Option<store::Frame>, with_timestamp: bool) -> HTTPResult {
692 if let Some(frame) = frame {
693 Ok(Response::builder()
694 .status(StatusCode::OK)
695 .header("Content-Type", "application/json")
696 .body(full(serialize_frame(&frame, with_timestamp)))?)
697 } else {
698 response_404()
699 }
700}
701
702async fn handle_stream_item_remove(store: &mut Store, id: Scru128Id) -> HTTPResult {
703 match store.remove(&id) {
704 Ok(()) => Ok(Response::builder()
705 .status(StatusCode::NO_CONTENT)
706 .body(empty())?),
707 Err(e) => {
708 tracing::error!("Failed to remove item {}: {:?}", id, e);
709
710 Ok(Response::builder()
711 .status(StatusCode::INTERNAL_SERVER_ERROR)
712 .body(full("internal-error"))?)
713 }
714 }
715}
716
717async fn handle_last_get(
718 store: &Store,
719 topic: Option<&str>,
720 last: usize,
721 follow: bool,
722 with_timestamp: bool,
723) -> HTTPResult {
724 if !follow {
725 let options = ReadOptions::builder()
726 .last(last)
727 .maybe_topic(topic.map(|t| t.to_string()))
728 .build();
729
730 let frames: Vec<Frame> = store.read_sync(options).collect();
731
732 if frames.is_empty() {
733 return response_404();
734 }
735
736 if last == 1 {
740 return response_frame_or_404(Some(frames.into_iter().next().unwrap()), with_timestamp);
741 }
742
743 let mut body = Vec::new();
744 for frame in frames {
745 body.extend(serialize_frame(&frame, with_timestamp).into_bytes());
746 body.push(b'\n');
747 }
748
749 return Ok(Response::builder()
750 .status(StatusCode::OK)
751 .header("Content-Type", "application/x-ndjson")
752 .body(full(body))?);
753 }
754
755 let rx = store.read(
758 ReadOptions::builder()
759 .last(last)
760 .maybe_topic(topic.map(|t| t.to_string()))
761 .follow(FollowOption::On)
762 .build(),
763 );
764
765 let encode = move |frame: &Frame| {
766 let mut bytes = serialize_frame(frame, with_timestamp).into_bytes();
767 bytes.push(b'\n');
768 bytes
769 };
770 let stream = futures::StreamExt::flat_map(ReceiverStream::new(rx), move |item| {
771 futures::stream::iter(body_frames(item, &encode))
772 })
773 .map(Ok::<_, crate::Error>);
774
775 Ok(Response::builder()
776 .status(StatusCode::OK)
777 .header("Content-Type", "application/x-ndjson")
778 .header("Trailer", LAGGED_TRAILER)
779 .body(StreamBody::new(stream).boxed())?)
780}
781
782async fn handle_import(store: &mut Store, body: hyper::body::Incoming) -> HTTPResult {
783 let bytes = body.collect().await?.to_bytes();
784 let frame: Frame = match serde_json::from_slice(&bytes) {
785 Ok(frame) => frame,
786 Err(e) => return response_400(format!("Invalid frame JSON: {e}")),
787 };
788
789 store.insert_frame(&frame)?;
790
791 Ok(Response::builder()
792 .status(StatusCode::OK)
793 .header("Content-Type", "application/json")
794 .body(full(serde_json::to_string(&frame).unwrap()))?)
795}
796
797fn response_404() -> HTTPResult {
798 Ok(Response::builder()
799 .status(StatusCode::NOT_FOUND)
800 .body(empty())?)
801}
802
803fn response_400(message: String) -> HTTPResult {
804 let body = full(message);
805 Ok(Response::builder()
806 .status(StatusCode::BAD_REQUEST)
807 .body(body)?)
808}
809
810fn response_500(message: String) -> HTTPResult {
811 let body = full(message);
812 Ok(Response::builder()
813 .status(StatusCode::INTERNAL_SERVER_ERROR)
814 .body(body)?)
815}
816
817fn full<T: Into<Bytes>>(chunk: T) -> BoxBody<Bytes, crate::Error> {
818 Full::new(chunk.into())
819 .map_err(|never| match never {})
820 .boxed()
821}
822
823fn empty() -> BoxBody<Bytes, crate::Error> {
824 Empty::<Bytes>::new()
825 .map_err(|never| match never {})
826 .boxed()
827}
828
829fn eval_engine(store: &Store) -> Result<nu::Engine, String> {
833 let mut engine = nu::prepared_base(store, true)
834 .map_err(|e| format!("Failed to build nushell engine: {e}"))?;
835 let modules = store.nu_modules_at(&scru128::new());
838 nu::load_modules(&mut engine.state, store, &modules)
839 .map_err(|e| format!("Failed to load modules: {e}"))?;
840 Ok(engine)
841}
842
843async fn handle_eval(store: &Store, body: hyper::body::Incoming) -> HTTPResult {
844 let bytes = body.collect().await?.to_bytes();
846 let script =
847 String::from_utf8(bytes.to_vec()).map_err(|e| format!("Invalid UTF-8 in script: {e}"))?;
848
849 let engine = eval_engine(store)?;
850
851 let result = std::thread::scope(|scope| {
858 scope
859 .spawn(|| engine.eval(nu_protocol::PipelineData::empty(), script))
860 .join()
861 .expect("eval thread panicked")
862 })
863 .map_err(|e| format!("Script evaluation failed:\n{e}"))?;
864
865 match result {
867 nu_protocol::PipelineData::ByteStream(stream, ..) => {
868 if let Some(mut reader) = stream.reader() {
870 use std::io::Read;
871
872 let (tx, rx) = tokio::sync::mpsc::channel(16);
873
874 std::thread::spawn(move || {
876 let mut buffer = [0u8; 8192];
877 loop {
878 match reader.read(&mut buffer) {
879 Ok(0) => break, Ok(n) => {
881 let chunk = Bytes::copy_from_slice(&buffer[..n]);
882 if tx
883 .blocking_send(Ok(hyper::body::Frame::data(chunk)))
884 .is_err()
885 {
886 break;
887 }
888 }
889 Err(e) => {
890 let _ = tx.blocking_send(Err(Box::new(e) as crate::Error));
891 break;
892 }
893 }
894 }
895 });
896
897 let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
898 let body = StreamBody::new(stream).boxed();
899 Ok(Response::builder()
900 .status(StatusCode::OK)
901 .header("Content-Type", "application/octet-stream")
902 .body(body)?)
903 } else {
904 Ok(Response::builder()
906 .status(StatusCode::OK)
907 .header("Content-Type", "application/octet-stream")
908 .body(empty())?)
909 }
910 }
911 nu_protocol::PipelineData::ListStream(stream, ..) => {
912 let (tx, rx) = tokio::sync::mpsc::channel(16);
914
915 std::thread::spawn(move || {
917 for value in stream.into_iter() {
918 if let nu_protocol::Value::Error { error, .. } = &value {
922 let mut trailers = hyper::HeaderMap::new();
923 let text = error_text(error);
924 trailers.insert(ERROR_TRAILER, header_text(&text));
925 let _ = tx.blocking_send(Ok(hyper::body::Frame::trailers(trailers)));
926 break;
927 }
928 let json = nu::value_to_json(&value);
929 match serde_json::to_vec(&json) {
930 Ok(mut json_bytes) => {
931 json_bytes.push(b'\n'); let chunk = Bytes::from(json_bytes);
933 if tx
934 .blocking_send(Ok(hyper::body::Frame::data(chunk)))
935 .is_err()
936 {
937 break;
938 }
939 }
940 Err(e) => {
941 let _ = tx.blocking_send(Err(Box::new(e) as crate::Error));
942 break;
943 }
944 }
945 }
946 });
947
948 let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
949 let body = StreamBody::new(stream).boxed();
950 Ok(Response::builder()
951 .status(StatusCode::OK)
952 .header("Content-Type", "application/x-ndjson")
953 .header("Trailer", ERROR_TRAILER)
954 .body(body)?)
955 }
956 nu_protocol::PipelineData::Value(value, ..) => {
957 match &value {
958 nu_protocol::Value::String { .. }
959 | nu_protocol::Value::Int { .. }
960 | nu_protocol::Value::Float { .. }
961 | nu_protocol::Value::Bool { .. } => {
962 let text = match value {
964 nu_protocol::Value::String { val, .. } => val.clone(),
965 nu_protocol::Value::Int { val, .. } => val.to_string(),
966 nu_protocol::Value::Float { val, .. } => val.to_string(),
967 nu_protocol::Value::Bool { val, .. } => val.to_string(),
968 _ => value.into_string().unwrap_or_else(|_| "".to_string()),
969 };
970 Ok(Response::builder()
971 .status(StatusCode::OK)
972 .header("Content-Type", "text/plain")
973 .body(full(text))?)
974 }
975 _ => {
976 let json = nu::value_to_json(&value);
978 let json_string = serde_json::to_string(&json)
979 .map_err(|e| format!("Failed to serialize JSON: {e}"))?;
980 Ok(Response::builder()
981 .status(StatusCode::OK)
982 .header("Content-Type", "application/json")
983 .body(full(json_string))?)
984 }
985 }
986 }
987 nu_protocol::PipelineData::Empty => {
988 Ok(Response::builder()
990 .status(StatusCode::NO_CONTENT)
991 .body(empty())?)
992 }
993 }
994}
995
996#[cfg(test)]
997mod tests {
998 use super::*;
999
1000 #[test]
1001 fn a_lag_ends_with_the_line_then_the_trailer() {
1002 let encode = |frame: &Frame| frame.topic.clone().into_bytes();
1003 let frames = body_frames(
1004 Err(crate::store::ReadError::Lagged { skipped: 506 }),
1005 &encode,
1006 );
1007 assert_eq!(frames.len(), 2);
1008 assert_eq!(frames[0].data_ref().unwrap().as_ref(), b"xs.lagged");
1009 let trailers = frames[1].trailers_ref().unwrap();
1010 assert_eq!(trailers.get(LAGGED_TRAILER).unwrap(), "506");
1011 }
1012
1013 #[tokio::test]
1014 async fn test_handle_stream_append_body() {
1015 let temp_dir = tempfile::tempdir().unwrap();
1016 let mut store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1017
1018 async fn append(
1019 store: &mut Store,
1020 meta: Option<&str>,
1021 chunks: Vec<&'static [u8]>,
1022 ) -> serde_json::Value {
1023 let stream = tokio_stream::iter(chunks.into_iter().map(|chunk| {
1024 Ok::<_, crate::Error>(hyper::body::Frame::data(Bytes::from_static(chunk)))
1025 }));
1026 let mut req = Request::post("/append/note");
1027 if let Some(meta) = meta {
1028 req = req.header("xs-meta", base64::prelude::BASE64_STANDARD.encode(meta));
1029 }
1030 let req = req.body(StreamBody::new(stream)).unwrap();
1031 let response = handle_stream_append(store, req, "note".into(), None, false)
1032 .await
1033 .unwrap();
1034 assert_eq!(StatusCode::OK, response.status());
1035 let body = response.into_body().collect().await.unwrap().to_bytes();
1036 serde_json::from_slice(&body).unwrap()
1037 }
1038
1039 let frame = append(&mut store, None, vec![]).await;
1041 assert!(frame["hash"].is_null());
1042 let frame = append(&mut store, None, vec![b"", b""]).await;
1043 assert!(frame["hash"].is_null());
1044
1045 let frame = append(&mut store, None, vec![b"", b"hello ", b"", b"world"]).await;
1048 let hash: ssri::Integrity = frame["hash"].as_str().unwrap().parse().unwrap();
1049 assert_eq!(
1050 b"hello world".to_vec(),
1051 store.cas_read(&hash).await.unwrap()
1052 );
1053
1054 let frame = append(&mut store, Some(r#"{"n":1}"#), vec![b"x"]).await;
1056 assert_eq!(serde_json::json!({"n": 1}), frame["meta"]);
1057 let hash: ssri::Integrity = frame["hash"].as_str().unwrap().parse().unwrap();
1058 assert_eq!(b"x".to_vec(), store.cas_read(&hash).await.unwrap());
1059 }
1060
1061 #[tokio::test]
1062 async fn test_handle_cas_gc() {
1063 let headers = hyper::HeaderMap::new();
1064 assert!(matches!(
1065 match_route(&Method::POST, "/gc", &headers, None),
1066 Routes::CasGc
1067 ));
1068
1069 let temp_dir = tempfile::tempdir().unwrap();
1070 let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1071 let hash = store.cas_insert_sync("in use").unwrap();
1072 store
1073 .append(Frame::builder("a").hash(hash).build())
1074 .unwrap();
1075 store.cas_insert_sync("just uploaded").unwrap();
1076
1077 let response = handle_cas_gc(&store).await.unwrap();
1078 assert_eq!(StatusCode::OK, response.status());
1079 let body = response.into_body().collect().await.unwrap().to_bytes();
1080 let report: serde_json::Value = serde_json::from_slice(&body).unwrap();
1081 assert_eq!(
1082 report,
1083 serde_json::json!({"in_use": 1, "recent": 1, "deleted": 0, "bytes_freed": 0})
1084 );
1085 }
1086
1087 #[test]
1088 fn test_match_route_last() {
1089 let headers = hyper::HeaderMap::new();
1090
1091 assert!(matches!(
1093 match_route(&Method::GET, "/last/test", &headers, None),
1094 Routes::LastGet { topic: Some(t), last: 1, follow: false, .. } if t == "test"
1095 ));
1096
1097 assert!(matches!(
1099 match_route(&Method::GET, "/last/test", &headers, Some("follow=true")),
1100 Routes::LastGet { topic: Some(t), last: 1, follow: true, .. } if t == "test"
1101 ));
1102
1103 assert!(matches!(
1105 match_route(&Method::GET, "/last", &headers, None),
1106 Routes::LastGet {
1107 topic: None,
1108 last: 1,
1109 follow: false,
1110 ..
1111 }
1112 ));
1113
1114 assert!(matches!(
1116 match_route(&Method::GET, "/last", &headers, Some("last=5")),
1117 Routes::LastGet {
1118 topic: None,
1119 last: 5,
1120 follow: false,
1121 ..
1122 }
1123 ));
1124
1125 assert!(matches!(
1127 match_route(&Method::GET, "/last/test", &headers, Some("last=3&follow=true")),
1128 Routes::LastGet { topic: Some(t), last: 3, follow: true, .. } if t == "test"
1129 ));
1130 }
1131
1132 #[tokio::test]
1133 async fn test_handle_stream_append_batch() {
1134 use crate::store::Store;
1135 use http_body_util::BodyExt;
1136
1137 let headers = hyper::HeaderMap::new();
1138 assert!(matches!(
1139 match_route(&Method::POST, "/append-batch", &headers, None),
1140 Routes::StreamAppendBatch {
1141 with_timestamp: false
1142 }
1143 ));
1144
1145 let temp_dir = tempfile::tempdir().unwrap();
1146 let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1147
1148 async fn post(store: &Store, body: &str) -> (StatusCode, Vec<serde_json::Value>) {
1149 let response = handle_stream_append_batch(
1150 store,
1151 Full::new(Bytes::copy_from_slice(body.as_bytes())),
1152 false,
1153 )
1154 .await
1155 .unwrap();
1156 let status = response.status();
1157 let body = response.into_body().collect().await.unwrap().to_bytes();
1158 let frames = body
1159 .split(|b| *b == b'\n')
1160 .filter(|line| !line.is_empty())
1161 .filter_map(|line| serde_json::from_slice(line).ok())
1162 .collect();
1163 (status, frames)
1164 }
1165
1166 let (status, frames) = post(
1168 &store,
1169 "{\"topic\":\"a\",\"meta\":{\"n\":1}}\n\n{\"topic\":\"b\",\"ttl\":\"last:2\"}\n",
1170 )
1171 .await;
1172 assert_eq!(StatusCode::OK, status);
1173 assert_eq!(2, frames.len());
1174 assert_eq!("a", frames[0]["topic"]);
1175 assert_eq!(serde_json::json!({"n": 1}), frames[0]["meta"]);
1176 assert_eq!("forever", frames[0]["ttl"]);
1177 assert_eq!("last:2", frames[1]["ttl"]);
1178 assert!(frames[0]["id"].as_str() < frames[1]["id"].as_str());
1179
1180 let (status, _) = post(&store, "{\"topic\":\"a\"}\n{\"topic\":\"a\",\"bogus\":1}\n").await;
1182 assert_eq!(StatusCode::BAD_REQUEST, status);
1183 let mut rx = store.read(ReadOptions::default());
1184 let mut stored = 0;
1185 while rx.recv().await.is_some() {
1186 stored += 1;
1187 }
1188 assert_eq!(2, stored);
1189
1190 let line = "{\"topic\":\"c\"}\n";
1192 let (status, frames) = post(&store, &line.repeat(MAX_BATCH)).await;
1193 assert_eq!(StatusCode::OK, status);
1194 assert_eq!(MAX_BATCH, frames.len());
1195 let (status, _) = post(&store, &line.repeat(MAX_BATCH + 1)).await;
1196 assert_eq!(StatusCode::BAD_REQUEST, status);
1197 let mut rx = store.read(ReadOptions::default());
1198 let mut stored = 0;
1199 while rx.recv().await.is_some() {
1200 stored += 1;
1201 }
1202 assert_eq!(2 + MAX_BATCH, stored);
1203 }
1204
1205 #[tokio::test]
1206 async fn test_handle_last_get_ndjson_format() {
1207 use crate::store::Store;
1208 use http_body_util::BodyExt;
1209
1210 let temp_dir = tempfile::tempdir().unwrap();
1211 let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1212
1213 store
1215 .append(crate::store::Frame::builder("test").build())
1216 .unwrap();
1217
1218 let response = handle_last_get(&store, Some("test"), 1, false, false)
1220 .await
1221 .unwrap();
1222 let body = response.into_body().collect().await.unwrap().to_bytes();
1223 let body_str = String::from_utf8(body.to_vec()).unwrap();
1224 assert!(
1225 !body_str.ends_with('\n'),
1226 "Response should be JSON (no trailing newline) when last=1, got: {:?}",
1227 body_str
1228 );
1229
1230 let response = handle_last_get(&store, Some("test"), 3, false, false)
1232 .await
1233 .unwrap();
1234 let body = response.into_body().collect().await.unwrap().to_bytes();
1235 let body_str = String::from_utf8(body.to_vec()).unwrap();
1236 assert!(
1237 body_str.ends_with('\n'),
1238 "Response should be NDJSON (end with newline) when last > 1, got: {:?}",
1239 body_str
1240 );
1241
1242 let headers = hyper::HeaderMap::new();
1244 let route = match_route(&Method::GET, "/last/test", &headers, Some("last=3"));
1245 if let Routes::LastGet {
1246 topic,
1247 last,
1248 follow,
1249 with_timestamp,
1250 } = route
1251 {
1252 assert_eq!(last, 3, "Route should parse last=3");
1253 let response = handle_last_get(&store, topic.as_deref(), last, follow, with_timestamp)
1254 .await
1255 .unwrap();
1256 let body = response.into_body().collect().await.unwrap().to_bytes();
1257 let body_str = String::from_utf8(body.to_vec()).unwrap();
1258 assert!(
1259 body_str.ends_with('\n'),
1260 "Full chain should return NDJSON when last=3, got: {:?}",
1261 body_str
1262 );
1263 } else {
1264 panic!("Expected Routes::LastGet");
1265 }
1266 }
1267
1268 #[tokio::test]
1269 async fn test_handle_eval_logic() {
1270 use crate::nu::Engine;
1272 use crate::store::Store;
1273 use nu_protocol::PipelineData;
1274
1275 let temp_dir = tempfile::tempdir().unwrap();
1277 let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1278
1279 let mut engine = Engine::new().unwrap();
1281
1282 crate::nu::add_core_commands(&mut engine, &store).unwrap();
1284
1285 engine
1287 .add_commands(vec![
1288 Box::new(crate::nu::commands::cat_command::CatCommand::new(
1289 store.clone(),
1290 )),
1291 Box::new(crate::nu::commands::last_command::LastCommand::new(
1292 store.clone(),
1293 )),
1294 Box::new(crate::nu::commands::append_command::AppendCommand::new(
1295 store.clone(),
1296 )),
1297 ])
1298 .unwrap();
1299
1300 let result = engine
1302 .eval(PipelineData::empty(), r#""hello world""#.to_string())
1303 .unwrap();
1304
1305 match result {
1306 PipelineData::Value(value, ..) => {
1307 let text = value.into_string().unwrap();
1308 assert_eq!(text, "hello world");
1309 }
1310 _ => panic!("Expected Value, got {:?}", result),
1311 }
1312
1313 let result = engine
1315 .eval(PipelineData::empty(), "2 + 3".to_string())
1316 .unwrap();
1317
1318 match result {
1319 PipelineData::Value(nu_protocol::Value::Int { val, .. }, ..) => {
1320 assert_eq!(val, 5);
1321 }
1322 _ => panic!("Expected Int Value, got {:?}", result),
1323 }
1324 }
1325
1326 #[tokio::test]
1328 async fn test_eval_uses_modules() {
1329 use crate::store::{Frame, Store};
1330 use nu_protocol::PipelineData;
1331
1332 let temp_dir = tempfile::tempdir().unwrap();
1333 let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1334
1335 let hash = store
1336 .cas_insert_sync(r#"export def add_nums [x, y] { $"sum is ($x + $y)" }"#)
1337 .unwrap();
1338 store
1339 .append(Frame::builder("xs.module.mymod").hash(hash).build())
1340 .unwrap();
1341
1342 let engine = super::eval_engine(&store).unwrap();
1343 let result = engine
1344 .eval(
1345 PipelineData::empty(),
1346 "use mymod; mymod add_nums 40 2".to_string(),
1347 )
1348 .unwrap()
1349 .into_value(nu_protocol::Span::test_data())
1350 .unwrap();
1351 assert_eq!(result.into_string().unwrap(), "sum is 42");
1352 }
1353
1354 #[tokio::test]
1358 async fn test_module_can_use_append_builtin() {
1359 use crate::store::{Frame, Store};
1360 use nu_protocol::{PipelineData, Span};
1361
1362 let temp_dir = tempfile::tempdir().unwrap();
1363 let store = Store::new(temp_dir.path().to_path_buf()).unwrap();
1364
1365 let hash = store
1366 .cas_insert_sync(r#"export def emit [] { {} | .append out --meta {via: "module"} }"#)
1367 .unwrap();
1368 store
1369 .append(Frame::builder("xs.module.emitter").hash(hash).build())
1370 .unwrap();
1371
1372 let engine = super::eval_engine(&store).unwrap();
1373 let value = engine
1374 .eval(
1375 PipelineData::empty(),
1376 "use emitter; emitter emit".to_string(),
1377 )
1378 .unwrap()
1379 .into_value(Span::test_data())
1380 .unwrap();
1381
1382 let json = crate::nu::value_to_json(&value);
1384 assert_eq!(json["topic"], "out");
1385 assert_eq!(json["meta"]["via"], "module");
1386 }
1387}