Skip to main content

xs/processor/service/
service.rs

1use scru128::Scru128Id;
2use tokio::task::JoinHandle;
3
4use nu_protocol::{ByteStream, ByteStreamType, PipelineData, Signals, Span, Value};
5use std::io::Read;
6use std::sync::atomic::AtomicBool;
7use std::sync::Arc;
8use std::time::Duration;
9use tokio::io::AsyncReadExt;
10use tokio_util::sync::CancellationToken;
11
12use crate::nu;
13use crate::nu::{value_to_json, ReturnOptions};
14use crate::store::{FollowOption, Frame, ReadOptions, Store};
15use serde_json::json;
16
17/// What the run loop does when the pipeline drains cleanly. An error always
18/// stops the service with `.fin.error`, whatever the policy.
19#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Deserialize, Default)]
20#[serde(rename_all = "kebab-case")]
21pub enum RestartPolicy {
22    /// Wait one second and run the closure again. No lifecycle frame.
23    #[default]
24    OnSuccess,
25    /// Emit `.fin.ok` and stop. The service stays down across boots.
26    Never,
27}
28
29#[derive(Clone, Debug, serde::Deserialize, Default)]
30pub struct ServiceScriptOptions {
31    pub duplex: Option<bool>,
32    pub restart: Option<RestartPolicy>,
33    pub return_options: Option<ReturnOptions>,
34}
35
36#[derive(Clone)]
37pub struct ServiceLoop {
38    pub topic: String,
39}
40
41#[derive(Clone)]
42pub struct Task {
43    pub id: Scru128Id,
44    pub run_closure: nu_protocol::engine::Closure,
45    pub return_options: Option<ReturnOptions>,
46    pub duplex: bool,
47    pub restart: RestartPolicy,
48    pub engine: nu::Engine,
49}
50
51#[cfg_attr(not(test), allow(dead_code))]
52#[derive(Debug, Clone)]
53pub enum ServiceEventKind {
54    Running,
55    /// output frame flushed; payload is raw bytes stored in CAS
56    Recv {
57        suffix: String,
58        data: Vec<u8>,
59    },
60    /// output frame flushed; payload is a JSON record stored as frame metadata
61    RecvMeta {
62        suffix: String,
63        meta: serde_json::Value,
64    },
65    Stopped(StopReason),
66    ParseError {
67        message: String,
68    },
69    Shutdown,
70}
71
72#[cfg_attr(not(test), allow(dead_code))]
73#[derive(Debug, Clone)]
74pub struct ServiceEvent {
75    pub kind: ServiceEventKind,
76    pub frame: Frame,
77}
78
79#[cfg_attr(not(test), allow(dead_code))]
80#[derive(Debug, Clone)]
81pub enum StopReason {
82    Finished,
83    Error { message: String },
84    Terminate,
85    Shutdown,
86    Update { update_id: Scru128Id },
87}
88
89pub(crate) fn emit_event(
90    store: &Store,
91    loop_ctx: &ServiceLoop,
92    source_id: Scru128Id,
93    return_opts: Option<&ReturnOptions>,
94    kind: ServiceEventKind,
95) -> Result<ServiceEvent, crate::Error> {
96    let frame = match &kind {
97        ServiceEventKind::Running => store.append(
98            Frame::builder(format!("xs.service.{}.active", loop_ctx.topic))
99                .meta(json!({ "source_id": source_id.to_string() }))
100                .build(),
101        )?,
102
103        ServiceEventKind::Recv { suffix, data } => {
104            // Data topic: app namespace, unchanged.
105            let hash = store.cas_insert_bytes_sync(data)?;
106            store.append(
107                Frame::builder(format!(
108                    "{topic}{suffix}",
109                    topic = loop_ctx.topic,
110                    suffix = suffix
111                ))
112                .hash(hash)
113                .maybe_ttl(return_opts.and_then(|o| o.ttl.clone()))
114                .meta(json!({ "source_id": source_id.to_string() }))
115                .build(),
116            )?
117        }
118
119        ServiceEventKind::RecvMeta { suffix, meta } => {
120            // Data topic: app namespace, unchanged.
121            let mut merged = meta.clone();
122            merged["source_id"] = json!(source_id.to_string());
123            store.append(
124                Frame::builder(format!(
125                    "{topic}{suffix}",
126                    topic = loop_ctx.topic,
127                    suffix = suffix
128                ))
129                .maybe_ttl(return_opts.and_then(|o| o.ttl.clone()))
130                .meta(merged)
131                .build(),
132            )?
133        }
134
135        ServiceEventKind::Stopped(reason) => {
136            // Per ADR 0005:
137            //   Finished  -> .fin.ok       (terminal, won't restart)
138            //   Error     -> .fin.error    (terminal)
139            //   Terminate -> .fin.term     (terminal)
140            //   Update    -> .replaced     (transient; successor coming)
141            //   Shutdown  -> not a lifecycle frame on its own; the server-
142            //                shutdown ack is the separate ServiceEventKind::
143            //                Shutdown emission below. Skip here.
144            let event_suffix = match reason {
145                StopReason::Finished => "fin.ok",
146                StopReason::Error { .. } => "fin.error",
147                StopReason::Terminate => "fin.term",
148                StopReason::Update { .. } => "replaced",
149                StopReason::Shutdown => {
150                    // No frame: the run_loop emits ServiceEventKind::Shutdown
151                    // (which becomes the .stopped frame) for the xs-stopping
152                    // path; emitting here would double up.
153                    return Ok(ServiceEvent {
154                        kind,
155                        frame: Frame::builder("").build(),
156                    });
157                }
158            };
159            let mut meta = json!({
160                "source_id": source_id.to_string(),
161            });
162            if let StopReason::Update { update_id } = reason {
163                meta["update_id"] = json!(update_id.to_string());
164            }
165            if let StopReason::Error { message } = reason {
166                meta["message"] = json!(message);
167            }
168            store.append(
169                Frame::builder(format!("xs.service.{}.{event_suffix}", loop_ctx.topic))
170                    .meta(meta)
171                    .build(),
172            )?
173        }
174
175        ServiceEventKind::ParseError { message } => store.append(
176            Frame::builder(format!("xs.service.{}.invalid", loop_ctx.topic))
177                .meta(json!({
178                    "source_id": source_id.to_string(),
179                    "reason": message,
180                }))
181                .build(),
182        )?,
183
184        ServiceEventKind::Shutdown => store.append(
185            Frame::builder(format!("xs.service.{}.stopped", loop_ctx.topic))
186                .meta(json!({ "source_id": source_id.to_string() }))
187                .build(),
188        )?,
189    };
190
191    Ok(ServiceEvent { kind, frame })
192}
193
194pub fn spawn(store: Store, spawn_frame: Frame) -> JoinHandle<()> {
195    tokio::spawn(async move { run(store, spawn_frame).await })
196}
197
198/// Clone the prepared base engine and specialize it for a `<name>.create`
199/// frame: load the modules visible as of that frame and stamp `service_id`.
200/// The read and write builtins already live on `base`, so neither the
201/// initial spawn nor a hot-replace re-registers builtins.
202fn specialize(
203    base: &nu::Engine,
204    store: &Store,
205    create_id: Scru128Id,
206) -> Result<nu::Engine, crate::Error> {
207    let mut engine = base.clone();
208    let modules = store.nu_modules_at(&create_id);
209    nu::load_modules(&mut engine.state, store, &modules)?;
210    engine.set_append_meta(&serde_json::json!({ "service_id": create_id.to_string() }));
211    Ok(engine)
212}
213
214async fn run(store: Store, spawn_frame: Frame) {
215    // Prepared once (nushell + stdlib + core + reads + Direct `.append`);
216    // cloned for the initial run and every hot-replace.
217    let base = match nu::prepared_base(&store, true) {
218        Ok(e) => e,
219        Err(_) => return,
220    };
221    let mut engine = match specialize(&base, &store, spawn_frame.id) {
222        Ok(e) => e,
223        Err(_) => return,
224    };
225
226    let hash = match spawn_frame.hash.clone() {
227        Some(h) => h,
228        None => return,
229    };
230    let mut reader = match store.cas_reader(hash).await {
231        Ok(r) => r,
232        Err(_) => return,
233    };
234    let mut script = String::new();
235    if reader.read_to_string(&mut script).await.is_err() {
236        return;
237    }
238
239    let loop_ctx = ServiceLoop {
240        // Topic is `xs.service.<name>.create`; strip both ends to get <name>.
241        topic: spawn_frame
242            .topic
243            .strip_prefix("xs.service.")
244            .and_then(|rest| rest.strip_suffix(".create"))
245            .unwrap_or(&spawn_frame.topic)
246            .to_string(),
247    };
248
249    let nu_config = match nu::parse_config(&mut engine, &script) {
250        Ok(cfg) => cfg,
251        Err(e) => {
252            let _ = emit_event(
253                &store,
254                &loop_ctx,
255                spawn_frame.id,
256                None,
257                ServiceEventKind::ParseError {
258                    message: e.to_string(),
259                },
260            );
261            return;
262        }
263    };
264    let opts: ServiceScriptOptions = nu_config.deserialize_options().unwrap_or_default();
265
266    // Create and set the interrupt signal on the engine state
267    let interrupt = Arc::new(AtomicBool::new(false));
268    engine.state.set_signals(Signals::new(interrupt.clone()));
269
270    let task = Task {
271        id: spawn_frame.id,
272        run_closure: nu_config.run_closure,
273        return_options: opts.return_options,
274        duplex: opts.duplex.unwrap_or(false),
275        restart: opts.restart.unwrap_or_default(),
276        engine,
277    };
278
279    run_loop(store, loop_ctx, task, base).await;
280}
281
282async fn run_loop(store: Store, loop_ctx: ServiceLoop, mut task: Task, base: nu::Engine) {
283    // Create the first start frame and set up a persistent control subscription
284    let start_event = emit_event(
285        &store,
286        &loop_ctx,
287        task.id,
288        task.return_options.as_ref(),
289        ServiceEventKind::Running,
290    )
291    .expect("failed to emit running event");
292    let mut start_id = start_event.frame.id;
293
294    let control_rx_options = ReadOptions::builder()
295        .follow(FollowOption::On)
296        .after(start_id)
297        .build();
298
299    let mut control_rx = store.read(control_rx_options);
300
301    enum LoopOutcome {
302        Continue,
303        Finished,
304        Update(Box<Task>, Scru128Id),
305        Terminate,
306        Shutdown,
307        Error(String),
308    }
309
310    impl core::fmt::Debug for LoopOutcome {
311        fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
312            match self {
313                LoopOutcome::Continue => write!(f, "Continue"),
314                LoopOutcome::Finished => write!(f, "Finished"),
315                LoopOutcome::Update(_, id) => f.debug_tuple("Update").field(id).finish(),
316                LoopOutcome::Terminate => write!(f, "Terminate"),
317                LoopOutcome::Shutdown => write!(f, "Shutdown"),
318                LoopOutcome::Error(e) => f.debug_tuple("Error").field(e).finish(),
319            }
320        }
321    }
322
323    impl From<&LoopOutcome> for StopReason {
324        fn from(value: &LoopOutcome) -> Self {
325            match value {
326                LoopOutcome::Continue | LoopOutcome::Finished => StopReason::Finished,
327                LoopOutcome::Update(_, id) => StopReason::Update { update_id: *id },
328                LoopOutcome::Terminate => StopReason::Terminate,
329                LoopOutcome::Shutdown => StopReason::Shutdown,
330                LoopOutcome::Error(e) => StopReason::Error { message: e.clone() },
331            }
332        }
333    }
334
335    loop {
336        // Set when the send read falls behind the live stream: the input
337        // then ends, and the run below stops with that error instead of
338        // finishing (ADR 0008).
339        let input_lag: Arc<std::sync::Mutex<Option<String>>> = Arc::default();
340        // Cancelled when the run is stopped: wakes the input read, which
341        // otherwise waits for the next `.send` frame.
342        let stop = CancellationToken::new();
343        let input_pipeline = if task.duplex {
344            let options = ReadOptions::builder()
345                .follow(FollowOption::On)
346                .after(start_id)
347                .build();
348            let send_rx = store.read(options);
349            build_input_pipeline(
350                store.clone(),
351                &loop_ctx,
352                &task,
353                send_rx,
354                input_lag.clone(),
355                stop.clone(),
356            )
357            .await
358        } else {
359            PipelineData::empty()
360        };
361
362        let (done_tx, done_rx) = tokio::sync::oneshot::channel();
363        spawn_thread(
364            store.clone(),
365            loop_ctx.clone(),
366            task.clone(),
367            input_pipeline,
368            done_tx,
369        );
370
371        let terminate_topic = format!("xs.service.{}.term", loop_ctx.topic);
372        let spawn_topic = format!("xs.service.{}.create", loop_ctx.topic);
373        tokio::pin!(done_rx);
374        let halt = |task: &Task| {
375            stop.cancel();
376            task.engine.state.signals().trigger();
377            task.engine.kill_job_by_name(&task.id.to_string());
378        };
379
380        let outcome = 'ctrl: loop {
381            tokio::select! {
382                biased;
383                maybe = control_rx.recv() => {
384                    match maybe {
385                        Some(Ok(frame)) if frame.topic == terminate_topic => {
386                            halt(&task);
387                            let _ = (&mut done_rx).await;
388                            break 'ctrl LoopOutcome::Terminate;
389                        }
390                        Some(Ok(frame)) if frame.topic == "xs.stopping" => {
391                            halt(&task);
392                            let _ = (&mut done_rx).await;
393                            break 'ctrl LoopOutcome::Shutdown;
394                        }
395                        Some(Ok(frame)) if frame.topic == spawn_topic => {
396                            if let Some(hash) = frame.hash.clone() {
397                                if let Ok(mut reader) = store.cas_reader(hash).await {
398                                    let mut script = String::new();
399                                    if reader.read_to_string(&mut script).await.is_ok() {
400                                        // Clone the prepared base; the read and write builtins
401                                        // are already on it, so a hot-replace never re-registers
402                                        // builtins.
403                                        let mut new_engine = match specialize(&base, &store, frame.id) {
404                                            Ok(e) => e,
405                                            Err(_) => continue,
406                                        };
407                                        match nu::parse_config(&mut new_engine, &script) {
408                                            Ok(cfg) => {
409                                                let opts: ServiceScriptOptions = cfg.deserialize_options().unwrap_or_default();
410                                                let interrupt = Arc::new(AtomicBool::new(false));
411                                                new_engine.state.set_signals(Signals::new(interrupt.clone()));
412
413                                                halt(&task);
414                                                let _ = (&mut done_rx).await;
415
416                                                let new_task = Task {
417                                                    id: frame.id,
418                                                    run_closure: cfg.run_closure,
419                                                    return_options: opts.return_options,
420                                                    duplex: opts.duplex.unwrap_or(false),
421                                                    restart: opts.restart.unwrap_or_default(),
422                                                    engine: new_engine,
423                                                };
424
425                                                break 'ctrl LoopOutcome::Update(Box::new(new_task), frame.id);
426                                            }
427                                            Err(e) => {
428                                                let _ = emit_event(
429                                                    &store,
430                                                    &loop_ctx,
431                                                    frame.id,
432                                                    None,
433                                                    ServiceEventKind::ParseError { message: e.to_string() },
434                                                );
435                                            }
436                                        }
437                                    }
438                                }
439                            }
440                        }
441                        // The control read fell behind the live stream and
442                        // missed frames, maybe a .term: stop the service.
443                        Some(Err(err)) => {
444                            halt(&task);
445                            let _ = (&mut done_rx).await;
446                            break 'ctrl LoopOutcome::Error(format!("control read {err}"));
447                        }
448                        Some(Ok(_)) => {}
449                        None => break 'ctrl LoopOutcome::Error("control".into()),
450                    }
451                }
452                res = &mut done_rx => {
453                    if let Some(err) = input_lag.lock().unwrap().take() {
454                        break 'ctrl LoopOutcome::Error(format!("send read {err}"));
455                    }
456                    break 'ctrl match res.unwrap_or(Err("thread failed".into())) {
457                        Ok(()) => match task.restart {
458                            RestartPolicy::OnSuccess => LoopOutcome::Continue,
459                            RestartPolicy::Never => LoopOutcome::Finished,
460                        },
461                        Err(e) => LoopOutcome::Error(e),
462                    };
463                }
464            }
465        };
466
467        let reason: StopReason = (&outcome).into();
468        // Pre-emit a Stopped frame only when it maps to a real lifecycle
469        // topic:
470        //   Update    -> .replaced (transient, successor coming)
471        //   Terminate -> .fin.term
472        //   Error     -> .fin.error
473        //   Finished  -> .fin.ok only under restart: never. Continue
474        //                auto-restarts, so it is not a terminal stop and
475        //                gets no frame.
476        //   Shutdown  -> no frame: the post-loop ServiceEventKind::Shutdown
477        //                emits the single .stopped frame for the xs-stopping
478        //                path.
479        match &outcome {
480            LoopOutcome::Continue | LoopOutcome::Shutdown => {}
481            _ => {
482                let _ = emit_event(
483                    &store,
484                    &loop_ctx,
485                    task.id,
486                    task.return_options.as_ref(),
487                    ServiceEventKind::Stopped(reason.clone()),
488                );
489            }
490        }
491
492        match outcome {
493            LoopOutcome::Continue => {
494                tokio::time::sleep(Duration::from_secs(1)).await;
495                if let Ok(event) = emit_event(
496                    &store,
497                    &loop_ctx,
498                    task.id,
499                    task.return_options.as_ref(),
500                    ServiceEventKind::Running,
501                ) {
502                    start_id = event.frame.id;
503                }
504            }
505            LoopOutcome::Update(new_task, _) => {
506                task = *new_task;
507                if let Ok(event) = emit_event(
508                    &store,
509                    &loop_ctx,
510                    task.id,
511                    task.return_options.as_ref(),
512                    ServiceEventKind::Running,
513                ) {
514                    start_id = event.frame.id;
515                }
516            }
517            LoopOutcome::Finished | LoopOutcome::Terminate | LoopOutcome::Error(_) => {
518                break;
519            }
520            LoopOutcome::Shutdown => {
521                let _ = emit_event(
522                    &store,
523                    &loop_ctx,
524                    task.id,
525                    task.return_options.as_ref(),
526                    ServiceEventKind::Shutdown,
527                );
528                break;
529            }
530        }
531    }
532}
533
534async fn build_input_pipeline(
535    store: Store,
536    loop_ctx: &ServiceLoop,
537    task: &Task,
538    mut rx: tokio::sync::mpsc::Receiver<crate::store::ReadItem>,
539    lag: Arc<std::sync::Mutex<Option<String>>>,
540    stop: CancellationToken,
541) -> PipelineData {
542    let topic = format!("{loop_topic}.send", loop_topic = loop_ctx.topic);
543    // The iterator is pulled from the service's own thread, off the runtime,
544    // so it can block on the runtime: until the next frame, or the stop.
545    let handle = tokio::runtime::Handle::current();
546    let iter = std::iter::from_fn(move || loop {
547        let next = handle.block_on(async {
548            tokio::select! {
549                biased;
550                _ = stop.cancelled() => None,
551                item = rx.recv() => item,
552            }
553        });
554        match next? {
555            // The send read fell behind the live stream: the service's input
556            // has a gap. The input ends here and the run loop stops the
557            // service with the error (ADR 0008). An error item in the stream
558            // would not do it: `each` turns it into a value and the run
559            // finishes as if nothing happened.
560            Err(err) => {
561                *lag.lock().unwrap() = Some(err.to_string());
562                return None;
563            }
564            Ok(frame) => {
565                if frame.topic == topic {
566                    if let Some(hash) = frame.hash {
567                        if let Ok(bytes) = store.cas_read_sync(&hash) {
568                            if let Ok(content) = String::from_utf8(bytes) {
569                                return Some(content);
570                            }
571                        }
572                    }
573                }
574            }
575        }
576    });
577
578    ByteStream::from_iter(
579        iter,
580        Span::unknown(),
581        task.engine.state.signals().clone(),
582        ByteStreamType::Unknown,
583    )
584    .into()
585}
586
587fn spawn_thread(
588    store: Store,
589    loop_ctx: ServiceLoop,
590    mut task: Task,
591    input_pipeline: PipelineData,
592    done_tx: tokio::sync::oneshot::Sender<Result<(), String>>,
593) {
594    let handle = tokio::runtime::Handle::current();
595    std::thread::spawn(move || {
596        let res = run_pipeline(&handle, &store, &loop_ctx, &mut task, input_pipeline);
597        let _ = done_tx.send(res);
598    });
599}
600
601fn run_pipeline(
602    handle: &tokio::runtime::Handle,
603    store: &Store,
604    loop_ctx: &ServiceLoop,
605    task: &mut Task,
606    input_pipeline: PipelineData,
607) -> Result<(), String> {
608    let exec_data = task
609        .engine
610        .run_closure_in_job_with_exit(
611            &task.run_closure,
612            vec![],
613            Some(input_pipeline),
614            task.id.to_string(),
615        )
616        .map_err(|e| format_err(&task.engine, &e))?;
617    let pipeline = exec_data.body;
618
619    let suffix = task
620        .return_options
621        .as_ref()
622        .and_then(|o| o.suffix.clone())
623        .unwrap_or_else(|| ".recv".into());
624    let use_cas = task
625        .return_options
626        .as_ref()
627        .and_then(|o| o.target.as_deref())
628        .is_some_and(|t| t == "cas");
629
630    let emit = |event| {
631        handle.block_on(async {
632            let _ = emit_event(
633                store,
634                loop_ctx,
635                task.id,
636                task.return_options.as_ref(),
637                event,
638            );
639        });
640    };
641
642    match pipeline {
643        PipelineData::Empty => {}
644        PipelineData::Value(value, _) => {
645            if let Some(event) = value_to_event(&value, &suffix, use_cas)? {
646                emit(event);
647            }
648        }
649        PipelineData::ListStream(mut stream, _) => {
650            while let Some(value) = stream.next_value() {
651                if let Some(event) = value_to_event(&value, &suffix, use_cas)? {
652                    emit(event);
653                }
654            }
655        }
656        PipelineData::ByteStream(stream, _) => {
657            if let Some(mut reader) = stream.reader() {
658                let mut buf = [0u8; 8192];
659                loop {
660                    match reader.read(&mut buf) {
661                        Ok(0) => break,
662                        Ok(n) => {
663                            emit(ServiceEventKind::Recv {
664                                suffix: suffix.clone(),
665                                data: buf[..n].to_vec(),
666                            });
667                        }
668                        Err(_) => break,
669                    }
670                }
671            }
672        }
673    }
674
675    // The pipeline drained; now check its externals' exit statuses, as nu's
676    // pipefail does. A non-zero exit is an error, not a clean completion.
677    // SIGPIPE is not an error. When the job was interrupted (term, replace,
678    // xs.stopping) its externals were killed on purpose, so skip the check.
679    if task.engine.state.signals().interrupted() {
680        return Ok(());
681    }
682    nu_protocol::process::check_exit_status_future(exec_data.exit)
683        .map_err(|e| format_err(&task.engine, &e))
684}
685
686fn format_err(engine: &nu::Engine, e: &nu_protocol::ShellError) -> String {
687    let working_set = nu_protocol::engine::StateWorkingSet::new(&engine.state);
688    nu_protocol::format_cli_error(None, &working_set, e, None)
689}
690
691pub(crate) fn value_to_event(
692    value: &Value,
693    suffix: &str,
694    use_cas: bool,
695) -> Result<Option<ServiceEventKind>, String> {
696    match value {
697        Value::Nothing { .. } => Ok(None),
698        Value::Record { .. } if !use_cas => Ok(Some(ServiceEventKind::RecvMeta {
699            suffix: suffix.to_string(),
700            meta: value_to_json(value),
701        })),
702        _ if use_cas => {
703            let data = match value {
704                Value::String { val, .. } => val.as_bytes().to_vec(),
705                Value::Binary { val, .. } => val.to_vec(),
706                _ => value_to_json(value).to_string().into_bytes(),
707            };
708            Ok(Some(ServiceEventKind::Recv {
709                suffix: suffix.to_string(),
710                data,
711            }))
712        }
713        _ => Err(format!(
714            "Service output must be a record when target is not \"cas\"; got {}. \
715             Set return_options.target to \"cas\" for non-record output.",
716            value.get_type()
717        )),
718    }
719}