Skip to main content

xs/processor/service/
service.rs

1use scru128::Scru128Id;
2use tokio::task::JoinHandle;
3
4use nu_protocol::{ByteStream, ByteStreamType, PipelineData, Signals, Span, Value};
5use std::io::Read;
6use std::sync::atomic::AtomicBool;
7use std::sync::Arc;
8use std::time::Duration;
9use tokio::io::AsyncReadExt;
10
11use crate::nu;
12use crate::nu::{value_to_json, ReturnOptions};
13use crate::store::{FollowOption, Frame, ReadOptions, Store};
14use serde_json::json;
15
16/// What the run loop does when the pipeline drains cleanly. An error always
17/// stops the service with `.fin.error`, whatever the policy.
18#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Deserialize, Default)]
19#[serde(rename_all = "kebab-case")]
20pub enum RestartPolicy {
21    /// Wait one second and run the closure again. No lifecycle frame.
22    #[default]
23    OnSuccess,
24    /// Emit `.fin.ok` and stop. The service stays down across boots.
25    Never,
26}
27
28#[derive(Clone, Debug, serde::Deserialize, Default)]
29pub struct ServiceScriptOptions {
30    pub duplex: Option<bool>,
31    pub restart: Option<RestartPolicy>,
32    pub return_options: Option<ReturnOptions>,
33}
34
35#[derive(Clone)]
36pub struct ServiceLoop {
37    pub topic: String,
38}
39
40#[derive(Clone)]
41pub struct Task {
42    pub id: Scru128Id,
43    pub run_closure: nu_protocol::engine::Closure,
44    pub return_options: Option<ReturnOptions>,
45    pub duplex: bool,
46    pub restart: RestartPolicy,
47    pub engine: nu::Engine,
48}
49
50#[cfg_attr(not(test), allow(dead_code))]
51#[derive(Debug, Clone)]
52pub enum ServiceEventKind {
53    Running,
54    /// output frame flushed; payload is raw bytes stored in CAS
55    Recv {
56        suffix: String,
57        data: Vec<u8>,
58    },
59    /// output frame flushed; payload is a JSON record stored as frame metadata
60    RecvMeta {
61        suffix: String,
62        meta: serde_json::Value,
63    },
64    Stopped(StopReason),
65    ParseError {
66        message: String,
67    },
68    Shutdown,
69}
70
71#[cfg_attr(not(test), allow(dead_code))]
72#[derive(Debug, Clone)]
73pub struct ServiceEvent {
74    pub kind: ServiceEventKind,
75    pub frame: Frame,
76}
77
78#[cfg_attr(not(test), allow(dead_code))]
79#[derive(Debug, Clone)]
80pub enum StopReason {
81    Finished,
82    Error { message: String },
83    Terminate,
84    Shutdown,
85    Update { update_id: Scru128Id },
86}
87
88pub(crate) fn emit_event(
89    store: &Store,
90    loop_ctx: &ServiceLoop,
91    source_id: Scru128Id,
92    return_opts: Option<&ReturnOptions>,
93    kind: ServiceEventKind,
94) -> Result<ServiceEvent, crate::Error> {
95    let frame = match &kind {
96        ServiceEventKind::Running => store.append(
97            Frame::builder(format!("xs.service.{}.active", loop_ctx.topic))
98                .meta(json!({ "source_id": source_id.to_string() }))
99                .build(),
100        )?,
101
102        ServiceEventKind::Recv { suffix, data } => {
103            // Data topic: app namespace, unchanged.
104            let hash = store.cas_insert_bytes_sync(data)?;
105            store.append(
106                Frame::builder(format!(
107                    "{topic}{suffix}",
108                    topic = loop_ctx.topic,
109                    suffix = suffix
110                ))
111                .hash(hash)
112                .maybe_ttl(return_opts.and_then(|o| o.ttl.clone()))
113                .meta(json!({ "source_id": source_id.to_string() }))
114                .build(),
115            )?
116        }
117
118        ServiceEventKind::RecvMeta { suffix, meta } => {
119            // Data topic: app namespace, unchanged.
120            let mut merged = meta.clone();
121            merged["source_id"] = json!(source_id.to_string());
122            store.append(
123                Frame::builder(format!(
124                    "{topic}{suffix}",
125                    topic = loop_ctx.topic,
126                    suffix = suffix
127                ))
128                .maybe_ttl(return_opts.and_then(|o| o.ttl.clone()))
129                .meta(merged)
130                .build(),
131            )?
132        }
133
134        ServiceEventKind::Stopped(reason) => {
135            // Per ADR 0005:
136            //   Finished  -> .fin.ok       (terminal, won't restart)
137            //   Error     -> .fin.error    (terminal)
138            //   Terminate -> .fin.term     (terminal)
139            //   Update    -> .replaced     (transient; successor coming)
140            //   Shutdown  -> not a lifecycle frame on its own; the server-
141            //                shutdown ack is the separate ServiceEventKind::
142            //                Shutdown emission below. Skip here.
143            let event_suffix = match reason {
144                StopReason::Finished => "fin.ok",
145                StopReason::Error { .. } => "fin.error",
146                StopReason::Terminate => "fin.term",
147                StopReason::Update { .. } => "replaced",
148                StopReason::Shutdown => {
149                    // No frame: the run_loop emits ServiceEventKind::Shutdown
150                    // (which becomes the .stopped frame) for the xs-stopping
151                    // path; emitting here would double up.
152                    return Ok(ServiceEvent {
153                        kind,
154                        frame: Frame::builder("").build(),
155                    });
156                }
157            };
158            let mut meta = json!({
159                "source_id": source_id.to_string(),
160            });
161            if let StopReason::Update { update_id } = reason {
162                meta["update_id"] = json!(update_id.to_string());
163            }
164            if let StopReason::Error { message } = reason {
165                meta["message"] = json!(message);
166            }
167            store.append(
168                Frame::builder(format!("xs.service.{}.{event_suffix}", loop_ctx.topic))
169                    .meta(meta)
170                    .build(),
171            )?
172        }
173
174        ServiceEventKind::ParseError { message } => store.append(
175            Frame::builder(format!("xs.service.{}.invalid", loop_ctx.topic))
176                .meta(json!({
177                    "source_id": source_id.to_string(),
178                    "reason": message,
179                }))
180                .build(),
181        )?,
182
183        ServiceEventKind::Shutdown => store.append(
184            Frame::builder(format!("xs.service.{}.stopped", loop_ctx.topic))
185                .meta(json!({ "source_id": source_id.to_string() }))
186                .build(),
187        )?,
188    };
189
190    Ok(ServiceEvent { kind, frame })
191}
192
193pub fn spawn(store: Store, spawn_frame: Frame) -> JoinHandle<()> {
194    tokio::spawn(async move { run(store, spawn_frame).await })
195}
196
197/// Clone the prepared base engine and specialize it for a `<name>.create`
198/// frame: load the modules visible as of that frame and stamp `service_id`.
199/// The read and write builtins already live on `base`, so neither the
200/// initial spawn nor a hot-replace re-registers builtins.
201fn specialize(
202    base: &nu::Engine,
203    store: &Store,
204    create_id: Scru128Id,
205) -> Result<nu::Engine, crate::Error> {
206    let mut engine = base.clone();
207    let modules = store.nu_modules_at(&create_id);
208    nu::load_modules(&mut engine.state, store, &modules)?;
209    engine.set_append_meta(&serde_json::json!({ "service_id": create_id.to_string() }));
210    Ok(engine)
211}
212
213async fn run(store: Store, spawn_frame: Frame) {
214    // Prepared once (nushell + stdlib + core + reads + Direct `.append`);
215    // cloned for the initial run and every hot-replace.
216    let base = match nu::prepared_base(&store, true) {
217        Ok(e) => e,
218        Err(_) => return,
219    };
220    let mut engine = match specialize(&base, &store, spawn_frame.id) {
221        Ok(e) => e,
222        Err(_) => return,
223    };
224
225    let hash = match spawn_frame.hash.clone() {
226        Some(h) => h,
227        None => return,
228    };
229    let mut reader = match store.cas_reader(hash).await {
230        Ok(r) => r,
231        Err(_) => return,
232    };
233    let mut script = String::new();
234    if reader.read_to_string(&mut script).await.is_err() {
235        return;
236    }
237
238    let loop_ctx = ServiceLoop {
239        // Topic is `xs.service.<name>.create`; strip both ends to get <name>.
240        topic: spawn_frame
241            .topic
242            .strip_prefix("xs.service.")
243            .and_then(|rest| rest.strip_suffix(".create"))
244            .unwrap_or(&spawn_frame.topic)
245            .to_string(),
246    };
247
248    let nu_config = match nu::parse_config(&mut engine, &script) {
249        Ok(cfg) => cfg,
250        Err(e) => {
251            let _ = emit_event(
252                &store,
253                &loop_ctx,
254                spawn_frame.id,
255                None,
256                ServiceEventKind::ParseError {
257                    message: e.to_string(),
258                },
259            );
260            return;
261        }
262    };
263    let opts: ServiceScriptOptions = nu_config.deserialize_options().unwrap_or_default();
264
265    // Create and set the interrupt signal on the engine state
266    let interrupt = Arc::new(AtomicBool::new(false));
267    engine.state.set_signals(Signals::new(interrupt.clone()));
268
269    let task = Task {
270        id: spawn_frame.id,
271        run_closure: nu_config.run_closure,
272        return_options: opts.return_options,
273        duplex: opts.duplex.unwrap_or(false),
274        restart: opts.restart.unwrap_or_default(),
275        engine,
276    };
277
278    run_loop(store, loop_ctx, task, base).await;
279}
280
281async fn run_loop(store: Store, loop_ctx: ServiceLoop, mut task: Task, base: nu::Engine) {
282    // Create the first start frame and set up a persistent control subscription
283    let start_event = emit_event(
284        &store,
285        &loop_ctx,
286        task.id,
287        task.return_options.as_ref(),
288        ServiceEventKind::Running,
289    )
290    .expect("failed to emit running event");
291    let mut start_id = start_event.frame.id;
292
293    let control_rx_options = ReadOptions::builder()
294        .follow(FollowOption::On)
295        .after(start_id)
296        .build();
297
298    let mut control_rx = store.read(control_rx_options);
299
300    enum LoopOutcome {
301        Continue,
302        Finished,
303        Update(Box<Task>, Scru128Id),
304        Terminate,
305        Shutdown,
306        Error(String),
307    }
308
309    impl core::fmt::Debug for LoopOutcome {
310        fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
311            match self {
312                LoopOutcome::Continue => write!(f, "Continue"),
313                LoopOutcome::Finished => write!(f, "Finished"),
314                LoopOutcome::Update(_, id) => f.debug_tuple("Update").field(id).finish(),
315                LoopOutcome::Terminate => write!(f, "Terminate"),
316                LoopOutcome::Shutdown => write!(f, "Shutdown"),
317                LoopOutcome::Error(e) => f.debug_tuple("Error").field(e).finish(),
318            }
319        }
320    }
321
322    impl From<&LoopOutcome> for StopReason {
323        fn from(value: &LoopOutcome) -> Self {
324            match value {
325                LoopOutcome::Continue | LoopOutcome::Finished => StopReason::Finished,
326                LoopOutcome::Update(_, id) => StopReason::Update { update_id: *id },
327                LoopOutcome::Terminate => StopReason::Terminate,
328                LoopOutcome::Shutdown => StopReason::Shutdown,
329                LoopOutcome::Error(e) => StopReason::Error { message: e.clone() },
330            }
331        }
332    }
333
334    loop {
335        // Set when the send read falls behind the live stream: the input
336        // then ends, and the run below stops with that error instead of
337        // finishing (ADR 0008).
338        let input_lag: Arc<std::sync::Mutex<Option<String>>> = Arc::default();
339        let input_pipeline = if task.duplex {
340            let options = ReadOptions::builder()
341                .follow(FollowOption::On)
342                .after(start_id)
343                .build();
344            let send_rx = store.read(options);
345            build_input_pipeline(store.clone(), &loop_ctx, &task, send_rx, input_lag.clone()).await
346        } else {
347            PipelineData::empty()
348        };
349
350        let (done_tx, done_rx) = tokio::sync::oneshot::channel();
351        spawn_thread(
352            store.clone(),
353            loop_ctx.clone(),
354            task.clone(),
355            input_pipeline,
356            done_tx,
357        );
358
359        let terminate_topic = format!("xs.service.{}.term", loop_ctx.topic);
360        let spawn_topic = format!("xs.service.{}.create", loop_ctx.topic);
361        tokio::pin!(done_rx);
362
363        let outcome = 'ctrl: loop {
364            tokio::select! {
365                biased;
366                maybe = control_rx.recv() => {
367                    match maybe {
368                        Some(Ok(frame)) if frame.topic == terminate_topic => {
369                            task.engine.state.signals().trigger();
370                            task.engine.kill_job_by_name(&task.id.to_string());
371                            let _ = (&mut done_rx).await;
372                            break 'ctrl LoopOutcome::Terminate;
373                        }
374                        Some(Ok(frame)) if frame.topic == "xs.stopping" => {
375                            task.engine.state.signals().trigger();
376                            task.engine.kill_job_by_name(&task.id.to_string());
377                            let _ = (&mut done_rx).await;
378                            break 'ctrl LoopOutcome::Shutdown;
379                        }
380                        Some(Ok(frame)) if frame.topic == spawn_topic => {
381                            if let Some(hash) = frame.hash.clone() {
382                                if let Ok(mut reader) = store.cas_reader(hash).await {
383                                    let mut script = String::new();
384                                    if reader.read_to_string(&mut script).await.is_ok() {
385                                        // Clone the prepared base; the read and write builtins
386                                        // are already on it, so a hot-replace never re-registers
387                                        // builtins.
388                                        let mut new_engine = match specialize(&base, &store, frame.id) {
389                                            Ok(e) => e,
390                                            Err(_) => continue,
391                                        };
392                                        match nu::parse_config(&mut new_engine, &script) {
393                                            Ok(cfg) => {
394                                                let opts: ServiceScriptOptions = cfg.deserialize_options().unwrap_or_default();
395                                                let interrupt = Arc::new(AtomicBool::new(false));
396                                                new_engine.state.set_signals(Signals::new(interrupt.clone()));
397
398                                                task.engine.state.signals().trigger();
399                                                task.engine.kill_job_by_name(&task.id.to_string());
400                                                let _ = (&mut done_rx).await;
401
402                                                let new_task = Task {
403                                                    id: frame.id,
404                                                    run_closure: cfg.run_closure,
405                                                    return_options: opts.return_options,
406                                                    duplex: opts.duplex.unwrap_or(false),
407                                                    restart: opts.restart.unwrap_or_default(),
408                                                    engine: new_engine,
409                                                };
410
411                                                break 'ctrl LoopOutcome::Update(Box::new(new_task), frame.id);
412                                            }
413                                            Err(e) => {
414                                                let _ = emit_event(
415                                                    &store,
416                                                    &loop_ctx,
417                                                    frame.id,
418                                                    None,
419                                                    ServiceEventKind::ParseError { message: e.to_string() },
420                                                );
421                                            }
422                                        }
423                                    }
424                                }
425                            }
426                        }
427                        // The control read fell behind the live stream and
428                        // missed frames, maybe a .term: stop the service.
429                        Some(Err(err)) => {
430                            task.engine.state.signals().trigger();
431                            task.engine.kill_job_by_name(&task.id.to_string());
432                            let _ = (&mut done_rx).await;
433                            break 'ctrl LoopOutcome::Error(format!("control read {err}"));
434                        }
435                        Some(Ok(_)) => {}
436                        None => break 'ctrl LoopOutcome::Error("control".into()),
437                    }
438                }
439                res = &mut done_rx => {
440                    if let Some(err) = input_lag.lock().unwrap().take() {
441                        break 'ctrl LoopOutcome::Error(format!("send read {err}"));
442                    }
443                    break 'ctrl match res.unwrap_or(Err("thread failed".into())) {
444                        Ok(()) => match task.restart {
445                            RestartPolicy::OnSuccess => LoopOutcome::Continue,
446                            RestartPolicy::Never => LoopOutcome::Finished,
447                        },
448                        Err(e) => LoopOutcome::Error(e),
449                    };
450                }
451            }
452        };
453
454        let reason: StopReason = (&outcome).into();
455        // Pre-emit a Stopped frame only when it maps to a real lifecycle
456        // topic:
457        //   Update    -> .replaced (transient, successor coming)
458        //   Terminate -> .fin.term
459        //   Error     -> .fin.error
460        //   Finished  -> .fin.ok only under restart: never. Continue
461        //                auto-restarts, so it is not a terminal stop and
462        //                gets no frame.
463        //   Shutdown  -> no frame: the post-loop ServiceEventKind::Shutdown
464        //                emits the single .stopped frame for the xs-stopping
465        //                path.
466        match &outcome {
467            LoopOutcome::Continue | LoopOutcome::Shutdown => {}
468            _ => {
469                let _ = emit_event(
470                    &store,
471                    &loop_ctx,
472                    task.id,
473                    task.return_options.as_ref(),
474                    ServiceEventKind::Stopped(reason.clone()),
475                );
476            }
477        }
478
479        match outcome {
480            LoopOutcome::Continue => {
481                tokio::time::sleep(Duration::from_secs(1)).await;
482                if let Ok(event) = emit_event(
483                    &store,
484                    &loop_ctx,
485                    task.id,
486                    task.return_options.as_ref(),
487                    ServiceEventKind::Running,
488                ) {
489                    start_id = event.frame.id;
490                }
491            }
492            LoopOutcome::Update(new_task, _) => {
493                task = *new_task;
494                if let Ok(event) = emit_event(
495                    &store,
496                    &loop_ctx,
497                    task.id,
498                    task.return_options.as_ref(),
499                    ServiceEventKind::Running,
500                ) {
501                    start_id = event.frame.id;
502                }
503            }
504            LoopOutcome::Finished | LoopOutcome::Terminate | LoopOutcome::Error(_) => {
505                break;
506            }
507            LoopOutcome::Shutdown => {
508                let _ = emit_event(
509                    &store,
510                    &loop_ctx,
511                    task.id,
512                    task.return_options.as_ref(),
513                    ServiceEventKind::Shutdown,
514                );
515                break;
516            }
517        }
518    }
519}
520
521async fn build_input_pipeline(
522    store: Store,
523    loop_ctx: &ServiceLoop,
524    task: &Task,
525    rx: tokio::sync::mpsc::Receiver<crate::store::ReadItem>,
526    lag: Arc<std::sync::Mutex<Option<String>>>,
527) -> PipelineData {
528    let topic = format!("{loop_topic}.send", loop_topic = loop_ctx.topic);
529    let signals = task.engine.state.signals().clone();
530    let mut rx = rx;
531    let iter = std::iter::from_fn(move || loop {
532        if signals.interrupted() {
533            return None;
534        }
535
536        match rx.try_recv() {
537            // The send read fell behind the live stream: the service's input
538            // has a gap. The input ends here and the run loop stops the
539            // service with the error (ADR 0008). An error item in the stream
540            // would not do it: `each` turns it into a value and the run
541            // finishes as if nothing happened.
542            Ok(Err(err)) => {
543                *lag.lock().unwrap() = Some(err.to_string());
544                return None;
545            }
546            Ok(Ok(frame)) => {
547                if frame.topic == topic {
548                    if let Some(hash) = frame.hash {
549                        if let Ok(bytes) = store.cas_read_sync(&hash) {
550                            if let Ok(content) = String::from_utf8(bytes) {
551                                return Some(content);
552                            }
553                        }
554                    }
555                }
556            }
557            Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {
558                std::thread::sleep(std::time::Duration::from_millis(10));
559                continue;
560            }
561            Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => {
562                return None;
563            }
564        }
565    });
566
567    ByteStream::from_iter(
568        iter,
569        Span::unknown(),
570        task.engine.state.signals().clone(),
571        ByteStreamType::Unknown,
572    )
573    .into()
574}
575
576fn spawn_thread(
577    store: Store,
578    loop_ctx: ServiceLoop,
579    mut task: Task,
580    input_pipeline: PipelineData,
581    done_tx: tokio::sync::oneshot::Sender<Result<(), String>>,
582) {
583    let handle = tokio::runtime::Handle::current();
584    std::thread::spawn(move || {
585        let res = run_pipeline(&handle, &store, &loop_ctx, &mut task, input_pipeline);
586        let _ = done_tx.send(res);
587    });
588}
589
590fn run_pipeline(
591    handle: &tokio::runtime::Handle,
592    store: &Store,
593    loop_ctx: &ServiceLoop,
594    task: &mut Task,
595    input_pipeline: PipelineData,
596) -> Result<(), String> {
597    let exec_data = task
598        .engine
599        .run_closure_in_job_with_exit(
600            &task.run_closure,
601            vec![],
602            Some(input_pipeline),
603            task.id.to_string(),
604        )
605        .map_err(|e| format_err(&task.engine, &e))?;
606    let pipeline = exec_data.body;
607
608    let suffix = task
609        .return_options
610        .as_ref()
611        .and_then(|o| o.suffix.clone())
612        .unwrap_or_else(|| ".recv".into());
613    let use_cas = task
614        .return_options
615        .as_ref()
616        .and_then(|o| o.target.as_deref())
617        .is_some_and(|t| t == "cas");
618
619    let emit = |event| {
620        handle.block_on(async {
621            let _ = emit_event(
622                store,
623                loop_ctx,
624                task.id,
625                task.return_options.as_ref(),
626                event,
627            );
628        });
629    };
630
631    match pipeline {
632        PipelineData::Empty => {}
633        PipelineData::Value(value, _) => {
634            if let Some(event) = value_to_event(&value, &suffix, use_cas)? {
635                emit(event);
636            }
637        }
638        PipelineData::ListStream(mut stream, _) => {
639            while let Some(value) = stream.next_value() {
640                if let Some(event) = value_to_event(&value, &suffix, use_cas)? {
641                    emit(event);
642                }
643            }
644        }
645        PipelineData::ByteStream(stream, _) => {
646            if let Some(mut reader) = stream.reader() {
647                let mut buf = [0u8; 8192];
648                loop {
649                    match reader.read(&mut buf) {
650                        Ok(0) => break,
651                        Ok(n) => {
652                            emit(ServiceEventKind::Recv {
653                                suffix: suffix.clone(),
654                                data: buf[..n].to_vec(),
655                            });
656                        }
657                        Err(_) => break,
658                    }
659                }
660            }
661        }
662    }
663
664    // The pipeline drained; now check its externals' exit statuses, as nu's
665    // pipefail does. A non-zero exit is an error, not a clean completion.
666    // SIGPIPE is not an error. When the job was interrupted (term, replace,
667    // xs.stopping) its externals were killed on purpose, so skip the check.
668    if task.engine.state.signals().interrupted() {
669        return Ok(());
670    }
671    nu_protocol::process::check_exit_status_future(exec_data.exit)
672        .map_err(|e| format_err(&task.engine, &e))
673}
674
675fn format_err(engine: &nu::Engine, e: &nu_protocol::ShellError) -> String {
676    let working_set = nu_protocol::engine::StateWorkingSet::new(&engine.state);
677    nu_protocol::format_cli_error(None, &working_set, e, None)
678}
679
680pub(crate) fn value_to_event(
681    value: &Value,
682    suffix: &str,
683    use_cas: bool,
684) -> Result<Option<ServiceEventKind>, String> {
685    match value {
686        Value::Nothing { .. } => Ok(None),
687        Value::Record { .. } if !use_cas => Ok(Some(ServiceEventKind::RecvMeta {
688            suffix: suffix.to_string(),
689            meta: value_to_json(value),
690        })),
691        _ if use_cas => {
692            let data = match value {
693                Value::String { val, .. } => val.as_bytes().to_vec(),
694                Value::Binary { val, .. } => val.to_vec(),
695                _ => value_to_json(value).to_string().into_bytes(),
696            };
697            Ok(Some(ServiceEventKind::Recv {
698                suffix: suffix.to_string(),
699                data,
700            }))
701        }
702        _ => Err(format!(
703            "Service output must be a record when target is not \"cas\"; got {}. \
704             Set return_options.target to \"cas\" for non-record output.",
705            value.get_type()
706        )),
707    }
708}