Skip to main content

xs/processor/service/
service.rs

1use scru128::Scru128Id;
2use tokio::task::JoinHandle;
3
4use nu_protocol::{ByteStream, ByteStreamType, PipelineData, Signals, Span, Value};
5use std::io::Read;
6use std::sync::atomic::AtomicBool;
7use std::sync::Arc;
8use std::time::Duration;
9use tokio::io::AsyncReadExt;
10
11use crate::nu;
12use crate::nu::{value_to_json, ReturnOptions};
13use crate::store::{FollowOption, Frame, ReadOptions, Store};
14use serde_json::json;
15
16/// What the run loop does when the pipeline drains cleanly. An error always
17/// stops the service with `.fin.error`, whatever the policy.
18#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Deserialize, Default)]
19#[serde(rename_all = "kebab-case")]
20pub enum RestartPolicy {
21    /// Wait one second and run the closure again. No lifecycle frame.
22    #[default]
23    OnSuccess,
24    /// Emit `.fin.ok` and stop. The service stays down across boots.
25    Never,
26}
27
28#[derive(Clone, Debug, serde::Deserialize, Default)]
29pub struct ServiceScriptOptions {
30    pub duplex: Option<bool>,
31    pub restart: Option<RestartPolicy>,
32    pub return_options: Option<ReturnOptions>,
33}
34
35#[derive(Clone)]
36pub struct ServiceLoop {
37    pub topic: String,
38}
39
40#[derive(Clone)]
41pub struct Task {
42    pub id: Scru128Id,
43    pub run_closure: nu_protocol::engine::Closure,
44    pub return_options: Option<ReturnOptions>,
45    pub duplex: bool,
46    pub restart: RestartPolicy,
47    pub engine: nu::Engine,
48}
49
50#[cfg_attr(not(test), allow(dead_code))]
51#[derive(Debug, Clone)]
52pub enum ServiceEventKind {
53    Running,
54    /// output frame flushed; payload is raw bytes stored in CAS
55    Recv {
56        suffix: String,
57        data: Vec<u8>,
58    },
59    /// output frame flushed; payload is a JSON record stored as frame metadata
60    RecvMeta {
61        suffix: String,
62        meta: serde_json::Value,
63    },
64    Stopped(StopReason),
65    ParseError {
66        message: String,
67    },
68    Shutdown,
69}
70
71#[cfg_attr(not(test), allow(dead_code))]
72#[derive(Debug, Clone)]
73pub struct ServiceEvent {
74    pub kind: ServiceEventKind,
75    pub frame: Frame,
76}
77
78#[cfg_attr(not(test), allow(dead_code))]
79#[derive(Debug, Clone)]
80pub enum StopReason {
81    Finished,
82    Error { message: String },
83    Terminate,
84    Shutdown,
85    Update { update_id: Scru128Id },
86}
87
88pub(crate) fn emit_event(
89    store: &Store,
90    loop_ctx: &ServiceLoop,
91    source_id: Scru128Id,
92    return_opts: Option<&ReturnOptions>,
93    kind: ServiceEventKind,
94) -> Result<ServiceEvent, Box<dyn std::error::Error + Send + Sync>> {
95    let frame = match &kind {
96        ServiceEventKind::Running => store.append(
97            Frame::builder(format!("xs.service.{}.active", loop_ctx.topic))
98                .meta(json!({ "source_id": source_id.to_string() }))
99                .build(),
100        )?,
101
102        ServiceEventKind::Recv { suffix, data } => {
103            // Data topic: app namespace, unchanged.
104            let hash = store.cas_insert_bytes_sync(data)?;
105            store.append(
106                Frame::builder(format!(
107                    "{topic}{suffix}",
108                    topic = loop_ctx.topic,
109                    suffix = suffix
110                ))
111                .hash(hash)
112                .maybe_ttl(return_opts.and_then(|o| o.ttl.clone()))
113                .meta(json!({ "source_id": source_id.to_string() }))
114                .build(),
115            )?
116        }
117
118        ServiceEventKind::RecvMeta { suffix, meta } => {
119            // Data topic: app namespace, unchanged.
120            let mut merged = meta.clone();
121            merged["source_id"] = json!(source_id.to_string());
122            store.append(
123                Frame::builder(format!(
124                    "{topic}{suffix}",
125                    topic = loop_ctx.topic,
126                    suffix = suffix
127                ))
128                .maybe_ttl(return_opts.and_then(|o| o.ttl.clone()))
129                .meta(merged)
130                .build(),
131            )?
132        }
133
134        ServiceEventKind::Stopped(reason) => {
135            // Per ADR 0005:
136            //   Finished  -> .fin.ok       (terminal, won't restart)
137            //   Error     -> .fin.error    (terminal)
138            //   Terminate -> .fin.term     (terminal)
139            //   Update    -> .replaced     (transient; successor coming)
140            //   Shutdown  -> not a lifecycle frame on its own; the server-
141            //                shutdown ack is the separate ServiceEventKind::
142            //                Shutdown emission below. Skip here.
143            let event_suffix = match reason {
144                StopReason::Finished => "fin.ok",
145                StopReason::Error { .. } => "fin.error",
146                StopReason::Terminate => "fin.term",
147                StopReason::Update { .. } => "replaced",
148                StopReason::Shutdown => {
149                    // No frame: the run_loop emits ServiceEventKind::Shutdown
150                    // (which becomes the .stopped frame) for the xs-stopping
151                    // path; emitting here would double up.
152                    return Ok(ServiceEvent {
153                        kind,
154                        frame: Frame::builder("").build(),
155                    });
156                }
157            };
158            let mut meta = json!({
159                "source_id": source_id.to_string(),
160            });
161            if let StopReason::Update { update_id } = reason {
162                meta["update_id"] = json!(update_id.to_string());
163            }
164            if let StopReason::Error { message } = reason {
165                meta["message"] = json!(message);
166            }
167            store.append(
168                Frame::builder(format!("xs.service.{}.{event_suffix}", loop_ctx.topic))
169                    .meta(meta)
170                    .build(),
171            )?
172        }
173
174        ServiceEventKind::ParseError { message } => store.append(
175            Frame::builder(format!("xs.service.{}.invalid", loop_ctx.topic))
176                .meta(json!({
177                    "source_id": source_id.to_string(),
178                    "reason": message,
179                }))
180                .build(),
181        )?,
182
183        ServiceEventKind::Shutdown => store.append(
184            Frame::builder(format!("xs.service.{}.stopped", loop_ctx.topic))
185                .meta(json!({ "source_id": source_id.to_string() }))
186                .build(),
187        )?,
188    };
189
190    Ok(ServiceEvent { kind, frame })
191}
192
193pub fn spawn(store: Store, spawn_frame: Frame) -> JoinHandle<()> {
194    tokio::spawn(async move { run(store, spawn_frame).await })
195}
196
197/// Clone the prepared base engine and specialize it for a `<name>.create`
198/// frame: load the modules visible as of that frame and stamp `service_id`.
199/// The read and write builtins already live on `base`, so neither the
200/// initial spawn nor a hot-replace re-registers builtins.
201fn specialize(
202    base: &nu::Engine,
203    store: &Store,
204    create_id: Scru128Id,
205) -> Result<nu::Engine, Box<dyn std::error::Error + Send + Sync>> {
206    let mut engine = base.clone();
207    let modules = store.nu_modules_at(&create_id);
208    nu::load_modules(&mut engine.state, store, &modules)?;
209    engine.set_append_meta(&serde_json::json!({ "service_id": create_id.to_string() }));
210    Ok(engine)
211}
212
213async fn run(store: Store, spawn_frame: Frame) {
214    // Prepared once (nushell + stdlib + core + reads + Direct `.append`);
215    // cloned for the initial run and every hot-replace.
216    let base = match nu::prepared_base(&store, true) {
217        Ok(e) => e,
218        Err(_) => return,
219    };
220    let mut engine = match specialize(&base, &store, spawn_frame.id) {
221        Ok(e) => e,
222        Err(_) => return,
223    };
224
225    let hash = match spawn_frame.hash.clone() {
226        Some(h) => h,
227        None => return,
228    };
229    let mut reader = match store.cas_reader(hash).await {
230        Ok(r) => r,
231        Err(_) => return,
232    };
233    let mut script = String::new();
234    if reader.read_to_string(&mut script).await.is_err() {
235        return;
236    }
237
238    let loop_ctx = ServiceLoop {
239        // Topic is `xs.service.<name>.create`; strip both ends to get <name>.
240        topic: spawn_frame
241            .topic
242            .strip_prefix("xs.service.")
243            .and_then(|rest| rest.strip_suffix(".create"))
244            .unwrap_or(&spawn_frame.topic)
245            .to_string(),
246    };
247
248    let nu_config = match nu::parse_config(&mut engine, &script) {
249        Ok(cfg) => cfg,
250        Err(e) => {
251            let _ = emit_event(
252                &store,
253                &loop_ctx,
254                spawn_frame.id,
255                None,
256                ServiceEventKind::ParseError {
257                    message: e.to_string(),
258                },
259            );
260            return;
261        }
262    };
263    let opts: ServiceScriptOptions = nu_config.deserialize_options().unwrap_or_default();
264
265    // Create and set the interrupt signal on the engine state
266    let interrupt = Arc::new(AtomicBool::new(false));
267    engine.state.set_signals(Signals::new(interrupt.clone()));
268
269    let task = Task {
270        id: spawn_frame.id,
271        run_closure: nu_config.run_closure,
272        return_options: opts.return_options,
273        duplex: opts.duplex.unwrap_or(false),
274        restart: opts.restart.unwrap_or_default(),
275        engine,
276    };
277
278    run_loop(store, loop_ctx, task, base).await;
279}
280
281async fn run_loop(store: Store, loop_ctx: ServiceLoop, mut task: Task, base: nu::Engine) {
282    // Create the first start frame and set up a persistent control subscription
283    let start_event = emit_event(
284        &store,
285        &loop_ctx,
286        task.id,
287        task.return_options.as_ref(),
288        ServiceEventKind::Running,
289    )
290    .expect("failed to emit running event");
291    let mut start_id = start_event.frame.id;
292
293    let control_rx_options = ReadOptions::builder()
294        .follow(FollowOption::On)
295        .after(start_id)
296        .build();
297
298    let mut control_rx = store.read(control_rx_options);
299
300    enum LoopOutcome {
301        Continue,
302        Finished,
303        Update(Box<Task>, Scru128Id),
304        Terminate,
305        Shutdown,
306        Error(String),
307    }
308
309    impl core::fmt::Debug for LoopOutcome {
310        fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
311            match self {
312                LoopOutcome::Continue => write!(f, "Continue"),
313                LoopOutcome::Finished => write!(f, "Finished"),
314                LoopOutcome::Update(_, id) => f.debug_tuple("Update").field(id).finish(),
315                LoopOutcome::Terminate => write!(f, "Terminate"),
316                LoopOutcome::Shutdown => write!(f, "Shutdown"),
317                LoopOutcome::Error(e) => f.debug_tuple("Error").field(e).finish(),
318            }
319        }
320    }
321
322    impl From<&LoopOutcome> for StopReason {
323        fn from(value: &LoopOutcome) -> Self {
324            match value {
325                LoopOutcome::Continue | LoopOutcome::Finished => StopReason::Finished,
326                LoopOutcome::Update(_, id) => StopReason::Update { update_id: *id },
327                LoopOutcome::Terminate => StopReason::Terminate,
328                LoopOutcome::Shutdown => StopReason::Shutdown,
329                LoopOutcome::Error(e) => StopReason::Error { message: e.clone() },
330            }
331        }
332    }
333
334    loop {
335        let input_pipeline = if task.duplex {
336            let options = ReadOptions::builder()
337                .follow(FollowOption::On)
338                .after(start_id)
339                .build();
340            let send_rx = store.read(options);
341            build_input_pipeline(store.clone(), &loop_ctx, &task, send_rx).await
342        } else {
343            PipelineData::empty()
344        };
345
346        let (done_tx, done_rx) = tokio::sync::oneshot::channel();
347        spawn_thread(
348            store.clone(),
349            loop_ctx.clone(),
350            task.clone(),
351            input_pipeline,
352            done_tx,
353        );
354
355        let terminate_topic = format!("xs.service.{}.term", loop_ctx.topic);
356        let spawn_topic = format!("xs.service.{}.create", loop_ctx.topic);
357        tokio::pin!(done_rx);
358
359        let outcome = 'ctrl: loop {
360            tokio::select! {
361                biased;
362                maybe = control_rx.recv() => {
363                    match maybe {
364                        Some(frame) if frame.topic == terminate_topic => {
365                            task.engine.state.signals().trigger();
366                            task.engine.kill_job_by_name(&task.id.to_string());
367                            let _ = (&mut done_rx).await;
368                            break 'ctrl LoopOutcome::Terminate;
369                        }
370                        Some(frame) if frame.topic == "xs.stopping" => {
371                            task.engine.state.signals().trigger();
372                            task.engine.kill_job_by_name(&task.id.to_string());
373                            let _ = (&mut done_rx).await;
374                            break 'ctrl LoopOutcome::Shutdown;
375                        }
376                        Some(frame) if frame.topic == spawn_topic => {
377                            if let Some(hash) = frame.hash.clone() {
378                                if let Ok(mut reader) = store.cas_reader(hash).await {
379                                    let mut script = String::new();
380                                    if reader.read_to_string(&mut script).await.is_ok() {
381                                        // Clone the prepared base; the read and write builtins
382                                        // are already on it, so a hot-replace never re-registers
383                                        // builtins.
384                                        let mut new_engine = match specialize(&base, &store, frame.id) {
385                                            Ok(e) => e,
386                                            Err(_) => continue,
387                                        };
388                                        match nu::parse_config(&mut new_engine, &script) {
389                                            Ok(cfg) => {
390                                                let opts: ServiceScriptOptions = cfg.deserialize_options().unwrap_or_default();
391                                                let interrupt = Arc::new(AtomicBool::new(false));
392                                                new_engine.state.set_signals(Signals::new(interrupt.clone()));
393
394                                                task.engine.state.signals().trigger();
395                                                task.engine.kill_job_by_name(&task.id.to_string());
396                                                let _ = (&mut done_rx).await;
397
398                                                let new_task = Task {
399                                                    id: frame.id,
400                                                    run_closure: cfg.run_closure,
401                                                    return_options: opts.return_options,
402                                                    duplex: opts.duplex.unwrap_or(false),
403                                                    restart: opts.restart.unwrap_or_default(),
404                                                    engine: new_engine,
405                                                };
406
407                                                break 'ctrl LoopOutcome::Update(Box::new(new_task), frame.id);
408                                            }
409                                            Err(e) => {
410                                                let _ = emit_event(
411                                                    &store,
412                                                    &loop_ctx,
413                                                    frame.id,
414                                                    None,
415                                                    ServiceEventKind::ParseError { message: e.to_string() },
416                                                );
417                                            }
418                                        }
419                                    }
420                                }
421                            }
422                        }
423                        Some(_) => {}
424                        None => break 'ctrl LoopOutcome::Error("control".into()),
425                    }
426                }
427                res = &mut done_rx => {
428                    break 'ctrl match res.unwrap_or(Err("thread failed".into())) {
429                        Ok(()) => match task.restart {
430                            RestartPolicy::OnSuccess => LoopOutcome::Continue,
431                            RestartPolicy::Never => LoopOutcome::Finished,
432                        },
433                        Err(e) => LoopOutcome::Error(e),
434                    };
435                }
436            }
437        };
438
439        let reason: StopReason = (&outcome).into();
440        // Pre-emit a Stopped frame only when it maps to a real lifecycle
441        // topic:
442        //   Update    -> .replaced (transient, successor coming)
443        //   Terminate -> .fin.term
444        //   Error     -> .fin.error
445        //   Finished  -> .fin.ok only under restart: never. Continue
446        //                auto-restarts, so it is not a terminal stop and
447        //                gets no frame.
448        //   Shutdown  -> no frame: the post-loop ServiceEventKind::Shutdown
449        //                emits the single .stopped frame for the xs-stopping
450        //                path.
451        match &outcome {
452            LoopOutcome::Continue | LoopOutcome::Shutdown => {}
453            _ => {
454                let _ = emit_event(
455                    &store,
456                    &loop_ctx,
457                    task.id,
458                    task.return_options.as_ref(),
459                    ServiceEventKind::Stopped(reason.clone()),
460                );
461            }
462        }
463
464        match outcome {
465            LoopOutcome::Continue => {
466                tokio::time::sleep(Duration::from_secs(1)).await;
467                if let Ok(event) = emit_event(
468                    &store,
469                    &loop_ctx,
470                    task.id,
471                    task.return_options.as_ref(),
472                    ServiceEventKind::Running,
473                ) {
474                    start_id = event.frame.id;
475                }
476            }
477            LoopOutcome::Update(new_task, _) => {
478                task = *new_task;
479                if let Ok(event) = emit_event(
480                    &store,
481                    &loop_ctx,
482                    task.id,
483                    task.return_options.as_ref(),
484                    ServiceEventKind::Running,
485                ) {
486                    start_id = event.frame.id;
487                }
488            }
489            LoopOutcome::Finished | LoopOutcome::Terminate | LoopOutcome::Error(_) => {
490                break;
491            }
492            LoopOutcome::Shutdown => {
493                let _ = emit_event(
494                    &store,
495                    &loop_ctx,
496                    task.id,
497                    task.return_options.as_ref(),
498                    ServiceEventKind::Shutdown,
499                );
500                break;
501            }
502        }
503    }
504}
505
506async fn build_input_pipeline(
507    store: Store,
508    loop_ctx: &ServiceLoop,
509    task: &Task,
510    rx: tokio::sync::mpsc::Receiver<Frame>,
511) -> PipelineData {
512    let topic = format!("{loop_topic}.send", loop_topic = loop_ctx.topic);
513    let signals = task.engine.state.signals().clone();
514    let mut rx = rx;
515    let iter = std::iter::from_fn(move || loop {
516        if signals.interrupted() {
517            return None;
518        }
519
520        match rx.try_recv() {
521            Ok(frame) => {
522                if frame.topic == topic {
523                    if let Some(hash) = frame.hash {
524                        if let Ok(bytes) = store.cas_read_sync(&hash) {
525                            if let Ok(content) = String::from_utf8(bytes) {
526                                return Some(content);
527                            }
528                        }
529                    }
530                }
531            }
532            Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {
533                std::thread::sleep(std::time::Duration::from_millis(10));
534                continue;
535            }
536            Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => {
537                return None;
538            }
539        }
540    });
541
542    ByteStream::from_iter(
543        iter,
544        Span::unknown(),
545        task.engine.state.signals().clone(),
546        ByteStreamType::Unknown,
547    )
548    .into()
549}
550
551fn spawn_thread(
552    store: Store,
553    loop_ctx: ServiceLoop,
554    mut task: Task,
555    input_pipeline: PipelineData,
556    done_tx: tokio::sync::oneshot::Sender<Result<(), String>>,
557) {
558    let handle = tokio::runtime::Handle::current();
559    std::thread::spawn(move || {
560        let res = run_pipeline(&handle, &store, &loop_ctx, &mut task, input_pipeline);
561        let _ = done_tx.send(res);
562    });
563}
564
565fn run_pipeline(
566    handle: &tokio::runtime::Handle,
567    store: &Store,
568    loop_ctx: &ServiceLoop,
569    task: &mut Task,
570    input_pipeline: PipelineData,
571) -> Result<(), String> {
572    let exec_data = task
573        .engine
574        .run_closure_in_job_with_exit(
575            &task.run_closure,
576            vec![],
577            Some(input_pipeline),
578            task.id.to_string(),
579        )
580        .map_err(|e| format_err(&task.engine, &e))?;
581    let pipeline = exec_data.body;
582
583    let suffix = task
584        .return_options
585        .as_ref()
586        .and_then(|o| o.suffix.clone())
587        .unwrap_or_else(|| ".recv".into());
588    let use_cas = task
589        .return_options
590        .as_ref()
591        .and_then(|o| o.target.as_deref())
592        .is_some_and(|t| t == "cas");
593
594    let emit = |event| {
595        handle.block_on(async {
596            let _ = emit_event(
597                store,
598                loop_ctx,
599                task.id,
600                task.return_options.as_ref(),
601                event,
602            );
603        });
604    };
605
606    match pipeline {
607        PipelineData::Empty => {}
608        PipelineData::Value(value, _) => {
609            if let Some(event) = value_to_event(&value, &suffix, use_cas)? {
610                emit(event);
611            }
612        }
613        PipelineData::ListStream(mut stream, _) => {
614            while let Some(value) = stream.next_value() {
615                if let Some(event) = value_to_event(&value, &suffix, use_cas)? {
616                    emit(event);
617                }
618            }
619        }
620        PipelineData::ByteStream(stream, _) => {
621            if let Some(mut reader) = stream.reader() {
622                let mut buf = [0u8; 8192];
623                loop {
624                    match reader.read(&mut buf) {
625                        Ok(0) => break,
626                        Ok(n) => {
627                            emit(ServiceEventKind::Recv {
628                                suffix: suffix.clone(),
629                                data: buf[..n].to_vec(),
630                            });
631                        }
632                        Err(_) => break,
633                    }
634                }
635            }
636        }
637    }
638
639    // The pipeline drained; now check its externals' exit statuses, as nu's
640    // pipefail does. A non-zero exit is an error, not a clean completion.
641    // SIGPIPE is not an error. When the job was interrupted (term, replace,
642    // xs.stopping) its externals were killed on purpose, so skip the check.
643    if task.engine.state.signals().interrupted() {
644        return Ok(());
645    }
646    nu_protocol::process::check_exit_status_future(exec_data.exit)
647        .map_err(|e| format_err(&task.engine, &e))
648}
649
650fn format_err(engine: &nu::Engine, e: &nu_protocol::ShellError) -> String {
651    let working_set = nu_protocol::engine::StateWorkingSet::new(&engine.state);
652    nu_protocol::format_cli_error(None, &working_set, e, None)
653}
654
655pub(crate) fn value_to_event(
656    value: &Value,
657    suffix: &str,
658    use_cas: bool,
659) -> Result<Option<ServiceEventKind>, String> {
660    match value {
661        Value::Nothing { .. } => Ok(None),
662        Value::Record { .. } if !use_cas => Ok(Some(ServiceEventKind::RecvMeta {
663            suffix: suffix.to_string(),
664            meta: value_to_json(value),
665        })),
666        _ if use_cas => {
667            let data = match value {
668                Value::String { val, .. } => val.as_bytes().to_vec(),
669                Value::Binary { val, .. } => val.to_vec(),
670                _ => value_to_json(value).to_string().into_bytes(),
671            };
672            Ok(Some(ServiceEventKind::Recv {
673                suffix: suffix.to_string(),
674                data,
675            }))
676        }
677        _ => Err(format!(
678            "Service output must be a record when target is not \"cas\"; got {}. \
679             Set return_options.target to \"cas\" for non-record output.",
680            value.get_type()
681        )),
682    }
683}