Skip to main content

serve_celld/
lib.rs

1//! # serve-celld (experimental)
2//!
3//! Run a [serve](https://docs.rs/everruns-serve) app durably on
4//! [celld](https://github.com/denoland/celld), self-hosted distributed
5//! Durable Objects, part of the [Everruns](https://everruns.com) ecosystem.
6//!
7//! > **Experimental.** Like serve, this is a proof of concept with no
8//! > compatibility promise.
9//!
10//! On celld a Durable Object (a *cell*) has one owner node at a time and a
11//! replicated SQLite database that survives a move to another node. A
12//! container the cell supervises does not: a move, a node restart or a reset
13//! destroys it. So the agent runs in the container and the cell keeps what
14//! must survive:
15//!
16//! - the latest **snapshot** of serve's state (the session catalog, the
17//!   engine's canonical event log and the workspace), taken whenever the
18//!   app is idle;
19//! - a **journal** of the requests that arrived since, replayed into a
20//!   fresh container after the snapshot is restored.
21//!
22//! This crate is the container half. It serves serve's own `/v1` wire API
23//! and three routes only the cell calls:
24//!
25//! - `GET /celld/state`: `{"boot_id", "booted", "busy"}`. A new `boot_id`
26//!   tells the cell the container is fresh and must be restored. Never boots
27//!   the app.
28//! - `GET /celld/snapshot`: a tar of the data directory, with every SQLite
29//!   database copied through the online backup API so the copy is
30//!   consistent. `409` while a turn runs.
31//! - `PUT /celld/restore`: unpack a snapshot into the data directory. `409`
32//!   once the app has booted, so a restore can never race live state.
33//!
34//! serve boots on the first other request, after the cell had its chance to
35//! restore. The [JavaScript cell example](https://github.com/everruns/everruns/tree/main/examples/serve/celld/worker)
36//! decides when to snapshot, what to journal and when to replay.
37//!
38//! ```no_run
39//! use serve::prelude::*;
40//!
41//! #[agent]
42//! fn assistant() -> Agent {
43//!     Agent::builder()
44//!         .model("anthropic/claude-sonnet-5")
45//!         .instructions("Be brief.")
46//!         .build()
47//! }
48//!
49//! #[tokio::main]
50//! async fn main() -> serve::Result {
51//!     // No command (what the container runs) or `celld`: the celld contract
52//!     // on :8080. `dev`, `start`, `eval`, `manifest`, `deploy`: serve's own.
53//!     serve_celld::start(App::builder().discover().build()).await
54//! }
55//! ```
56
57#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))]
58
59use everruns::sqlite as rusqlite;
60use std::ffi::OsString;
61use std::io::Read;
62use std::net::SocketAddr;
63use std::path::{Path, PathBuf};
64use std::sync::Arc;
65
66use anyhow::{Context, anyhow, bail};
67use axum::body::{Body, Bytes};
68use axum::extract::{DefaultBodyLimit, Request, State};
69use axum::http::{StatusCode, header};
70use axum::response::{IntoResponse, Response};
71use axum::routing::{get, put};
72use axum::{Json, Router};
73use clap::Parser;
74use serde_json::{Value, json};
75use serve::{App, Mode, Server};
76use tokio::sync::{Mutex, OnceCell};
77use tower::ServiceExt;
78
79/// The port the container listens on, and the cell's `defaultPort`.
80pub const PORT: u16 = 8080;
81
82/// The largest snapshot `PUT /celld/restore` accepts.
83pub const MAX_SNAPSHOT_BYTES: usize = 512 * 1024 * 1024;
84
85/// The `celld` command line.
86#[derive(Parser, Debug)]
87#[command(about = "Serve this app as a celld container (experimental).")]
88struct Cli {
89    /// Port to listen on.
90    #[arg(long, env = "PORT", default_value_t = PORT)]
91    port: u16,
92    /// Run in serve's `dev` mode (simulator fallback, no secret check)
93    /// instead of `start`, to try the contract locally.
94    #[arg(long)]
95    dev: bool,
96}
97
98/// How [`router`] boots the app.
99#[derive(Clone, Debug)]
100pub struct Options {
101    /// serve's mode: [`Mode::Start`] in production.
102    pub mode: Mode,
103    /// Where serve keeps its state inside the container. Ephemeral by
104    /// design: the cell holds the durable copy. Defaults to
105    /// [`default_data_dir`].
106    pub data_dir: PathBuf,
107}
108
109impl Options {
110    /// Production defaults: `start` mode, state in [`default_data_dir`].
111    pub fn new(mode: Mode) -> Self {
112        Self {
113            mode,
114            data_dir: default_data_dir(),
115        }
116    }
117}
118
119/// `SERVE_DATA_DIR` when set, else `serve-celld` in the temp directory.
120pub fn default_data_dir() -> PathBuf {
121    std::env::var_os("SERVE_DATA_DIR")
122        .filter(|dir| !dir.is_empty())
123        .map(PathBuf::from)
124        .unwrap_or_else(|| std::env::temp_dir().join("serve-celld"))
125}
126
127/// Run the app: with no command, or with `celld`, serve the celld container
128/// contract; any other command is serve's (see [`serve::start`]).
129pub async fn start(app: App) -> serve::Result {
130    let mut args: Vec<OsString> = std::env::args_os().collect();
131    match args.get(1).and_then(|arg| arg.to_str()) {
132        None => {}
133        Some("celld") => {
134            args.remove(1);
135        }
136        Some(_) => return serve::start(app).await,
137    }
138    let cli = Cli::parse_from(args);
139    let options = Options::new(if cli.dev { Mode::Dev } else { Mode::Start });
140    let name = app.name().to_string();
141    let target = Target::new(app, options.clone())?;
142    let boot_id = target.boot_id.clone();
143    let router = target.router();
144
145    let addr = SocketAddr::from(([0, 0, 0, 0], cli.port));
146    let listener = tokio::net::TcpListener::bind(addr).await?;
147    println!(
148        "serve-celld · {name} · {:?} · boot {boot_id} · state in {} · listening on {addr}",
149        options.mode,
150        options.data_dir.display()
151    );
152    axum::serve(listener, router)
153        .with_graceful_shutdown(async {
154            let _ = tokio::signal::ctrl_c().await;
155        })
156        .await?;
157    Ok(())
158}
159
160/// The celld container contract around `app`: `/celld/*` and serve's `/v1`
161/// wire API.
162///
163/// The app is checked at once (secrets in `start`, every agent resolved), but
164/// the server that keeps state boots on the first request outside
165/// `/celld/*`, so the cell can restore a snapshot first.
166pub fn router(app: App, options: Options) -> serve::Result<Router> {
167    Ok(Target::new(app, options)?.router())
168}
169
170/// The state of one container process.
171#[derive(Clone)]
172struct Target {
173    app: App,
174    options: Arc<Options>,
175    /// New on every process start: how the cell tells a fresh container.
176    boot_id: String,
177    booted: Arc<OnceCell<Booted>>,
178    /// Serializes booting against restoring and snapshotting.
179    gate: Arc<Mutex<()>>,
180}
181
182struct Booted {
183    server: Server,
184    wire: Router,
185}
186
187impl Target {
188    fn new(app: App, options: Options) -> serve::Result<Self> {
189        // A throwaway in-memory boot: fails fast on a bad app.
190        drop(boot(app.clone(), &options, false)?);
191        Ok(Self {
192            app,
193            options: Arc::new(options),
194            boot_id: uuid::Uuid::now_v7().to_string(),
195            booted: Arc::new(OnceCell::new()),
196            gate: Arc::new(Mutex::new(())),
197        })
198    }
199
200    fn router(self) -> Router {
201        Router::new()
202            .route("/celld/state", get(state))
203            .route("/celld/snapshot", get(snapshot))
204            .route(
205                "/celld/restore",
206                put(restore).layer(DefaultBodyLimit::max(MAX_SNAPSHOT_BYTES)),
207            )
208            .fallback(forward)
209            .with_state(self)
210    }
211
212    /// The stateful server, booted on first use.
213    async fn booted(&self) -> Result<&Booted, Response> {
214        if let Some(booted) = self.booted.get() {
215            return Ok(booted);
216        }
217        let _gate = self.gate.lock().await;
218        self.booted
219            .get_or_try_init(|| async {
220                std::fs::create_dir_all(&self.options.data_dir)?;
221                let server = boot(self.app.clone(), &self.options, true)?;
222                server.spawn_schedules();
223                let wire = server.router();
224                Ok::<_, serve::Error>(Booted { server, wire })
225            })
226            .await
227            .map_err(|err| {
228                eprintln!("serve-celld: boot failed: {err:#}");
229                problem(StatusCode::INTERNAL_SERVER_ERROR, format!("{err:#}"))
230            })
231    }
232
233    fn busy(&self) -> bool {
234        self.booted.get().is_some_and(|booted| booted.server.busy())
235    }
236}
237
238/// Boot serve. `persistent: false` boots in memory.
239fn boot(app: App, options: &Options, persistent: bool) -> serve::Result<Server> {
240    let workspace = options.data_dir.join("workspace");
241    let mut builder =
242        Server::builder(app, options.mode).microvm(move |agent| microvm(agent, &workspace));
243    if persistent {
244        builder = builder.data_dir(options.data_dir.clone());
245    }
246    builder.build()
247}
248
249/// `[sandbox] kind = "microvm"` on celld: a real shell and file tools over a
250/// workspace inside the data directory, so snapshots carry it.
251///
252/// Decision: no kernel containment inside the container. The container is
253/// the boundary, with celld's fence around it; for code you did not write,
254/// give the container class a runtime with its own kernel (`runsc`,
255/// `kata`).
256fn microvm(agent: everruns::AgentBuilder, workspace: &Path) -> everruns::AgentBuilder {
257    agent
258        .workspace(workspace)
259        .workspace_policy(everruns::WorkspacePolicy::read_write())
260        .capability(everruns::HostShell::new().containment(everruns::ContainmentMode::FullAccess))
261}
262
263/// `GET /celld/state`. Never boots the app.
264async fn state(State(target): State<Target>) -> Json<Value> {
265    Json(json!({
266        "boot_id": target.boot_id,
267        "booted": target.booted.get().is_some(),
268        "busy": target.busy(),
269    }))
270}
271
272/// `GET /celld/snapshot`: the data directory as a tar.
273async fn snapshot(State(target): State<Target>) -> Response {
274    let _gate = target.gate.lock().await;
275    if target.busy() {
276        return problem(
277            StatusCode::CONFLICT,
278            "a turn is running; snapshot when idle".into(),
279        );
280    }
281    let dir = target.options.data_dir.clone();
282    match tokio::task::spawn_blocking(move || snapshot_dir(&dir)).await {
283        Ok(Ok(bytes)) => (
284            [(header::CONTENT_TYPE, "application/x-tar")],
285            Body::from(bytes),
286        )
287            .into_response(),
288        Ok(Err(err)) => {
289            eprintln!("serve-celld: snapshot failed: {err:#}");
290            problem(StatusCode::INTERNAL_SERVER_ERROR, format!("{err:#}"))
291        }
292        Err(err) => problem(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()),
293    }
294}
295
296/// `PUT /celld/restore`: replace the data directory with a snapshot.
297async fn restore(State(target): State<Target>, body: Bytes) -> Response {
298    let _gate = target.gate.lock().await;
299    if target.booted.get().is_some() {
300        return problem(
301            StatusCode::CONFLICT,
302            "the app has booted; restore only into a fresh container".into(),
303        );
304    }
305    let dir = target.options.data_dir.clone();
306    match tokio::task::spawn_blocking(move || restore_dir(&dir, &body)).await {
307        Ok(Ok(files)) => Json(json!({ "restored_files": files })).into_response(),
308        Ok(Err(err)) => problem(StatusCode::BAD_REQUEST, format!("{err:#}")),
309        Err(err) => problem(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()),
310    }
311}
312
313/// Everything else: serve's `/v1` wire API and `/health`.
314async fn forward(State(target): State<Target>, request: Request<Body>) -> Response {
315    match target.booted().await {
316        Ok(booted) => match booted.wire.clone().oneshot(request).await {
317            Ok(response) => response,
318            Err(never) => match never {},
319        },
320        Err(response) => response,
321    }
322}
323
324/// Tar `dir`. SQLite databases go through the online backup API, which reads
325/// a consistent copy while serve keeps them open; their `-wal`, `-shm` and
326/// `-journal` siblings and lock files are left out, since the copy already
327/// holds every committed page. Other regular files are copied as they are;
328/// symbolic links and special files are skipped.
329pub fn snapshot_dir(dir: &Path) -> anyhow::Result<Vec<u8>> {
330    let mut builder = tar::Builder::new(Vec::new());
331    builder.mode(tar::HeaderMode::Deterministic);
332    if dir.is_dir() {
333        let scratch = tempfile_dir()?;
334        append_dir(&mut builder, dir, Path::new(""), &scratch)?;
335        let _ = std::fs::remove_dir_all(&scratch);
336    }
337    Ok(builder.into_inner()?)
338}
339
340fn append_dir(
341    builder: &mut tar::Builder<Vec<u8>>,
342    root: &Path,
343    rel: &Path,
344    scratch: &Path,
345) -> anyhow::Result<()> {
346    let mut entries: Vec<_> = std::fs::read_dir(root.join(rel))?
347        .collect::<Result<_, _>>()
348        .with_context(|| format!("read {}", root.join(rel).display()))?;
349    entries.sort_by_key(|entry| entry.file_name());
350    for entry in entries {
351        let name = entry.file_name();
352        let path = rel.join(&name);
353        let kind = entry.file_type()?;
354        let text = name.to_string_lossy();
355        if text.ends_with(".lock") || is_sqlite_sidecar(&text) {
356            continue;
357        }
358        if kind.is_dir() {
359            builder.append_dir(&path, entry.path())?;
360            append_dir(builder, root, &path, scratch)?;
361        } else if kind.is_file() {
362            if is_sqlite(&entry.path())? {
363                let copy = scratch.join(uuid::Uuid::now_v7().to_string());
364                rusqlite::backup(entry.path(), &copy)
365                    .with_context(|| format!("back up {}", entry.path().display()))?;
366                builder.append_path_with_name(&copy, &path)?;
367                let _ = std::fs::remove_file(&copy);
368            } else {
369                builder.append_path_with_name(entry.path(), &path)?;
370            }
371        }
372    }
373    Ok(())
374}
375
376fn is_sqlite_sidecar(name: &str) -> bool {
377    ["-wal", "-shm", "-journal"]
378        .iter()
379        .any(|suffix| name.ends_with(suffix))
380}
381
382fn is_sqlite(path: &Path) -> anyhow::Result<bool> {
383    let mut magic = [0u8; 16];
384    let mut file = std::fs::File::open(path)?;
385    let read = file.read(&mut magic)?;
386    Ok(read == 16 && &magic == b"SQLite format 3\0")
387}
388
389fn tempfile_dir() -> anyhow::Result<PathBuf> {
390    let dir = std::env::temp_dir().join(format!("serve-celld-snap-{}", uuid::Uuid::now_v7()));
391    std::fs::create_dir_all(&dir)?;
392    Ok(dir)
393}
394
395/// Replace `dir` with the snapshot in `bytes`. Accepts only regular files
396/// and directories with relative paths that stay inside `dir`; anything else
397/// fails the whole restore before `dir` is touched. Returns the file count.
398pub fn restore_dir(dir: &Path, bytes: &[u8]) -> anyhow::Result<usize> {
399    // Validate first, so a bad snapshot leaves the directory as it was.
400    let mut archive = tar::Archive::new(bytes);
401    let mut files = 0;
402    for entry in archive.entries()? {
403        let entry = entry?;
404        let path = entry.path()?.into_owned();
405        let safe = path
406            .components()
407            .all(|part| matches!(part, std::path::Component::Normal(_)));
408        if !safe {
409            bail!(
410                "snapshot entry `{}` leaves the data directory",
411                path.display()
412            );
413        }
414        match entry.header().entry_type() {
415            tar::EntryType::Regular => files += 1,
416            tar::EntryType::Directory => {}
417            other => bail!(
418                "snapshot entry `{}` is a {other:?}, not a file or directory",
419                path.display()
420            ),
421        }
422    }
423    if dir.exists() {
424        std::fs::remove_dir_all(dir).with_context(|| format!("clear {}", dir.display()))?;
425    }
426    std::fs::create_dir_all(dir)?;
427    let mut archive = tar::Archive::new(bytes);
428    archive.set_preserve_permissions(false);
429    for entry in archive.entries()? {
430        let mut entry = entry?;
431        if !entry.unpack_in(dir)? {
432            return Err(anyhow!("snapshot entry escaped {}", dir.display()));
433        }
434    }
435    Ok(files)
436}
437
438/// RFC 9457 problem details, as serve's own routes answer.
439fn problem(status: StatusCode, detail: String) -> Response {
440    let body = json!({
441        "title": status.canonical_reason().unwrap_or("Error"),
442        "status": status.as_u16(),
443        "detail": detail,
444    });
445    (
446        status,
447        [(header::CONTENT_TYPE, "application/problem+json")],
448        body.to_string(),
449    )
450        .into_response()
451}
452
453#[cfg(test)]
454mod tests {
455    use super::*;
456
457    fn files(dir: &Path) -> Vec<String> {
458        let mut out = Vec::new();
459        fn walk(root: &Path, dir: &Path, out: &mut Vec<String>) {
460            for entry in std::fs::read_dir(dir).unwrap() {
461                let entry = entry.unwrap();
462                if entry.file_type().unwrap().is_dir() {
463                    walk(root, &entry.path(), out);
464                } else {
465                    let rel = entry.path().strip_prefix(root).unwrap().to_path_buf();
466                    out.push(rel.to_string_lossy().into_owned());
467                }
468            }
469        }
470        walk(dir, dir, &mut out);
471        out.sort();
472        out
473    }
474
475    #[test]
476    fn snapshot_round_trips_an_open_wal_database_and_files() {
477        let src = tempfile::tempdir().unwrap();
478        std::fs::create_dir_all(src.path().join("workspace/notes")).unwrap();
479        std::fs::write(src.path().join("workspace/notes/a.md"), "hello").unwrap();
480        let db = src.path().join("serve.db");
481        let conn = rusqlite::open(&db).unwrap();
482        conn.execute_batch(
483            "PRAGMA journal_mode = WAL; CREATE TABLE t (v TEXT); INSERT INTO t VALUES ('kept');",
484        )
485        .unwrap();
486        // Still open, rows only in the WAL: the backup must see them anyway.
487        assert!(src.path().join("serve.db-wal").exists());
488        std::fs::write(src.path().join("serve.db.lock"), "").unwrap();
489
490        let bytes = snapshot_dir(src.path()).unwrap();
491        let dst = tempfile::tempdir().unwrap();
492        std::fs::write(dst.path().join("stale.txt"), "gone").unwrap();
493        let count = restore_dir(dst.path(), &bytes).unwrap();
494
495        assert_eq!(count, 2);
496        assert_eq!(files(dst.path()), ["serve.db", "workspace/notes/a.md"]);
497        assert_eq!(
498            std::fs::read_to_string(dst.path().join("workspace/notes/a.md")).unwrap(),
499            "hello"
500        );
501        let copy = rusqlite::open(dst.path().join("serve.db")).unwrap();
502        let value: String = copy.query_row("SELECT v FROM t", [], |r| r.get(0)).unwrap();
503        assert_eq!(value, "kept");
504        drop(conn);
505    }
506
507    #[test]
508    fn snapshot_of_a_missing_directory_is_empty_and_restores_to_empty() {
509        let missing = tempfile::tempdir().unwrap().path().join("nope");
510        let bytes = snapshot_dir(&missing).unwrap();
511        let dst = tempfile::tempdir().unwrap();
512        assert_eq!(restore_dir(&dst.path().join("data"), &bytes).unwrap(), 0);
513        assert!(dst.path().join("data").is_dir());
514    }
515
516    #[test]
517    fn restore_refuses_escaping_paths_and_links_without_touching_the_directory() {
518        let dst = tempfile::tempdir().unwrap();
519        std::fs::write(dst.path().join("keep.txt"), "kept").unwrap();
520
521        let mut escape = tar::Builder::new(Vec::new());
522        let mut header = tar::Header::new_gnu();
523        header.set_size(1);
524        header.set_entry_type(tar::EntryType::Regular);
525        header.set_cksum();
526        // `append_data` refuses `..`, so write the raw name.
527        header.as_gnu_mut().unwrap().name[..9].copy_from_slice(b"../x.txt\0");
528        header.set_cksum();
529        escape.append(&header, &b"x"[..]).unwrap();
530        let err = restore_dir(dst.path(), &escape.into_inner().unwrap()).unwrap_err();
531        assert!(
532            err.to_string().contains("leaves the data directory"),
533            "{err}"
534        );
535
536        let mut link = tar::Builder::new(Vec::new());
537        let mut header = tar::Header::new_gnu();
538        header.set_entry_type(tar::EntryType::Symlink);
539        header.set_size(0);
540        link.append_link(&mut header, "evil", "/etc/passwd")
541            .unwrap();
542        let err = restore_dir(dst.path(), &link.into_inner().unwrap()).unwrap_err();
543        assert!(err.to_string().contains("Symlink"), "{err}");
544
545        assert!(restore_dir(dst.path(), b"definitely not a tar archive at all").is_err());
546        assert_eq!(
547            std::fs::read_to_string(dst.path().join("keep.txt")).unwrap(),
548            "kept"
549        );
550    }
551}