Skip to main content

cellix_doc/
notebook.rs

1//! A Jupyter notebook as a page. Its cells become Markdown that edits like any other page,
2//! with a hidden line above each saying which cell it is, and saving puts the text back into
3//! the JSON beside the outputs it already had. Cells don't run here: that's the kernel's work,
4//! and a shell in the side pane can do it.
5
6use base64::Engine;
7use serde::Serialize;
8use serde_json::{Value, json};
9
10/// What a cell printed or drew.
11#[derive(Clone, Debug, PartialEq, Eq)]
12pub enum Output {
13    Text(String),
14    Error(String),
15    Image(Vec<u8>),
16}
17
18#[derive(Clone, Debug, PartialEq, Eq)]
19pub struct Notebook {
20    json: Value,
21    pub language: String,
22    /// Cells waiting for or in the kernel.
23    pub running: Vec<usize>,
24}
25
26/// The hidden line above a cell.
27#[derive(Clone, Copy, Debug, PartialEq, Eq)]
28pub struct Marker {
29    /// Which cell of the file this is. None for one added since.
30    pub cell: Option<usize>,
31    pub code: bool,
32}
33
34/// Reads a hidden line.
35pub fn marker(line: &str) -> Option<Marker> {
36    let inner = line
37        .trim()
38        .strip_prefix("<!-- cell ")?
39        .strip_suffix(" -->")?;
40    let (cell, kind) = inner.split_once(' ')?;
41    Some(Marker {
42        cell: cell.parse().ok(),
43        code: kind == "code",
44    })
45}
46
47/// A new code cell's lines, in the language of the notebook.
48pub fn new_code_cell(language: &str) -> String {
49    format!("<!-- cell new code -->\n```{language}\n\n```")
50}
51
52/// A new text cell's lines.
53pub fn new_text_cell() -> String {
54    "<!-- cell new text -->\n".to_string()
55}
56
57/// A field that may be a string or a list of lines.
58fn joined(value: Option<&Value>) -> String {
59    match value {
60        Some(Value::String(text)) => text.clone(),
61        Some(Value::Array(lines)) => lines.iter().filter_map(Value::as_str).collect(),
62        _ => String::new(),
63    }
64}
65
66/// Lines the way a notebook stores them, each keeping its newline.
67fn stored(text: &str) -> Value {
68    let lines: Vec<&str> = text.split_inclusive('\n').collect();
69    json!(lines)
70}
71
72fn no_colours(text: &str) -> String {
73    let mut out = String::with_capacity(text.len());
74    let mut chars = text.chars();
75    while let Some(c) = chars.next() {
76        if c == '\u{1b}' {
77            for inner in chars.by_ref() {
78                if inner.is_ascii_alphabetic() {
79                    break;
80                }
81            }
82        } else {
83            out.push(c);
84        }
85    }
86    out
87}
88
89fn output_of(value: &Value) -> Option<Output> {
90    match value.get("output_type")?.as_str()? {
91        "stream" => Some(Output::Text(joined(value.get("text")))),
92        "error" => {
93            let trace = value.get("traceback").and_then(Value::as_array);
94            let lines = trace.into_iter().flatten().filter_map(Value::as_str);
95            Some(Output::Error(no_colours(
96                &lines.collect::<Vec<_>>().join("\n"),
97            )))
98        }
99        _ => {
100            let data = value.get("data")?;
101            let picture = ["image/png", "image/jpeg"]
102                .iter()
103                .find_map(|kind| data.get(*kind));
104            if let Some(picture) = picture {
105                let text: String = joined(Some(picture)).split_whitespace().collect();
106                let bytes = base64::engine::general_purpose::STANDARD.decode(text);
107                return bytes.ok().map(Output::Image);
108            }
109            data.get("text/plain")
110                .map(|text| Output::Text(joined(Some(text))))
111        }
112    }
113}
114
115impl Notebook {
116    pub fn read(text: &str) -> Result<Self, String> {
117        let json: Value = serde_json::from_str(text).map_err(|e| e.to_string())?;
118        if json.get("cells").and_then(Value::as_array).is_none() {
119            return Err("this has no cells".to_string());
120        }
121        let language = json
122            .pointer("/metadata/kernelspec/language")
123            .or_else(|| json.pointer("/metadata/language_info/name"))
124            .and_then(Value::as_str)
125            .unwrap_or("python")
126            .to_string();
127        Ok(Self {
128            json,
129            language,
130            running: Vec::new(),
131        })
132    }
133
134    /// The kernel's name, as the notebook gives it.
135    pub fn kernel(&self) -> String {
136        let name = self.json.pointer("/metadata/kernelspec/name");
137        name.and_then(Value::as_str)
138            .unwrap_or("python3")
139            .to_string()
140    }
141
142    /// Clears what every cell printed.
143    pub fn clear_all_outputs(&mut self) {
144        let Some(cells) = self.json["cells"].as_array_mut() else {
145            return;
146        };
147        for cell in cells.iter_mut().filter(|c| c.get("outputs").is_some()) {
148            cell["outputs"] = json!([]);
149            cell["execution_count"] = Value::Null;
150        }
151    }
152
153    /// Clears what a cell last printed, as it starts again.
154    pub fn clear_outputs(&mut self, cell: usize) {
155        self.json["cells"][cell]["outputs"] = json!([]);
156    }
157
158    /// Adds something a running cell printed or drew, joining text to the text before it.
159    pub fn add_output(&mut self, cell: usize, output: Value) {
160        let Some(list) = self.json["cells"][cell]["outputs"].as_array_mut() else {
161            return;
162        };
163        let stream = |value: &Value| value["output_type"] == "stream";
164        if let Some(last) = list.last_mut()
165            && stream(last)
166            && stream(&output)
167            && last["name"] == output["name"]
168        {
169            let text = joined(last.get("text")) + &joined(output.get("text"));
170            last["text"] = json!(text);
171            return;
172        }
173        list.push(output);
174    }
175
176    /// Notes the number a cell ran as.
177    pub fn set_count(&mut self, cell: usize, count: Option<u64>) {
178        self.json["cells"][cell]["execution_count"] = json!(count);
179    }
180
181    fn cells(&self) -> &[Value] {
182        self.json["cells"].as_array().map_or(&[], Vec::as_slice)
183    }
184
185    /// The text the page edits.
186    pub fn page(&self) -> String {
187        let mut out = String::new();
188        for (index, cell) in self.cells().iter().enumerate() {
189            let source = joined(cell.get("source"));
190            let source = source.trim_end();
191            if cell.get("cell_type").and_then(Value::as_str) == Some("code") {
192                let language = &self.language;
193                out.push_str(&format!(
194                    "<!-- cell {index} code -->\n```{language}\n{source}\n```\n\n"
195                ));
196            } else {
197                out.push_str(&format!("<!-- cell {index} text -->\n{source}\n\n"));
198            }
199        }
200        out
201    }
202
203    /// What a cell printed and drew when it last ran.
204    pub fn outputs(&self, cell: usize) -> Vec<Output> {
205        let cell = self.cells().get(cell);
206        let outputs = cell
207            .and_then(|c| c.get("outputs"))
208            .and_then(Value::as_array);
209        outputs
210            .into_iter()
211            .flatten()
212            .filter_map(output_of)
213            .collect()
214    }
215
216    /// The number a cell last ran as.
217    pub fn count(&self, cell: usize) -> Option<u64> {
218        self.cells().get(cell)?.get("execution_count")?.as_u64()
219    }
220
221    /// The notebook with the page's text in it, as Jupyter writes it.
222    pub fn write(&self, page: &str) -> String {
223        let mut cells = Vec::new();
224        let mut current: Option<(Marker, Vec<&str>)> = None;
225        let mut finish = |found: Option<(Marker, Vec<&str>)>| {
226            if let Some((mark, lines)) = found {
227                cells.push(self.cell_from(mark, &lines));
228            }
229        };
230        for line in page.split('\n') {
231            if let Some(mark) = marker(line) {
232                finish(current.take());
233                current = Some((mark, Vec::new()));
234            } else if let Some((_, lines)) = &mut current {
235                lines.push(line);
236            }
237        }
238        finish(current.take());
239        let mut out = self.json.clone();
240        out["cells"] = Value::Array(cells);
241        let mut bytes = Vec::new();
242        let format = serde_json::ser::PrettyFormatter::with_indent(b" ");
243        let mut writer = serde_json::Serializer::with_formatter(&mut bytes, format);
244        let written = out.serialize(&mut writer);
245        match written {
246            Ok(()) => format!("{}\n", String::from_utf8_lossy(&bytes)),
247            Err(_) => self.json.to_string(),
248        }
249    }
250
251    fn cell_from(&self, mark: Marker, lines: &[&str]) -> Value {
252        let inside = if mark.code {
253            let open = lines.iter().position(|l| l.starts_with("```"));
254            let close = lines.iter().rposition(|l| l.trim() == "```");
255            match (open, close) {
256                (Some(open), Some(close)) if close > open => &lines[open + 1..close],
257                _ => lines,
258            }
259        } else {
260            lines
261        };
262        let text = inside.join("\n");
263        let source = stored(text.trim_matches('\n'));
264        let kept = mark.cell.and_then(|index| self.cells().get(index));
265        match kept {
266            Some(cell) => {
267                let mut cell = cell.clone();
268                cell["source"] = source;
269                cell
270            }
271            None if mark.code => json!({
272                "cell_type": "code",
273                "execution_count": null,
274                "metadata": {},
275                "outputs": [],
276                "source": source,
277            }),
278            None => json!({"cell_type": "markdown", "metadata": {}, "source": source}),
279        }
280    }
281}
282
283#[cfg(test)]
284mod tests {
285    use super::*;
286
287    pub const SAMPLE: &str = r##"{
288 "cells": [
289  {"cell_type": "markdown", "metadata": {}, "source": ["# Training\n", "\n", "Words.\n"]},
290  {"cell_type": "code", "execution_count": 3, "metadata": {"tags": ["x"]},
291   "outputs": [
292    {"output_type": "stream", "name": "stdout", "text": ["loss 0.5\n", "loss 0.2\n"]},
293    {"output_type": "execute_result", "data": {"text/plain": ["42"]}, "metadata": {}, "execution_count": 3},
294    {"output_type": "display_data", "data": {"image/png": "iVBORw0KGgo=\n", "text/plain": ["<Figure>"]}, "metadata": {}},
295    {"output_type": "error", "ename": "E", "evalue": "v", "traceback": ["\u001b[31mBoom\u001b[0m", "line 2"]}
296   ],
297   "source": ["import torch\n", "print(1)"]}
298 ],
299 "metadata": {"kernelspec": {"language": "python", "name": "python3"}},
300 "nbformat": 4, "nbformat_minor": 5
301}"##;
302
303    #[test]
304    fn a_notebook_reads_as_a_page() {
305        let notebook = Notebook::read(SAMPLE).unwrap();
306        let page = notebook.page();
307        assert!(page.contains("# Training"));
308        assert!(
309            page.contains("```python\nimport torch\nprint(1)\n```"),
310            "{page}"
311        );
312        assert_eq!(marker("<!-- cell 1 code -->").unwrap().cell, Some(1));
313        assert!(marker("<!-- cell 1 code -->").unwrap().code);
314        assert!(!marker("<!-- cell 0 text -->").unwrap().code);
315    }
316
317    #[test]
318    fn outputs_come_out_in_order_without_terminal_colours() {
319        let notebook = Notebook::read(SAMPLE).unwrap();
320        let outputs = notebook.outputs(1);
321        assert_eq!(outputs[0], Output::Text("loss 0.5\nloss 0.2\n".to_string()));
322        assert_eq!(outputs[1], Output::Text("42".to_string()));
323        assert!(matches!(outputs[2], Output::Image(_)));
324        assert_eq!(outputs[3], Output::Error("Boom\nline 2".to_string()));
325        assert_eq!(notebook.count(1), Some(3));
326    }
327
328    #[test]
329    fn nothing_changed_is_nothing_lost() {
330        let notebook = Notebook::read(SAMPLE).unwrap();
331        let again: Value = serde_json::from_str(&notebook.write(&notebook.page())).unwrap();
332        let original: Value = serde_json::from_str(SAMPLE).unwrap();
333        assert_eq!(
334            again["cells"][1]["outputs"],
335            original["cells"][1]["outputs"]
336        );
337        assert_eq!(
338            again["cells"][1]["metadata"],
339            original["cells"][1]["metadata"]
340        );
341        assert_eq!(
342            joined(Some(&again["cells"][0]["source"])),
343            "# Training\n\nWords."
344        );
345        assert_eq!(again["nbformat_minor"], 5);
346    }
347
348    #[test]
349    fn an_edit_reaches_its_cell_and_a_new_cell_is_added() {
350        let notebook = Notebook::read(SAMPLE).unwrap();
351        let page = notebook.page().replace("print(1)", "print(2)");
352        let page = format!("{page}{}\n", new_code_cell("python"));
353        let page = page.replace("```python\n\n```", "```python\nx = 1\n```");
354        let written: Value = serde_json::from_str(&notebook.write(&page)).unwrap();
355        let cells = written["cells"].as_array().unwrap();
356        assert_eq!(cells.len(), 3);
357        assert_eq!(joined(Some(&cells[1]["source"])), "import torch\nprint(2)");
358        assert_eq!(cells[1]["outputs"].as_array().unwrap().len(), 4);
359        assert_eq!(joined(Some(&cells[2]["source"])), "x = 1");
360        assert_eq!(cells[2]["cell_type"], "code");
361    }
362
363    #[test]
364    fn a_cell_run_again_replaces_what_it_printed() {
365        let mut notebook = Notebook::read(SAMPLE).unwrap();
366        notebook.clear_outputs(1);
367        let line = |text: &str| json!({"output_type": "stream", "name": "stdout", "text": text});
368        notebook.add_output(1, line("one\n"));
369        notebook.add_output(1, line("two\n"));
370        notebook.set_count(1, Some(9));
371        assert_eq!(
372            notebook.outputs(1),
373            [Output::Text("one\ntwo\n".to_string())]
374        );
375        assert_eq!(notebook.count(1), Some(9));
376        let written: Value = serde_json::from_str(&notebook.write(&notebook.page())).unwrap();
377        assert_eq!(written["cells"][1]["execution_count"], 9);
378    }
379
380    #[test]
381    fn something_else_is_not_a_notebook() {
382        assert!(Notebook::read("{}").is_err());
383        assert!(Notebook::read("not json").is_err());
384    }
385}