Skip to main content

datumesh_http/
slice.rs

1//! `GET /slice/<call with *>`: the members of a stored function's slice as of an instant, as
2//! an Arrow IPC stream or as JSON pages, paginated by key.
3//!
4//! Members come from the store's `slice_values` (values, hashes and cells, no receipt bytes),
5//! in SPEC 15.13 key order. A slice read as of an instant at or before now is kept in a small
6//! memory cache (its members cannot change, the store being append-only), so repeated reads
7//! of it are warm. A latest read (no `asof`) reuses the cached members of its slice while no
8//! member changed after the cached instant, and its `next` links pin that instant.
9//!
10//! JSON: a page envelope (SPEC 5.6, spec-0.4.2): `{"outcome": "ok", "slice", "members",
11//! "next"?, "fn_hash"}`, `members` the members' `ok` envelopes ascending by key, `next` the URL
12//! of the following page (`limit=` and `after=` the last key, hex of its JCS bytes), absent on
13//! the last page. Pages are of `limit` members (default 1,000, at most 10,000), all with the
14//! asof of the first page.
15//!
16//! Arrow (`Accept: application/vnd.apache.arrow.stream`): one stream, record batches of up
17//! to 65,536 rows, a column per open argument and `value`, `type`, `receipt`, `tx`, `asof`
18//! (UTF-8: lexical forms, so no number is rounded), schema metadata `slice_id`, `asof`,
19//! `function`. The whole slice (up to the configured bound) in one stream.
20
21use std::cmp::Ordering;
22use std::sync::Arc;
23
24use arrow_array::{ArrayRef, RecordBatch, StringArray};
25use arrow_schema::{DataType, Field, Schema};
26use datumesh_core::json::{self, Value};
27use datumesh_core::{lexical, Arg, ErrorCode, Scalar, Slice};
28use datumesh_store::SliceValue;
29
30use crate::call::Target;
31use crate::envelope::{self, Cache, Reply};
32use crate::mesh::Inner;
33
34/// The media type of Arrow IPC streams.
35pub const ARROW: &str = "application/vnd.apache.arrow.stream";
36/// The media type of the CSV form of a slice (RFC 4180, UTF-8).
37const CSV: &str = "text/csv; charset=utf-8";
38
39/// The open arguments of a target, ascending.
40fn open_args(t: &Target) -> Vec<String> {
41    t.cell.args.iter().filter(|(_, a)| matches!(a, Arg::Slice)).map(|(k, _)| k.clone()).collect()
42}
43
44fn key_cmp(a: &[Scalar], b: &[Scalar]) -> Ordering {
45    for (x, y) in a.iter().zip(b) {
46        let o = datumesh_formula::eval::cmp_scalar(x, y);
47        if o != Ordering::Equal {
48            return o;
49        }
50    }
51    a.len().cmp(&b.len())
52}
53
54fn key_of(m: &SliceValue, open: &[String]) -> Vec<Scalar> {
55    open.iter().filter_map(|k| m.cell.args.get(k).and_then(Arg::scalar).cloned()).collect()
56}
57
58/// A cursor: hex of the JCS bytes of the key's typed scalars.
59fn cursor(key: &[Scalar]) -> String {
60    lexical::to_hex(&Value::Array(key.iter().map(Scalar::to_value).collect()).to_jcs())
61}
62
63fn parse_cursor(s: &str) -> Option<Vec<Scalar>> {
64    if !s.len().is_multiple_of(2) || s.len() > 1 << 16 {
65        return None;
66    }
67    let bytes: Option<Vec<u8>> =
68        (0..s.len()).step_by(2).map(|i| u8::from_str_radix(s.get(i..i + 2)?, 16).ok()).collect();
69    let v = json::parse(&bytes?).ok()?;
70    let Value::Array(items) = v else { return None };
71    items.iter().map(|i| Scalar::from_json(&i.to_jcs()).ok()).collect()
72}
73
74/// The form of a slice answer: JSON pages (the default), an Arrow stream or CSV (Build 2 Part 6, G1: each
75/// its own address, `?format=arrow` and `?format=csv`, or chosen by `Accept` for Arrow).
76#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
77pub(crate) enum Form {
78    #[default]
79    Json,
80    Arrow,
81    Csv,
82}
83
84/// What a slice request asks besides the slice.
85#[derive(Clone, Debug, Default)]
86pub(crate) struct SliceQuery {
87    pub(crate) limit: Option<usize>,
88    pub(crate) after: Option<String>,
89    pub(crate) form: Form,
90}
91
92/// Splits `?format=` off a slice request's query: the form it names (`None` when there is none), and
93/// the rest of the request. An unknown format is `E_PARSE_URI`.
94pub(crate) fn split_format(rest: &str) -> Result<(String, Option<Form>), ErrorCode> {
95    let Some((path, query)) = rest.split_once('?') else {
96        return Ok((rest.to_string(), None));
97    };
98    let mut form = None;
99    let mut kept = Vec::new();
100    for p in query.split('&') {
101        if let Some(v) = p.strip_prefix("format=") {
102            form = Some(match v {
103                "json" => Form::Json,
104                "arrow" => Form::Arrow,
105                "csv" => Form::Csv,
106                _ => return Err(ErrorCode::ParseUri),
107            });
108        } else {
109            kept.push(p);
110        }
111    }
112    if kept.is_empty() {
113        Ok((path.to_string(), form))
114    } else {
115        Ok((format!("{path}?{}", kept.join("&")), form))
116    }
117}
118
119/// A CSV field: quoted when it holds a comma, a quote or a line break (RFC 4180).
120pub(crate) fn csv_field(t: &str) -> String {
121    if t.contains([',', '"', '\r', '\n']) {
122        format!("\"{}\"", t.replace('"', "\"\""))
123    } else {
124        t.to_string()
125    }
126}
127
128/// Splits `?limit=` and `?after=` off a slice request's query; the rest is the slice's.
129pub(crate) fn split_query(rest: &str) -> Result<(String, SliceQuery), ErrorCode> {
130    let Some((path, query)) = rest.split_once('?') else {
131        return Ok((rest.to_string(), SliceQuery::default()));
132    };
133    let mut q = SliceQuery::default();
134    let mut kept = Vec::new();
135    for p in query.split('&') {
136        if let Some(v) = p.strip_prefix("limit=") {
137            q.limit = Some(v.parse().map_err(|_| ErrorCode::ParseUri)?);
138        } else if let Some(v) = p.strip_prefix("after=") {
139            q.after = Some(v.to_string());
140        } else {
141            kept.push(p);
142        }
143    }
144    if kept.is_empty() {
145        Ok((path.to_string(), q))
146    } else {
147        Ok((format!("{path}?{}", kept.join("&")), q))
148    }
149}
150
151impl Inner {
152    /// The members of a slice as of `a`, and the instant they answer as of (`a`, or for a
153    /// latest request, the `at` of a cached entry they are reused from). Cached when `a` is at
154    /// or before now. A latest request (`latest`, `a` = now) reuses the cached entry of the
155    /// slice with the greatest `at <= now` when no member changed after that `at`
156    /// (`slice_changed_after`), so a first page of an unchanged series is a cache hit.
157    fn members(
158        &self,
159        slice: &Slice,
160        a: &str,
161        now: &str,
162        latest: bool,
163    ) -> Result<(Arc<Vec<SliceValue>>, String), ErrorCode> {
164        {
165            let c = self.slices.lock().unwrap_or_else(|p| p.into_inner());
166            if latest {
167                let reuse = c
168                    .iter()
169                    .filter(|(id, at, _)| id == slice.id() && at.as_str() <= now)
170                    .max_by(|x, y| x.1.cmp(&y.1))
171                    .map(|(_, at, m)| (at.clone(), m.clone()));
172                drop(c);
173                if let Some((at, m)) = reuse {
174                    match self.store.slice_changed_after(slice, &at) {
175                        Ok(false) => return Ok((m, at)),
176                        Ok(true) => {}
177                        Err(e) => tracing::warn!(target: "datumesh_http", error = %e, "slice_changed_after"),
178                    }
179                }
180            } else if let Some((_, _, m)) = c.iter().find(|(id, at, _)| id == slice.id() && at == a) {
181                return Ok((m.clone(), a.to_string()));
182            }
183        }
184        let m = Arc::new(self.store.slice_values(slice, a).map_err(|e| {
185            tracing::warn!(target: "datumesh_http", error = %e, "slice_values");
186            ErrorCode::InputUnavailable
187        })?);
188        if a <= now {
189            let mut c = self.slices.lock().unwrap_or_else(|p| p.into_inner());
190            c.retain(|(id, at, _)| !(id == slice.id() && at == a));
191            c.push((slice.id().to_string(), a.to_string(), m.clone()));
192            if c.len() > 8 {
193                c.remove(0);
194            }
195        }
196        Ok((m, a.to_string()))
197    }
198
199    /// Answers a slice request (blocking) in the form asked.
200    pub(crate) fn slice(&self, rest: &str, now: &str, form: Form) -> Reply {
201        let (rest, mut q) = match split_query(rest) {
202            Ok(x) => x,
203            Err(c) => return envelope::bad_request_reply(c),
204        };
205        q.form = form;
206        let target = match crate::call::resolve(&rest, now, &self.config.aliases, self, true) {
207            Ok(t) => t,
208            Err(r) => return self.rejection(r, None),
209        };
210        if target.bound.descriptor.lang != "stored" {
211            // Members of a computed function would each be evaluated: not served in 0.1.0.
212            return envelope::error_reply(ErrorCode::EvalUnavailable);
213        }
214        if self.foreign.is_foreign(&target.fn_name()) {
215            return envelope::error_reply(ErrorCode::InputUnavailable);
216        }
217        if self.config.policy.rule(&target.fn_name()).is_some() {
218            return Reply::json(403, envelope::unlicensed(), Cache::Private);
219        }
220        let a = target.asof().unwrap_or(now).to_string();
221        let slice = match Slice::from_value(&target.cell.to_value()) {
222            Ok(s) => s,
223            Err(e) => {
224                return envelope::bad_request_reply(if envelope::is_request_code(e.code()) {
225                    e.code()
226                } else {
227                    ErrorCode::ParseUri
228                })
229            }
230        };
231        let (members, at) = match self.members(&slice, &a, now, target.asof().is_none()) {
232            Ok(m) => m,
233            Err(c) => return envelope::error_reply(c),
234        };
235        let open = open_args(&target);
236        let total = members.len().min(self.config.slice_max);
237        let cache = if target.asof().is_some_and(|x| x <= now) {
238            Cache::Immutable
239        } else {
240            Cache::Public(self.config.ttl.latest_max_age.as_secs())
241        };
242        let start = match &q.after {
243            None => 0,
244            Some(c) => match parse_cursor(c) {
245                Some(k) => members[..total].partition_point(|m| key_cmp(&key_of(m, &open), &k) != Ordering::Greater),
246                None => return envelope::bad_request_reply(ErrorCode::ParseUri),
247            },
248        };
249        if q.form == Form::Csv {
250            // One row a member, in key order: its key arguments, its value and its tx. Not paginated: the
251            // whole slice up to the bound is one file.
252            let mut out = String::new();
253            let mut head: Vec<String> = open.clone();
254            head.extend(["value".to_string(), "tx".to_string()]);
255            out.push_str(&head.iter().map(|h| csv_field(h)).collect::<Vec<_>>().join(","));
256            out.push_str("\r\n");
257            for m in &members[..total] {
258                let mut cells: Vec<String> = open
259                    .iter()
260                    .map(|k| {
261                        m.cell.args.get(k).and_then(Arg::scalar).map(|x| x.lexical().to_string()).unwrap_or_default()
262                    })
263                    .collect();
264                cells.push(m.value.lexical().to_string());
265                cells.push(m.version.tx.clone());
266                out.push_str(&cells.iter().map(|c| csv_field(c)).collect::<Vec<_>>().join(","));
267                out.push_str("\r\n");
268            }
269            return Reply {
270                status: 200,
271                body: out.into_bytes().into(),
272                content_type: CSV,
273                cache,
274                canonical: None,
275                retry_after: None,
276            };
277        }
278        if q.form == Form::Arrow {
279            let end = match q.limit {
280                Some(l) => (start + l).min(total),
281                None => total,
282            };
283            // The instant the members answer as of (CONTRACT-perf H7): `at`, the `asof` the JSON `next` pins.
284            return match arrow_stream(&members[start..end], &open, &slice, &at, &target.fn_name()) {
285                Ok(body) => Reply {
286                    status: 200,
287                    body: body.into(),
288                    content_type: ARROW,
289                    cache,
290                    canonical: None,
291                    retry_after: None,
292                },
293                Err(e) => {
294                    tracing::warn!(target: "datumesh_http", error = %e, "arrow");
295                    envelope::error_reply(ErrorCode::EvalUnavailable)
296                }
297            };
298        }
299        let limit = q.limit.unwrap_or(self.config.slice_page).clamp(1, crate::config::MAX_SLICE_PAGE);
300        let end = (start + limit).min(total);
301        // A page (5.6): the members' envelopes, in key order.
302        let mut page = Vec::with_capacity(end - start);
303        for m in &members[start..end] {
304            match self.receipt_info_by_hash(&m.version.receipt_hash) {
305                Some(info) => page.push(crate::envelope::with_fn_hash(
306                    datumesh_core::Answer::ok(info.parsed.clone(), info.trust),
307                    Some(&info.parsed.cell().fn_hash),
308                )),
309                None => return crate::envelope::error_answer_reply(ErrorCode::InputUnavailable, Some(slice.fn_hash())),
310            }
311        }
312        let next = (end < total).then(|| {
313            let k = key_of(&members[end - 1], &open);
314            format!("{}/slice/{}&limit={limit}&after={}", self.config.origin(), target.rest_asof(&at), cursor(&k))
315        });
316        let fn_hash = slice.fn_hash().to_string();
317        let answer = datumesh_core::Answer::page(datumesh_core::SlicePage::new(slice, page, next));
318        crate::envelope::answer_reply(&crate::envelope::with_fn_hash(answer, Some(&fn_hash)), cache)
319    }
320}
321
322pub(crate) fn arrow_stream(
323    members: &[SliceValue],
324    open: &[String],
325    slice: &Slice,
326    asof: &str,
327    function: &str,
328) -> Result<Vec<u8>, String> {
329    let mut fields: Vec<Field> = open.iter().map(|k| Field::new(k, DataType::Utf8, false)).collect();
330    for c in ["value", "type", "receipt", "tx", "asof"] {
331        fields.push(Field::new(c, DataType::Utf8, false));
332    }
333    let metadata = [
334        ("slice_id".to_string(), slice.id().to_string()),
335        ("asof".to_string(), asof.to_string()),
336        ("function".to_string(), function.to_string()),
337    ]
338    .into_iter()
339    .collect();
340    let schema = Arc::new(Schema::new_with_metadata(fields, metadata));
341    let mut out = Vec::new();
342    {
343        let mut w = arrow_ipc::writer::StreamWriter::try_new(&mut out, &schema).map_err(|e| e.to_string())?;
344        for chunk in members.chunks(65_536) {
345            let mut cols: Vec<ArrayRef> = Vec::new();
346            for k in open {
347                let a: StringArray = chunk
348                    .iter()
349                    .map(|m| Some(m.cell.args.get(k).and_then(Arg::scalar).map(Scalar::lexical).unwrap_or_default()))
350                    .collect();
351                cols.push(Arc::new(a));
352            }
353            let v: StringArray = chunk.iter().map(|m| Some(m.value.lexical())).collect();
354            let t: StringArray = chunk.iter().map(|m| Some(m.value.ty().as_str())).collect();
355            let r: StringArray = chunk.iter().map(|m| Some(m.version.receipt_hash.as_str())).collect();
356            let tx: StringArray = chunk.iter().map(|m| Some(m.version.tx.as_str())).collect();
357            let at: StringArray = chunk.iter().map(|m| Some(m.version.asof.as_str())).collect();
358            for a in [v, t, r, tx, at] {
359                cols.push(Arc::new(a));
360            }
361            let batch = RecordBatch::try_new(schema.clone(), cols).map_err(|e| e.to_string())?;
362            w.write(&batch).map_err(|e| e.to_string())?;
363        }
364        w.finish().map_err(|e| e.to_string())?;
365    }
366    Ok(out)
367}