Skip to main content

datumesh_http/
slice.rs

1//! `GET /slice/<call with *>`: the members of a stored function's slice as of an instant, as
2//! an Arrow IPC stream or as JSON pages, paginated by key.
3//!
4//! Members come from the store's `slice_values` (values, hashes and cells, no receipt bytes),
5//! in SPEC 15.13 key order. A slice read as of an instant at or before now is kept in a small
6//! memory cache (its members cannot change, the store being append-only), so repeated reads
7//! of it are warm.
8//!
9//! JSON: a page envelope (SPEC 5.6, spec-0.4.2): `{"outcome": "ok", "slice", "members",
10//! "next"?, "fn_hash"}`, `members` the members' `ok` envelopes ascending by key, `next` the URL
11//! of the following page (`limit=` and `after=` the last key, hex of its JCS bytes), absent on
12//! the last page. Pages are of `limit` members (default 1,000, at most 10,000), all with the
13//! asof of the first page.
14//!
15//! Arrow (`Accept: application/vnd.apache.arrow.stream`): one stream, record batches of up
16//! to 65,536 rows, a column per open argument and `value`, `type`, `receipt`, `tx`, `asof`
17//! (UTF-8: lexical forms, so no number is rounded), schema metadata `slice_id`, `asof`,
18//! `function`. The whole slice (up to the configured bound) in one stream.
19
20use std::cmp::Ordering;
21use std::sync::Arc;
22
23use arrow_array::{ArrayRef, RecordBatch, StringArray};
24use arrow_schema::{DataType, Field, Schema};
25use datumesh_core::json::{self, Value};
26use datumesh_core::{lexical, Arg, ErrorCode, Scalar, Slice};
27use datumesh_store::SliceValue;
28
29use crate::call::Target;
30use crate::envelope::{self, Cache, Reply};
31use crate::mesh::Inner;
32
33/// The media type of Arrow IPC streams.
34pub const ARROW: &str = "application/vnd.apache.arrow.stream";
35
36/// The open arguments of a target, ascending.
37fn open_args(t: &Target) -> Vec<String> {
38    t.cell.args.iter().filter(|(_, a)| matches!(a, Arg::Slice)).map(|(k, _)| k.clone()).collect()
39}
40
41fn key_cmp(a: &[Scalar], b: &[Scalar]) -> Ordering {
42    for (x, y) in a.iter().zip(b) {
43        let o = datumesh_formula::eval::cmp_scalar(x, y);
44        if o != Ordering::Equal {
45            return o;
46        }
47    }
48    a.len().cmp(&b.len())
49}
50
51fn key_of(m: &SliceValue, open: &[String]) -> Vec<Scalar> {
52    open.iter().filter_map(|k| m.cell.args.get(k).and_then(Arg::scalar).cloned()).collect()
53}
54
55/// A cursor: hex of the JCS bytes of the key's typed scalars.
56fn cursor(key: &[Scalar]) -> String {
57    lexical::to_hex(&Value::Array(key.iter().map(Scalar::to_value).collect()).to_jcs())
58}
59
60fn parse_cursor(s: &str) -> Option<Vec<Scalar>> {
61    if !s.len().is_multiple_of(2) || s.len() > 1 << 16 {
62        return None;
63    }
64    let bytes: Option<Vec<u8>> =
65        (0..s.len()).step_by(2).map(|i| u8::from_str_radix(s.get(i..i + 2)?, 16).ok()).collect();
66    let v = json::parse(&bytes?).ok()?;
67    let Value::Array(items) = v else { return None };
68    items.iter().map(|i| Scalar::from_json(&i.to_jcs()).ok()).collect()
69}
70
71/// What a slice request asks besides the slice.
72#[derive(Clone, Debug, Default)]
73pub(crate) struct SliceQuery {
74    pub(crate) limit: Option<usize>,
75    pub(crate) after: Option<String>,
76    pub(crate) arrow: bool,
77}
78
79/// Splits `?limit=` and `?after=` off a slice request's query; the rest is the slice's.
80pub(crate) fn split_query(rest: &str) -> Result<(String, SliceQuery), ErrorCode> {
81    let Some((path, query)) = rest.split_once('?') else {
82        return Ok((rest.to_string(), SliceQuery::default()));
83    };
84    let mut q = SliceQuery::default();
85    let mut kept = Vec::new();
86    for p in query.split('&') {
87        if let Some(v) = p.strip_prefix("limit=") {
88            q.limit = Some(v.parse().map_err(|_| ErrorCode::ParseUri)?);
89        } else if let Some(v) = p.strip_prefix("after=") {
90            q.after = Some(v.to_string());
91        } else {
92            kept.push(p);
93        }
94    }
95    if kept.is_empty() {
96        Ok((path.to_string(), q))
97    } else {
98        Ok((format!("{path}?{}", kept.join("&")), q))
99    }
100}
101
102impl Inner {
103    /// The members of a slice as of `a` (cached when `a` is at or before now).
104    fn members(&self, slice: &Slice, a: &str, now: &str) -> Result<Arc<Vec<SliceValue>>, ErrorCode> {
105        {
106            let c = self.slices.lock().unwrap_or_else(|p| p.into_inner());
107            if let Some((_, _, m)) = c.iter().find(|(id, at, _)| id == slice.id() && at == a) {
108                return Ok(m.clone());
109            }
110        }
111        let m = Arc::new(self.store.slice_values(slice, a).map_err(|e| {
112            tracing::warn!(target: "datumesh_http", error = %e, "slice_values");
113            ErrorCode::InputUnavailable
114        })?);
115        if a <= now {
116            let mut c = self.slices.lock().unwrap_or_else(|p| p.into_inner());
117            c.push((slice.id().to_string(), a.to_string(), m.clone()));
118            if c.len() > 8 {
119                c.remove(0);
120            }
121        }
122        Ok(m)
123    }
124
125    /// Answers a slice request (blocking).
126    pub(crate) fn slice(&self, rest: &str, now: &str, arrow: bool) -> Reply {
127        let (rest, mut q) = match split_query(rest) {
128            Ok(x) => x,
129            Err(c) => return envelope::bad_request_reply(c),
130        };
131        q.arrow = arrow;
132        let target = match crate::call::resolve(&rest, now, &self.config.aliases, self, true) {
133            Ok(t) => t,
134            Err(r) => return self.rejection(r, None),
135        };
136        if target.bound.descriptor.lang != "stored" {
137            // Members of a computed function would each be evaluated: not served in 0.1.0.
138            return envelope::error_reply(ErrorCode::EvalUnavailable);
139        }
140        if self.foreign.is_foreign(&target.fn_name()) {
141            return envelope::error_reply(ErrorCode::InputUnavailable);
142        }
143        if self.config.policy.rule(&target.fn_name()).is_some() {
144            return Reply::json(403, envelope::unlicensed(), Cache::Private);
145        }
146        let a = target.asof().unwrap_or(now).to_string();
147        let slice = match Slice::from_value(&target.cell.to_value()) {
148            Ok(s) => s,
149            Err(e) => {
150                return envelope::bad_request_reply(if envelope::is_request_code(e.code()) {
151                    e.code()
152                } else {
153                    ErrorCode::ParseUri
154                })
155            }
156        };
157        let members = match self.members(&slice, &a, now) {
158            Ok(m) => m,
159            Err(c) => return envelope::error_reply(c),
160        };
161        let open = open_args(&target);
162        let total = members.len().min(self.config.slice_max);
163        let cache = if target.asof().is_some_and(|x| x <= now) {
164            Cache::Immutable
165        } else {
166            Cache::Public(self.config.ttl.latest_max_age.as_secs())
167        };
168        let start = match &q.after {
169            None => 0,
170            Some(c) => match parse_cursor(c) {
171                Some(k) => members[..total].partition_point(|m| key_cmp(&key_of(m, &open), &k) != Ordering::Greater),
172                None => return envelope::bad_request_reply(ErrorCode::ParseUri),
173            },
174        };
175        if q.arrow {
176            let end = match q.limit {
177                Some(l) => (start + l).min(total),
178                None => total,
179            };
180            return match arrow_stream(&members[start..end], &open, &slice, &a, &target.fn_name()) {
181                Ok(body) => Reply {
182                    status: 200,
183                    body: body.into(),
184                    content_type: ARROW,
185                    cache,
186                    canonical: None,
187                    retry_after: None,
188                },
189                Err(e) => {
190                    tracing::warn!(target: "datumesh_http", error = %e, "arrow");
191                    envelope::error_reply(ErrorCode::EvalUnavailable)
192                }
193            };
194        }
195        let limit = q.limit.unwrap_or(self.config.slice_page).clamp(1, crate::config::MAX_SLICE_PAGE);
196        let end = (start + limit).min(total);
197        // A page (5.6): the members' envelopes, in key order.
198        let mut page = Vec::with_capacity(end - start);
199        for m in &members[start..end] {
200            match self.receipt_info_by_hash(&m.version.receipt_hash) {
201                Some(info) => page.push(crate::envelope::with_fn_hash(
202                    datumesh_core::Answer::ok(info.parsed.clone(), info.trust),
203                    Some(&info.parsed.cell().fn_hash),
204                )),
205                None => return crate::envelope::error_answer_reply(ErrorCode::InputUnavailable, Some(slice.fn_hash())),
206            }
207        }
208        let next = (end < total).then(|| {
209            let k = key_of(&members[end - 1], &open);
210            format!("{}/slice/{}&limit={limit}&after={}", self.config.origin(), target.rest_asof(&a), cursor(&k))
211        });
212        let fn_hash = slice.fn_hash().to_string();
213        let answer = datumesh_core::Answer::page(datumesh_core::SlicePage::new(slice, page, next));
214        crate::envelope::answer_reply(&crate::envelope::with_fn_hash(answer, Some(&fn_hash)), cache)
215    }
216}
217
218fn arrow_stream(
219    members: &[SliceValue],
220    open: &[String],
221    slice: &Slice,
222    asof: &str,
223    function: &str,
224) -> Result<Vec<u8>, String> {
225    let mut fields: Vec<Field> = open.iter().map(|k| Field::new(k, DataType::Utf8, false)).collect();
226    for c in ["value", "type", "receipt", "tx", "asof"] {
227        fields.push(Field::new(c, DataType::Utf8, false));
228    }
229    let metadata = [
230        ("slice_id".to_string(), slice.id().to_string()),
231        ("asof".to_string(), asof.to_string()),
232        ("function".to_string(), function.to_string()),
233    ]
234    .into_iter()
235    .collect();
236    let schema = Arc::new(Schema::new_with_metadata(fields, metadata));
237    let mut out = Vec::new();
238    {
239        let mut w = arrow_ipc::writer::StreamWriter::try_new(&mut out, &schema).map_err(|e| e.to_string())?;
240        for chunk in members.chunks(65_536) {
241            let mut cols: Vec<ArrayRef> = Vec::new();
242            for k in open {
243                let a: StringArray = chunk
244                    .iter()
245                    .map(|m| Some(m.cell.args.get(k).and_then(Arg::scalar).map(Scalar::lexical).unwrap_or_default()))
246                    .collect();
247                cols.push(Arc::new(a));
248            }
249            let v: StringArray = chunk.iter().map(|m| Some(m.value.lexical())).collect();
250            let t: StringArray = chunk.iter().map(|m| Some(m.value.ty().as_str())).collect();
251            let r: StringArray = chunk.iter().map(|m| Some(m.version.receipt_hash.as_str())).collect();
252            let tx: StringArray = chunk.iter().map(|m| Some(m.version.tx.as_str())).collect();
253            let at: StringArray = chunk.iter().map(|m| Some(m.version.asof.as_str())).collect();
254            for a in [v, t, r, tx, at] {
255                cols.push(Arc::new(a));
256            }
257            let batch = RecordBatch::try_new(schema.clone(), cols).map_err(|e| e.to_string())?;
258            w.write(&batch).map_err(|e| e.to_string())?;
259        }
260        w.finish().map_err(|e| e.to_string())?;
261    }
262    Ok(out)
263}