1use std::cmp::Ordering;
22use std::sync::Arc;
23
24use arrow_array::{ArrayRef, RecordBatch, StringArray};
25use arrow_schema::{DataType, Field, Schema};
26use datumesh_core::json::{self, Value};
27use datumesh_core::{lexical, Arg, ErrorCode, Scalar, Slice};
28use datumesh_store::SliceValue;
29
30use crate::call::Target;
31use crate::envelope::{self, Cache, Reply};
32use crate::mesh::Inner;
33
34pub const ARROW: &str = "application/vnd.apache.arrow.stream";
36const CSV: &str = "text/csv; charset=utf-8";
38
39fn open_args(t: &Target) -> Vec<String> {
41 t.cell.args.iter().filter(|(_, a)| matches!(a, Arg::Slice)).map(|(k, _)| k.clone()).collect()
42}
43
44fn key_cmp(a: &[Scalar], b: &[Scalar]) -> Ordering {
45 for (x, y) in a.iter().zip(b) {
46 let o = datumesh_formula::eval::cmp_scalar(x, y);
47 if o != Ordering::Equal {
48 return o;
49 }
50 }
51 a.len().cmp(&b.len())
52}
53
54fn key_of(m: &SliceValue, open: &[String]) -> Vec<Scalar> {
55 open.iter().filter_map(|k| m.cell.args.get(k).and_then(Arg::scalar).cloned()).collect()
56}
57
58fn cursor(key: &[Scalar]) -> String {
60 lexical::to_hex(&Value::Array(key.iter().map(Scalar::to_value).collect()).to_jcs())
61}
62
63fn parse_cursor(s: &str) -> Option<Vec<Scalar>> {
64 if !s.len().is_multiple_of(2) || s.len() > 1 << 16 {
65 return None;
66 }
67 let bytes: Option<Vec<u8>> =
68 (0..s.len()).step_by(2).map(|i| u8::from_str_radix(s.get(i..i + 2)?, 16).ok()).collect();
69 let v = json::parse(&bytes?).ok()?;
70 let Value::Array(items) = v else { return None };
71 items.iter().map(|i| Scalar::from_json(&i.to_jcs()).ok()).collect()
72}
73
74#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
77pub(crate) enum Form {
78 #[default]
79 Json,
80 Arrow,
81 Csv,
82}
83
84#[derive(Clone, Debug, Default)]
86pub(crate) struct SliceQuery {
87 pub(crate) limit: Option<usize>,
88 pub(crate) after: Option<String>,
89 pub(crate) form: Form,
90}
91
92pub(crate) fn split_format(rest: &str) -> Result<(String, Option<Form>), ErrorCode> {
95 let Some((path, query)) = rest.split_once('?') else {
96 return Ok((rest.to_string(), None));
97 };
98 let mut form = None;
99 let mut kept = Vec::new();
100 for p in query.split('&') {
101 if let Some(v) = p.strip_prefix("format=") {
102 form = Some(match v {
103 "json" => Form::Json,
104 "arrow" => Form::Arrow,
105 "csv" => Form::Csv,
106 _ => return Err(ErrorCode::ParseUri),
107 });
108 } else {
109 kept.push(p);
110 }
111 }
112 if kept.is_empty() {
113 Ok((path.to_string(), form))
114 } else {
115 Ok((format!("{path}?{}", kept.join("&")), form))
116 }
117}
118
119pub(crate) fn csv_field(t: &str) -> String {
121 if t.contains([',', '"', '\r', '\n']) {
122 format!("\"{}\"", t.replace('"', "\"\""))
123 } else {
124 t.to_string()
125 }
126}
127
128pub(crate) fn split_query(rest: &str) -> Result<(String, SliceQuery), ErrorCode> {
130 let Some((path, query)) = rest.split_once('?') else {
131 return Ok((rest.to_string(), SliceQuery::default()));
132 };
133 let mut q = SliceQuery::default();
134 let mut kept = Vec::new();
135 for p in query.split('&') {
136 if let Some(v) = p.strip_prefix("limit=") {
137 q.limit = Some(v.parse().map_err(|_| ErrorCode::ParseUri)?);
138 } else if let Some(v) = p.strip_prefix("after=") {
139 q.after = Some(v.to_string());
140 } else {
141 kept.push(p);
142 }
143 }
144 if kept.is_empty() {
145 Ok((path.to_string(), q))
146 } else {
147 Ok((format!("{path}?{}", kept.join("&")), q))
148 }
149}
150
151impl Inner {
152 fn members(
158 &self,
159 slice: &Slice,
160 a: &str,
161 now: &str,
162 latest: bool,
163 ) -> Result<(Arc<Vec<SliceValue>>, String), ErrorCode> {
164 {
165 let c = self.slices.lock().unwrap_or_else(|p| p.into_inner());
166 if latest {
167 let reuse = c
168 .iter()
169 .filter(|(id, at, _)| id == slice.id() && at.as_str() <= now)
170 .max_by(|x, y| x.1.cmp(&y.1))
171 .map(|(_, at, m)| (at.clone(), m.clone()));
172 drop(c);
173 if let Some((at, m)) = reuse {
174 match self.store.slice_changed_after(slice, &at) {
175 Ok(false) => return Ok((m, at)),
176 Ok(true) => {}
177 Err(e) => tracing::warn!(target: "datumesh_http", error = %e, "slice_changed_after"),
178 }
179 }
180 } else if let Some((_, _, m)) = c.iter().find(|(id, at, _)| id == slice.id() && at == a) {
181 return Ok((m.clone(), a.to_string()));
182 }
183 }
184 let m = Arc::new(self.store.slice_values_limit(slice, a, self.config.slice_max + 1).map_err(|e| {
187 tracing::warn!(target: "datumesh_http", error = %e, "slice_values");
188 ErrorCode::InputUnavailable
189 })?);
190 if a <= now {
191 let mut c = self.slices.lock().unwrap_or_else(|p| p.into_inner());
192 c.retain(|(id, at, _)| !(id == slice.id() && at == a));
193 c.push((slice.id().to_string(), a.to_string(), m.clone()));
194 if c.len() > 8 {
195 c.remove(0);
196 }
197 }
198 Ok((m, a.to_string()))
199 }
200
201 pub(crate) fn slice(&self, rest: &str, now: &str, form: Form) -> Reply {
203 let (rest, mut q) = match split_query(rest) {
204 Ok(x) => x,
205 Err(c) => return envelope::bad_request_reply(c),
206 };
207 q.form = form;
208 let target = match crate::call::resolve(&rest, now, &self.config.aliases, self, true) {
209 Ok(t) => t,
210 Err(r) => return self.rejection(r, None),
211 };
212 if target.bound.descriptor.lang != "stored" {
213 return envelope::error_reply(ErrorCode::EvalUnavailable);
215 }
216 if self.foreign.is_foreign(&target.fn_name()) {
217 return envelope::error_reply(ErrorCode::InputUnavailable);
218 }
219 if self.config.policy.rule(&target.fn_name()).is_some() {
220 return Reply::json(403, envelope::unlicensed(), Cache::Private);
221 }
222 let a = target.asof().unwrap_or(now).to_string();
223 let slice = match Slice::from_value(&target.cell.to_value()) {
224 Ok(s) => s,
225 Err(e) => {
226 return envelope::bad_request_reply(if envelope::is_request_code(e.code()) {
227 e.code()
228 } else {
229 ErrorCode::ParseUri
230 })
231 }
232 };
233 let (members, at) = match self.members(&slice, &a, now, target.asof().is_none()) {
234 Ok(m) => m,
235 Err(c) => return envelope::error_reply(c),
236 };
237 let open = open_args(&target);
238 let total = members.len().min(self.config.slice_max);
239 let cache = if target.asof().is_some_and(|x| x <= now) {
240 Cache::Immutable
241 } else {
242 Cache::Public(self.config.ttl.latest_max_age.as_secs())
243 };
244 let start = match &q.after {
245 None => 0,
246 Some(c) => match parse_cursor(c) {
247 Some(k) => members[..total].partition_point(|m| key_cmp(&key_of(m, &open), &k) != Ordering::Greater),
248 None => return envelope::bad_request_reply(ErrorCode::ParseUri),
249 },
250 };
251 if q.form == Form::Csv {
252 let mut out = String::new();
255 let mut head: Vec<String> = open.clone();
256 head.extend(["value".to_string(), "tx".to_string()]);
257 out.push_str(&head.iter().map(|h| csv_field(h)).collect::<Vec<_>>().join(","));
258 out.push_str("\r\n");
259 for m in &members[..total] {
260 let mut cells: Vec<String> = open
261 .iter()
262 .map(|k| {
263 m.cell.args.get(k).and_then(Arg::scalar).map(|x| x.lexical().to_string()).unwrap_or_default()
264 })
265 .collect();
266 cells.push(m.value.lexical().to_string());
267 cells.push(m.version.tx.clone());
268 out.push_str(&cells.iter().map(|c| csv_field(c)).collect::<Vec<_>>().join(","));
269 out.push_str("\r\n");
270 }
271 return Reply {
272 status: 200,
273 body: out.into_bytes().into(),
274 content_type: CSV,
275 cache,
276 canonical: None,
277 retry_after: None,
278 };
279 }
280 if q.form == Form::Arrow {
281 let end = match q.limit {
282 Some(l) => (start + l).min(total),
283 None => total,
284 };
285 return match arrow_stream(&members[start..end], &open, &slice, &at, &target.fn_name()) {
287 Ok(body) => Reply {
288 status: 200,
289 body: body.into(),
290 content_type: ARROW,
291 cache,
292 canonical: None,
293 retry_after: None,
294 },
295 Err(e) => {
296 tracing::warn!(target: "datumesh_http", error = %e, "arrow");
297 envelope::error_reply(ErrorCode::EvalUnavailable)
298 }
299 };
300 }
301 let limit = q.limit.unwrap_or(self.config.slice_page).clamp(1, crate::config::MAX_SLICE_PAGE);
302 let end = (start + limit).min(total);
303 let mut page = Vec::with_capacity(end - start);
305 for m in &members[start..end] {
306 match self.receipt_info_by_hash(&m.version.receipt_hash) {
307 Some(info) => page.push(crate::envelope::with_fn_hash(
308 datumesh_core::Answer::ok(info.parsed.clone(), info.trust),
309 Some(&info.parsed.cell().fn_hash),
310 )),
311 None => return crate::envelope::error_answer_reply(ErrorCode::InputUnavailable, Some(slice.fn_hash())),
312 }
313 }
314 let next = (end < total).then(|| {
315 let k = key_of(&members[end - 1], &open);
316 format!("{}/slice/{}&limit={limit}&after={}", self.config.origin(), target.rest_asof(&at), cursor(&k))
317 });
318 let fn_hash = slice.fn_hash().to_string();
319 let answer = datumesh_core::Answer::page(datumesh_core::SlicePage::new(slice, page, next));
320 crate::envelope::answer_reply(&crate::envelope::with_fn_hash(answer, Some(&fn_hash)), cache)
321 }
322}
323
324pub(crate) fn arrow_stream(
325 members: &[SliceValue],
326 open: &[String],
327 slice: &Slice,
328 asof: &str,
329 function: &str,
330) -> Result<Vec<u8>, String> {
331 let mut fields: Vec<Field> = open.iter().map(|k| Field::new(k, DataType::Utf8, false)).collect();
332 for c in ["value", "type", "receipt", "tx", "asof"] {
333 fields.push(Field::new(c, DataType::Utf8, false));
334 }
335 let metadata = [
336 ("slice_id".to_string(), slice.id().to_string()),
337 ("asof".to_string(), asof.to_string()),
338 ("function".to_string(), function.to_string()),
339 ]
340 .into_iter()
341 .collect();
342 let schema = Arc::new(Schema::new_with_metadata(fields, metadata));
343 let mut out = Vec::new();
344 {
345 let mut w = arrow_ipc::writer::StreamWriter::try_new(&mut out, &schema).map_err(|e| e.to_string())?;
346 for chunk in members.chunks(65_536) {
347 let mut cols: Vec<ArrayRef> = Vec::new();
348 for k in open {
349 let a: StringArray = chunk
350 .iter()
351 .map(|m| Some(m.cell.args.get(k).and_then(Arg::scalar).map(Scalar::lexical).unwrap_or_default()))
352 .collect();
353 cols.push(Arc::new(a));
354 }
355 let v: StringArray = chunk.iter().map(|m| Some(m.value.lexical())).collect();
356 let t: StringArray = chunk.iter().map(|m| Some(m.value.ty().as_str())).collect();
357 let r: StringArray = chunk.iter().map(|m| Some(m.version.receipt_hash.as_str())).collect();
358 let tx: StringArray = chunk.iter().map(|m| Some(m.version.tx.as_str())).collect();
359 let at: StringArray = chunk.iter().map(|m| Some(m.version.asof.as_str())).collect();
360 for a in [v, t, r, tx, at] {
361 cols.push(Arc::new(a));
362 }
363 let batch = RecordBatch::try_new(schema.clone(), cols).map_err(|e| e.to_string())?;
364 w.write(&batch).map_err(|e| e.to_string())?;
365 }
366 w.finish().map_err(|e| e.to_string())?;
367 }
368 Ok(out)
369}