1use std::cmp::Ordering;
21use std::sync::Arc;
22
23use arrow_array::{ArrayRef, RecordBatch, StringArray};
24use arrow_schema::{DataType, Field, Schema};
25use datumesh_core::json::{self, Value};
26use datumesh_core::{lexical, Arg, ErrorCode, Scalar, Slice};
27use datumesh_store::SliceValue;
28
29use crate::call::Target;
30use crate::envelope::{self, Cache, Reply};
31use crate::mesh::Inner;
32
33pub const ARROW: &str = "application/vnd.apache.arrow.stream";
35
36fn open_args(t: &Target) -> Vec<String> {
38 t.cell.args.iter().filter(|(_, a)| matches!(a, Arg::Slice)).map(|(k, _)| k.clone()).collect()
39}
40
41fn key_cmp(a: &[Scalar], b: &[Scalar]) -> Ordering {
42 for (x, y) in a.iter().zip(b) {
43 let o = datumesh_formula::eval::cmp_scalar(x, y);
44 if o != Ordering::Equal {
45 return o;
46 }
47 }
48 a.len().cmp(&b.len())
49}
50
51fn key_of(m: &SliceValue, open: &[String]) -> Vec<Scalar> {
52 open.iter().filter_map(|k| m.cell.args.get(k).and_then(Arg::scalar).cloned()).collect()
53}
54
55fn cursor(key: &[Scalar]) -> String {
57 lexical::to_hex(&Value::Array(key.iter().map(Scalar::to_value).collect()).to_jcs())
58}
59
60fn parse_cursor(s: &str) -> Option<Vec<Scalar>> {
61 if !s.len().is_multiple_of(2) || s.len() > 1 << 16 {
62 return None;
63 }
64 let bytes: Option<Vec<u8>> =
65 (0..s.len()).step_by(2).map(|i| u8::from_str_radix(s.get(i..i + 2)?, 16).ok()).collect();
66 let v = json::parse(&bytes?).ok()?;
67 let Value::Array(items) = v else { return None };
68 items.iter().map(|i| Scalar::from_json(&i.to_jcs()).ok()).collect()
69}
70
71#[derive(Clone, Debug, Default)]
73pub(crate) struct SliceQuery {
74 pub(crate) limit: Option<usize>,
75 pub(crate) after: Option<String>,
76 pub(crate) arrow: bool,
77}
78
79pub(crate) fn split_query(rest: &str) -> Result<(String, SliceQuery), ErrorCode> {
81 let Some((path, query)) = rest.split_once('?') else {
82 return Ok((rest.to_string(), SliceQuery::default()));
83 };
84 let mut q = SliceQuery::default();
85 let mut kept = Vec::new();
86 for p in query.split('&') {
87 if let Some(v) = p.strip_prefix("limit=") {
88 q.limit = Some(v.parse().map_err(|_| ErrorCode::ParseUri)?);
89 } else if let Some(v) = p.strip_prefix("after=") {
90 q.after = Some(v.to_string());
91 } else {
92 kept.push(p);
93 }
94 }
95 if kept.is_empty() {
96 Ok((path.to_string(), q))
97 } else {
98 Ok((format!("{path}?{}", kept.join("&")), q))
99 }
100}
101
102impl Inner {
103 fn members(&self, slice: &Slice, a: &str, now: &str) -> Result<Arc<Vec<SliceValue>>, ErrorCode> {
105 {
106 let c = self.slices.lock().unwrap_or_else(|p| p.into_inner());
107 if let Some((_, _, m)) = c.iter().find(|(id, at, _)| id == slice.id() && at == a) {
108 return Ok(m.clone());
109 }
110 }
111 let m = Arc::new(self.store.slice_values(slice, a).map_err(|e| {
112 tracing::warn!(target: "datumesh_http", error = %e, "slice_values");
113 ErrorCode::InputUnavailable
114 })?);
115 if a <= now {
116 let mut c = self.slices.lock().unwrap_or_else(|p| p.into_inner());
117 c.push((slice.id().to_string(), a.to_string(), m.clone()));
118 if c.len() > 8 {
119 c.remove(0);
120 }
121 }
122 Ok(m)
123 }
124
125 pub(crate) fn slice(&self, rest: &str, now: &str, arrow: bool) -> Reply {
127 let (rest, mut q) = match split_query(rest) {
128 Ok(x) => x,
129 Err(c) => return envelope::bad_request_reply(c),
130 };
131 q.arrow = arrow;
132 let target = match crate::call::resolve(&rest, now, &self.config.aliases, self, true) {
133 Ok(t) => t,
134 Err(r) => return self.rejection(r, None),
135 };
136 if target.bound.descriptor.lang != "stored" {
137 return envelope::error_reply(ErrorCode::EvalUnavailable);
139 }
140 if self.foreign.is_foreign(&target.fn_name()) {
141 return envelope::error_reply(ErrorCode::InputUnavailable);
142 }
143 if self.config.policy.rule(&target.fn_name()).is_some() {
144 return Reply::json(403, envelope::unlicensed(), Cache::Private);
145 }
146 let a = target.asof().unwrap_or(now).to_string();
147 let slice = match Slice::from_value(&target.cell.to_value()) {
148 Ok(s) => s,
149 Err(e) => {
150 return envelope::bad_request_reply(if envelope::is_request_code(e.code()) {
151 e.code()
152 } else {
153 ErrorCode::ParseUri
154 })
155 }
156 };
157 let members = match self.members(&slice, &a, now) {
158 Ok(m) => m,
159 Err(c) => return envelope::error_reply(c),
160 };
161 let open = open_args(&target);
162 let total = members.len().min(self.config.slice_max);
163 let cache = if target.asof().is_some_and(|x| x <= now) {
164 Cache::Immutable
165 } else {
166 Cache::Public(self.config.ttl.latest_max_age.as_secs())
167 };
168 let start = match &q.after {
169 None => 0,
170 Some(c) => match parse_cursor(c) {
171 Some(k) => members[..total].partition_point(|m| key_cmp(&key_of(m, &open), &k) != Ordering::Greater),
172 None => return envelope::bad_request_reply(ErrorCode::ParseUri),
173 },
174 };
175 if q.arrow {
176 let end = match q.limit {
177 Some(l) => (start + l).min(total),
178 None => total,
179 };
180 return match arrow_stream(&members[start..end], &open, &slice, &a, &target.fn_name()) {
181 Ok(body) => Reply {
182 status: 200,
183 body: body.into(),
184 content_type: ARROW,
185 cache,
186 canonical: None,
187 retry_after: None,
188 },
189 Err(e) => {
190 tracing::warn!(target: "datumesh_http", error = %e, "arrow");
191 envelope::error_reply(ErrorCode::EvalUnavailable)
192 }
193 };
194 }
195 let limit = q.limit.unwrap_or(self.config.slice_page).clamp(1, crate::config::MAX_SLICE_PAGE);
196 let end = (start + limit).min(total);
197 let mut page = Vec::with_capacity(end - start);
199 for m in &members[start..end] {
200 match self.receipt_info_by_hash(&m.version.receipt_hash) {
201 Some(info) => page.push(crate::envelope::with_fn_hash(
202 datumesh_core::Answer::ok(info.parsed.clone(), info.trust),
203 Some(&info.parsed.cell().fn_hash),
204 )),
205 None => return crate::envelope::error_answer_reply(ErrorCode::InputUnavailable, Some(slice.fn_hash())),
206 }
207 }
208 let next = (end < total).then(|| {
209 let k = key_of(&members[end - 1], &open);
210 format!("{}/slice/{}&limit={limit}&after={}", self.config.origin(), target.rest_asof(&a), cursor(&k))
211 });
212 let fn_hash = slice.fn_hash().to_string();
213 let answer = datumesh_core::Answer::page(datumesh_core::SlicePage::new(slice, page, next));
214 crate::envelope::answer_reply(&crate::envelope::with_fn_hash(answer, Some(&fn_hash)), cache)
215 }
216}
217
218fn arrow_stream(
219 members: &[SliceValue],
220 open: &[String],
221 slice: &Slice,
222 asof: &str,
223 function: &str,
224) -> Result<Vec<u8>, String> {
225 let mut fields: Vec<Field> = open.iter().map(|k| Field::new(k, DataType::Utf8, false)).collect();
226 for c in ["value", "type", "receipt", "tx", "asof"] {
227 fields.push(Field::new(c, DataType::Utf8, false));
228 }
229 let metadata = [
230 ("slice_id".to_string(), slice.id().to_string()),
231 ("asof".to_string(), asof.to_string()),
232 ("function".to_string(), function.to_string()),
233 ]
234 .into_iter()
235 .collect();
236 let schema = Arc::new(Schema::new_with_metadata(fields, metadata));
237 let mut out = Vec::new();
238 {
239 let mut w = arrow_ipc::writer::StreamWriter::try_new(&mut out, &schema).map_err(|e| e.to_string())?;
240 for chunk in members.chunks(65_536) {
241 let mut cols: Vec<ArrayRef> = Vec::new();
242 for k in open {
243 let a: StringArray = chunk
244 .iter()
245 .map(|m| Some(m.cell.args.get(k).and_then(Arg::scalar).map(Scalar::lexical).unwrap_or_default()))
246 .collect();
247 cols.push(Arc::new(a));
248 }
249 let v: StringArray = chunk.iter().map(|m| Some(m.value.lexical())).collect();
250 let t: StringArray = chunk.iter().map(|m| Some(m.value.ty().as_str())).collect();
251 let r: StringArray = chunk.iter().map(|m| Some(m.version.receipt_hash.as_str())).collect();
252 let tx: StringArray = chunk.iter().map(|m| Some(m.version.tx.as_str())).collect();
253 let at: StringArray = chunk.iter().map(|m| Some(m.version.asof.as_str())).collect();
254 for a in [v, t, r, tx, at] {
255 cols.push(Arc::new(a));
256 }
257 let batch = RecordBatch::try_new(schema.clone(), cols).map_err(|e| e.to_string())?;
258 w.write(&batch).map_err(|e| e.to_string())?;
259 }
260 w.finish().map_err(|e| e.to_string())?;
261 }
262 Ok(out)
263}