Skip to main content

graphrt_core/
file.rs

1//! File access contract.
2//!
3//! The runtime never touches storage itself. Nodes that need file bytes (document
4//! extraction, uploads, vision) ask the [`FileResolver`] registered in
5//! [`crate::node::Services`] as `Arc<dyn FileResolver>`. Hosts plug in their own
6//! storage (S3, blob store, database); this module provides an in-process
7//! resolver for inline data URLs and a [`ChainResolver`] to combine several.
8
9use std::fmt;
10use std::sync::Arc;
11
12use futures::future::BoxFuture;
13
14use crate::value::{File, FileTransfer};
15
16/// Error reading or storing a file.
17#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
18#[non_exhaustive]
19pub enum FileError {
20    #[error("no resolver can read file `{0}`")]
21    Unsupported(String),
22    #[error("file `{name}` exceeds the {limit}-byte limit")]
23    TooLarge { name: String, limit: u64 },
24    #[error("file `{name}` not found")]
25    NotFound { name: String },
26    #[error("access to file `{name}` denied: {reason}")]
27    Denied { name: String, reason: String },
28    #[error("reading file `{name}` failed: {message}")]
29    Io { name: String, message: String },
30    #[error("invalid file `{name}`: {message}")]
31    Invalid { name: String, message: String },
32}
33
34impl FileError {
35    /// Whether retrying might succeed (transient I/O).
36    pub fn is_retryable(&self) -> bool {
37        matches!(self, FileError::Io { .. })
38    }
39}
40
41/// File content with its detected media type.
42#[derive(Clone, Debug, PartialEq, Eq)]
43pub struct FileBytes {
44    pub bytes: Vec<u8>,
45    /// Media type if known (from the source or the file metadata).
46    pub mime_type: Option<String>,
47}
48
49/// Reads (and optionally stores) file content.
50pub trait FileResolver: Send + Sync + fmt::Debug {
51    /// Read the whole file, failing with [`FileError::TooLarge`] past `max_bytes`.
52    /// Return [`FileError::Unsupported`] for files this resolver does not handle,
53    /// so a [`ChainResolver`] can try the next one.
54    fn read<'a>(&'a self, file: &'a File, max_bytes: u64) -> BoxFuture<'a, Result<FileBytes, FileError>>;
55
56    /// Persist bytes produced by a node and return a [`File`] referencing them.
57    /// Default: unsupported.
58    fn store<'a>(
59        &'a self,
60        name: &'a str,
61        mime_type: Option<&'a str>,
62        bytes: Vec<u8>,
63    ) -> BoxFuture<'a, Result<File, FileError>> {
64        let _ = (mime_type, bytes);
65        Box::pin(futures::future::ready(Err(FileError::Unsupported(name.to_owned()))))
66    }
67}
68
69/// Resolves `inline` files and any file whose `url` is a `data:` URL.
70#[derive(Clone, Copy, Debug, Default)]
71pub struct DataUrlResolver;
72
73impl FileResolver for DataUrlResolver {
74    fn read<'a>(&'a self, file: &'a File, max_bytes: u64) -> BoxFuture<'a, Result<FileBytes, FileError>> {
75        let result = match file.url.as_deref() {
76            Some(url) if url.starts_with("data:") => decode_data_url(&file.name, url, max_bytes),
77            _ if file.transfer == FileTransfer::Inline => {
78                Err(FileError::Invalid { name: file.name.clone(), message: "inline file without a data URL".into() })
79            }
80            _ => Err(FileError::Unsupported(file.name.clone())),
81        };
82        let result = result.map(|mut fb| {
83            if fb.mime_type.is_none() {
84                fb.mime_type.clone_from(&file.mime_type);
85            }
86            fb
87        });
88        Box::pin(futures::future::ready(result))
89    }
90}
91
92/// Tries resolvers in order; the first one not returning `Unsupported` wins.
93#[derive(Clone, Debug, Default)]
94pub struct ChainResolver {
95    resolvers: Vec<Arc<dyn FileResolver>>,
96}
97
98impl ChainResolver {
99    pub fn new() -> Self {
100        Self::default()
101    }
102
103    /// Chain starting with the [`DataUrlResolver`].
104    pub fn with_data_urls() -> Self {
105        Self::new().then(Arc::new(DataUrlResolver))
106    }
107
108    pub fn then(mut self, resolver: Arc<dyn FileResolver>) -> Self {
109        self.resolvers.push(resolver);
110        self
111    }
112}
113
114impl FileResolver for ChainResolver {
115    fn read<'a>(&'a self, file: &'a File, max_bytes: u64) -> BoxFuture<'a, Result<FileBytes, FileError>> {
116        Box::pin(async move {
117            for resolver in &self.resolvers {
118                match resolver.read(file, max_bytes).await {
119                    Err(FileError::Unsupported(_)) => continue,
120                    other => return other,
121                }
122            }
123            Err(FileError::Unsupported(file.name.clone()))
124        })
125    }
126
127    fn store<'a>(
128        &'a self,
129        name: &'a str,
130        mime_type: Option<&'a str>,
131        bytes: Vec<u8>,
132    ) -> BoxFuture<'a, Result<File, FileError>> {
133        Box::pin(async move {
134            // Only the first resolver that supports storage is used; bytes cannot be
135            // cloned cheaply, so resolvers are probed in order with the same buffer.
136            let mut bytes = Some(bytes);
137            for resolver in &self.resolvers {
138                let Some(b) = bytes.take() else { break };
139                match resolver.store(name, mime_type, b.clone()).await {
140                    Err(FileError::Unsupported(_)) => bytes = Some(b),
141                    other => return other,
142                }
143            }
144            Err(FileError::Unsupported(name.to_owned()))
145        })
146    }
147}
148
149/// Decode a `data:[<mime>][;base64],<payload>` URL.
150pub fn decode_data_url(name: &str, url: &str, max_bytes: u64) -> Result<FileBytes, FileError> {
151    let invalid = |m: &str| FileError::Invalid { name: name.to_owned(), message: m.to_owned() };
152    let rest = url.strip_prefix("data:").ok_or_else(|| invalid("not a data URL"))?;
153    let (meta, payload) = rest.split_once(',').ok_or_else(|| invalid("data URL without ','"))?;
154    // The `;base64` flag is case-insensitive (RFC 2397 media-type parameters).
155    let is_base64 = meta.len() >= 7 && meta.get(meta.len() - 7..).is_some_and(|s| s.eq_ignore_ascii_case(";base64"));
156    let meta = if is_base64 { meta.get(..meta.len() - 7).unwrap_or_default() } else { meta };
157    let mime = meta.split(';').next().filter(|m| !m.is_empty()).map(str::to_owned);
158    let too_large = || FileError::TooLarge { name: name.to_owned(), limit: max_bytes };
159    let bytes = if is_base64 {
160        // Reject before decoding: base64 yields 3 bytes per 4 characters.
161        let estimate = u64::try_from(payload.len() / 4 * 3).unwrap_or(u64::MAX);
162        if estimate > max_bytes.saturating_add(3) {
163            return Err(too_large());
164        }
165        base64_decode(payload).ok_or_else(|| invalid("invalid base64 payload"))?
166    } else {
167        percent_decode(payload)
168    };
169    if u64::try_from(bytes.len()).unwrap_or(u64::MAX) > max_bytes {
170        return Err(FileError::TooLarge { name: name.to_owned(), limit: max_bytes });
171    }
172    Ok(FileBytes { bytes, mime_type: mime })
173}
174
175/// Standard and URL-safe base64 decoder (padding optional, whitespace ignored).
176pub fn base64_decode(input: &str) -> Option<Vec<u8>> {
177    fn val(c: u8) -> Option<u32> {
178        match c {
179            b'A'..=b'Z' => Some(u32::from(c - b'A')),
180            b'a'..=b'z' => Some(u32::from(c - b'a') + 26),
181            b'0'..=b'9' => Some(u32::from(c - b'0') + 52),
182            b'+' | b'-' => Some(62),
183            b'/' | b'_' => Some(63),
184            _ => None,
185        }
186    }
187    let mut out = Vec::with_capacity(input.len() * 3 / 4);
188    let mut buf: u32 = 0;
189    let mut bits = 0u32;
190    let mut padding = false;
191    for c in input.bytes() {
192        if c.is_ascii_whitespace() {
193            continue;
194        }
195        if c == b'=' {
196            padding = true;
197            continue;
198        }
199        // Data after padding means a corrupted or concatenated payload.
200        if padding {
201            return None;
202        }
203        buf = (buf << 6) | val(c)?;
204        bits += 6;
205        if bits >= 8 {
206            bits -= 8;
207            out.push(u8::try_from((buf >> bits) & 0xff).ok()?);
208        }
209    }
210    // A single leftover character cannot encode a byte.
211    if bits >= 6 {
212        return None;
213    }
214    Some(out)
215}
216
217/// Standard base64 encoder with padding.
218pub fn base64_encode(bytes: &[u8]) -> String {
219    const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
220    let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
221    for chunk in bytes.chunks(3) {
222        let b = [
223            chunk.first().copied().unwrap_or(0),
224            chunk.get(1).copied().unwrap_or(0),
225            chunk.get(2).copied().unwrap_or(0),
226        ];
227        let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
228        for i in 0..4 {
229            if i <= chunk.len() {
230                let idx = usize::try_from((n >> (18 - 6 * i)) & 63).unwrap_or(0);
231                out.push(char::from(TABLE.get(idx).copied().unwrap_or(b'A')));
232            } else {
233                out.push('=');
234            }
235        }
236    }
237    out
238}
239
240fn percent_decode(s: &str) -> Vec<u8> {
241    let bytes = s.as_bytes();
242    let mut out = Vec::with_capacity(bytes.len());
243    let mut i = 0;
244    while let Some(&b) = bytes.get(i) {
245        if b == b'%' {
246            let hex = bytes.get(i + 1..i + 3).and_then(|h| std::str::from_utf8(h).ok());
247            if let Some(v) = hex.and_then(|h| u8::from_str_radix(h, 16).ok()) {
248                out.push(v);
249                i += 3;
250                continue;
251            }
252        }
253        out.push(b);
254        i += 1;
255    }
256    out
257}
258
259#[cfg(test)]
260mod tests {
261    use super::*;
262
263    #[test]
264    fn base64_roundtrip() {
265        for s in ["", "a", "ab", "abc", "hello world", "\u{0}\u{1}\u{ff}"] {
266            let enc = base64_encode(s.as_bytes());
267            assert_eq!(base64_decode(&enc).unwrap(), s.as_bytes(), "{s:?}");
268        }
269        assert_eq!(base64_encode(b"hello"), "aGVsbG8=");
270    }
271
272    #[test]
273    fn data_url_decoding() {
274        let fb = decode_data_url("f", "data:text/plain;base64,aGVsbG8=", 100).unwrap();
275        assert_eq!(fb.bytes, b"hello");
276        assert_eq!(fb.mime_type.as_deref(), Some("text/plain"));
277        let fb = decode_data_url("f", "data:,a%20b", 100).unwrap();
278        assert_eq!(fb.bytes, b"a b");
279        // `;BASE64` is the same flag.
280        let fb = decode_data_url("f", "data:text/plain;BASE64,aGVsbG8=", 100).unwrap();
281        assert_eq!(fb.bytes, b"hello");
282        // Oversized payloads are rejected before decoding.
283        let big = format!("data:;base64,{}", "A".repeat(4000));
284        assert!(matches!(decode_data_url("f", &big, 100), Err(FileError::TooLarge { .. })));
285    }
286
287    #[test]
288    fn base64_rejects_trailing_garbage() {
289        assert_eq!(base64_decode("aGk="), Some(b"hi".to_vec()));
290        assert_eq!(base64_decode("aGk=aGk="), None);
291        assert_eq!(base64_decode("aGVsbG8"), Some(b"hello".to_vec()));
292        assert_eq!(base64_decode("a"), None);
293        assert!(matches!(decode_data_url("f", "data:,abc", 2), Err(FileError::TooLarge { .. })));
294    }
295
296    #[test]
297    fn chain_falls_through_unsupported() {
298        let chain = ChainResolver::with_data_urls();
299        let remote = File { name: "x".into(), url: Some("https://e.com/x".into()), ..File::default() };
300        let r = futures::executor::block_on(chain.read(&remote, 10));
301        assert!(matches!(r, Err(FileError::Unsupported(_))));
302        let inline = File { name: "y".into(), url: Some("data:,hi".into()), ..File::default() };
303        let r = futures::executor::block_on(chain.read(&inline, 10)).unwrap();
304        assert_eq!(r.bytes, b"hi");
305    }
306}