Skip to main content

simpleplatform_sdk/
storage.rs

1//! Putting a file into the platform's store, and reading one back out.
2//!
3//! Two ways in — bytes this action already holds, and a file behind a URL — and
4//! one thing out of both: a [`DocumentHandle`], which is the value a `:document`
5//! field holds. The way out is [`read`], which takes that handle and answers with
6//! the file's bytes.
7//!
8//! ```
9//! # use simpleplatform_sdk::prelude::*;
10//! # use simpleplatform_sdk::testing;
11//! use simpleplatform_sdk::storage::Target;
12//!
13//! # let _session = testing::install(|_name, _params| {
14//! #     Ok(json!({
15//! #         "file_hash": "c3ab8ff13720e8ad9047dd39466b3c89",
16//! #         "filename": "report.pdf",
17//! #         "mime_type": "application/pdf",
18//! #         "size": 6,
19//! #         "storage_path": "documents/c3/ab/c3ab8ff13720e8ad9047dd39466b3c89",
20//! #     }))
21//! # });
22//! let target = Target::new("dev.simple.system", "documents", "attachment");
23//! let handle = simple::storage::upload_buffer(b"foobar", "report.pdf", "application/pdf", target)?;
24//!
25//! assert_eq!(handle.filename, "report.pdf");
26//! assert_eq!(handle.size, 6);
27//! # Ok::<(), Error>(())
28//! ```
29//!
30//! # The handle is the point
31//!
32//! An upload stores the file and answers with a handle to it. Attaching that
33//! handle to a record is a separate step, and the one that changes tenant data:
34//! write the handle into the `:document` field with [`crate::graphql::mutate`]
35//! once you have it. So the two halves stay separable — the file is in the store
36//! whether or not the record was written, and the store is the only thing
37//! [`upload_buffer`] and [`upload_external`] touch.
38//!
39//! # The same bytes are the same file
40//!
41//! The store is addressed by the SHA-256 of the contents, which is the
42//! `file_hash` on the handle. Identical bytes land under the identical hash and
43//! are kept once, so uploading a file the store already holds costs a hash and
44//! answers with the handle that was already there.
45//!
46//! # Reading a file back
47//!
48//! ```
49//! # use simpleplatform_sdk::prelude::*;
50//! # use simpleplatform_sdk::storage::DocumentHandle;
51//! # use simpleplatform_sdk::testing;
52//! # let file = b"%PDF-1.7 ...".to_vec();
53//! # let served = file.clone();
54//! # let _session = testing::install(|_name, _params| Ok(json!({ "size": 12 })))
55//! #     .with_bytes(move |_name, params| {
56//! #         let offset = params["offset"].as_u64().unwrap() as usize;
57//! #         let length = params["length"].as_u64().unwrap() as usize;
58//! #         Ok(served[offset..offset + length].to_vec())
59//! #     });
60//! # let handle = DocumentHandle {
61//! #     file_hash: "9f86d081884c".into(),
62//! #     filename: "statement.pdf".into(),
63//! #     mime_type: "application/pdf".into(),
64//! #     size: 12,
65//! #     storage_path: "_staged/9f86d081884c".into(),
66//! # };
67//! let bytes = simple::storage::read(&handle)?;
68//!
69//! assert_eq!(bytes, file);
70//! # Ok::<(), Error>(())
71//! ```
72//!
73//! The bytes cross from the host as they are, with no JSON and no base64. The
74//! size is asked for first, the buffer is allocated once at exactly that size,
75//! and the file arrives in ranges of at most [`MAX_RANGE_BYTES`], each written by
76//! the host straight into its place in that buffer. [`read_range`] reads part of
77//! a file the same way, from the size first, and [`size`] answers how large it
78//! is without reading any of it.
79//!
80//! Reading is for server actions. A browser action is refused.
81//!
82//! # Why the bytes are encoded here
83//!
84//! The call travels as JSON, and JSON carries text. So a buffer is base64 on the
85//! wire — encoded once, at this boundary, in the same standard alphabet with the
86//! same padding every SDK uses, and decoded by the platform before the file is
87//! stored. An action passes `&[u8]` and never sees the encoding.
88
89use std::fmt;
90
91use serde::{Deserialize, Serialize};
92use serde_json::{json, Value};
93
94use crate::codes::Code;
95use crate::error::{Error, Fault};
96use crate::host;
97
98/// The host action that stores a file and answers with its handle.
99const UPLOAD_EXTERNAL: &str = "action:storage/upload-external";
100
101/// The host action that answers a stored file's size.
102const STAT: &str = "action:storage/stat";
103
104/// The host action that answers one range of a stored file as its bytes.
105const READ: &str = "action:storage/read";
106
107/// The longest range the host answers one read with.
108///
109/// [`read`] and [`read_range`] cover anything longer in ranges of this size,
110/// into one buffer, so it bounds what the host holds for one call rather than
111/// what an action can read.
112pub const MAX_RANGE_BYTES: u64 = 16 * 1024 * 1024;
113
114/// Where a size stops being one every SDK counts exactly.
115///
116/// The TypeScript and Go SDKs read the size as a floating-point number, which
117/// holds every whole number below 2^53 exactly and rounds past it. A size at or
118/// past this is refused here too, so the same answer from the store reads the
119/// same in all three.
120const MAX_EXACT_SIZE: u64 = 1 << 53;
121
122/// A stored file, as a `:document` field holds it.
123///
124/// This is a pointer to the contents rather than the contents: it is what an
125/// upload answers with, what a record stores, and what identifies the file
126/// afterwards.
127#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
128pub struct DocumentHandle {
129    /// The SHA-256 of the contents, which is the file's identity in the store.
130    pub file_hash: String,
131    /// The name the file is stored under.
132    pub filename: String,
133    /// The media type of the contents, such as `application/pdf`.
134    pub mime_type: String,
135    /// The size of the contents in bytes.
136    pub size: u64,
137    /// Where the store keeps the file.
138    pub storage_path: String,
139}
140
141/// The record field a stored file is destined for.
142#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
143pub struct Target {
144    /// The app that owns the table.
145    pub app_id: String,
146    /// The table the record lives in.
147    pub table_name: String,
148    /// The `:document` field the handle is stored in.
149    pub field_name: String,
150}
151
152impl Target {
153    /// A target named by its app, its table and its field.
154    ///
155    /// ```
156    /// use simpleplatform_sdk::storage::Target;
157    ///
158    /// let target = Target::new("dev.simple.system", "documents", "attachment");
159    ///
160    /// assert_eq!(target.field_name, "attachment");
161    /// ```
162    pub fn new(
163        app_id: impl Into<String>,
164        table_name: impl Into<String>,
165        field_name: impl Into<String>,
166    ) -> Target {
167        Target {
168            app_id: app_id.into(),
169            table_name: table_name.into(),
170            field_name: field_name.into(),
171        }
172    }
173}
174
175/// The credential a URL is read with.
176///
177/// It is `#[non_exhaustive]`: a `match` on it needs a `_` arm, so a scheme added
178/// later does not break an action that was already written.
179#[non_exhaustive]
180#[derive(Clone, Serialize)]
181#[serde(tag = "type", rename_all = "lowercase")]
182pub enum Auth {
183    /// A username and a password.
184    Basic {
185        /// The username.
186        username: String,
187        /// The password.
188        password: String,
189    },
190    /// A bearer token.
191    Bearer {
192        /// The token.
193        bearer_token: String,
194    },
195}
196
197impl Auth {
198    /// Read the URL with a bearer token.
199    ///
200    /// ```
201    /// use simpleplatform_sdk::storage::Auth;
202    ///
203    /// let auth = Auth::bearer("t-1234");
204    ///
205    /// assert!(matches!(auth, Auth::Bearer { .. }));
206    /// ```
207    pub fn bearer(bearer_token: impl Into<String>) -> Auth {
208        Auth::Bearer {
209            bearer_token: bearer_token.into(),
210        }
211    }
212
213    /// Read the URL with a username and a password.
214    pub fn basic(username: impl Into<String>, password: impl Into<String>) -> Auth {
215        Auth::Basic {
216            username: username.into(),
217            password: password.into(),
218        }
219    }
220}
221
222impl fmt::Debug for Auth {
223    /// The scheme, and never the secret.
224    ///
225    /// A credential is written by hand and read by a host, and the one place it
226    /// has no business appearing is a log line. So this renders the scheme, and
227    /// the username that names the account, and stops there.
228    ///
229    /// ```
230    /// use simpleplatform_sdk::storage::Auth;
231    ///
232    /// assert_eq!(format!("{:?}", Auth::bearer("t-1234")), "Bearer { .. }");
233    /// ```
234    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
235        match self {
236            Auth::Basic { username, .. } => formatter
237                .debug_struct("Basic")
238                .field("username", username)
239                .finish_non_exhaustive(),
240            Auth::Bearer { .. } => formatter.debug_struct("Bearer").finish_non_exhaustive(),
241        }
242    }
243}
244
245/// The file an external upload reads.
246#[derive(Clone, Debug, Serialize)]
247pub struct Source {
248    /// The URL the file is read from.
249    pub url: String,
250    /// The credential the URL is read with, for a URL that needs one.
251    #[serde(skip_serializing_if = "Option::is_none")]
252    pub auth: Option<Auth>,
253}
254
255impl Source {
256    /// A file at this URL, read without a credential.
257    pub fn url(url: impl Into<String>) -> Source {
258        Source {
259            url: url.into(),
260            auth: None,
261        }
262    }
263
264    /// The same file, read with this credential.
265    ///
266    /// ```
267    /// use simpleplatform_sdk::storage::{Auth, Source};
268    ///
269    /// let source = Source::url("https://example.com/report.pdf").with_auth(Auth::bearer("t-1234"));
270    ///
271    /// assert!(source.auth.is_some());
272    /// ```
273    pub fn with_auth(mut self, auth: Auth) -> Source {
274        self.auth = Some(auth);
275        self
276    }
277}
278
279/// Store bytes this action already holds.
280///
281/// The buffer, the name and the media type are all required, and each is checked
282/// before anything is sent: an empty buffer, a blank name, a blank media type or
283/// an incomplete [`Target`] is refused where it costs nothing.
284///
285/// ```
286/// # use simpleplatform_sdk::prelude::*;
287/// # use simpleplatform_sdk::testing;
288/// use simpleplatform_sdk::storage::Target;
289///
290/// # let _session = testing::install(|_name, params| {
291/// #     assert_eq!(params["source"]["bytes"], json!("Zm9vYmFy"));
292/// #     Ok(json!({
293/// #         "file_hash": "c3ab8ff137",
294/// #         "filename": "notes.txt",
295/// #         "mime_type": "text/plain",
296/// #         "size": 6,
297/// #         "storage_path": "documents/c3/ab/c3ab8ff137",
298/// #     }))
299/// # });
300/// let handle = simple::storage::upload_buffer(
301///     b"foobar",
302///     "notes.txt",
303///     "text/plain",
304///     Target::new("dev.simple.system", "documents", "attachment"),
305/// )?;
306///
307/// assert_eq!(handle.file_hash, "c3ab8ff137");
308/// # Ok::<(), Error>(())
309/// ```
310pub fn upload_buffer(
311    bytes: &[u8],
312    filename: &str,
313    mime_type: &str,
314    target: Target,
315) -> Result<DocumentHandle, Error> {
316    if bytes.is_empty() {
317        return Err(
318            Error::invalid("An upload needs at least one byte, and this buffer is empty.")
319                .hint("Pass the file's contents."),
320        );
321    }
322
323    if filename.trim().is_empty() {
324        return Err(Error::invalid("An upload needs a filename.")
325            .hint("Pass the name the file is stored under, such as report.pdf."));
326    }
327
328    if mime_type.trim().is_empty() {
329        return Err(Error::invalid("An upload needs a media type.")
330            .hint("Pass the type of the contents, such as application/pdf."));
331    }
332
333    check_target(&target)?;
334
335    let source = json!({
336        "bytes": encode(bytes),
337        "filename": filename,
338        "mime_type": mime_type,
339    });
340
341    send(source, target)
342}
343
344/// Store a file the platform reads from a URL.
345///
346/// The URL and any credential on it are checked before anything is sent, as is
347/// the [`Target`].
348///
349/// ```
350/// # use simpleplatform_sdk::prelude::*;
351/// # use simpleplatform_sdk::testing;
352/// use simpleplatform_sdk::storage::{Auth, Source, Target};
353///
354/// # let _session = testing::install(|_name, params| {
355/// #     assert_eq!(params["source"]["auth"]["type"], json!("bearer"));
356/// #     Ok(json!({
357/// #         "file_hash": "9f86d081884c",
358/// #         "filename": "statement.pdf",
359/// #         "mime_type": "application/pdf",
360/// #         "size": 81_920,
361/// #         "storage_path": "documents/9f/86/9f86d081884c",
362/// #     }))
363/// # });
364/// let handle = simple::storage::upload_external(
365///     Source::url("https://example.com/statement.pdf").with_auth(Auth::bearer("t-1234")),
366///     Target::new("dev.simple.system", "documents", "attachment"),
367/// )?;
368///
369/// assert_eq!(handle.size, 81_920);
370/// # Ok::<(), Error>(())
371/// ```
372pub fn upload_external(source: Source, target: Target) -> Result<DocumentHandle, Error> {
373    if source.url.trim().is_empty() {
374        return Err(
375            Error::invalid("An external upload needs a URL to read from.")
376                .hint("Pass the address of the file, or use upload_buffer for bytes you hold."),
377        );
378    }
379
380    if let Some(auth) = &source.auth {
381        check_auth(auth)?;
382    }
383
384    check_target(&target)?;
385
386    send(serde_json::to_value(&source)?, target)
387}
388
389/// One upload, one round trip, one handle.
390fn send(source: Value, target: Target) -> Result<DocumentHandle, Error> {
391    let params = json!({ "source": source, "target": serde_json::to_value(&target)? });
392
393    let stored = host::transport()?
394        .call(UPLOAD_EXTERNAL.to_string(), params)
395        .map_err(|cause| {
396            Error::Host(Fault::new(Code::unspecified(), cause.message())).hint(
397                "Nothing was attached to the record. The store is addressed by content, \
398                 so the same bytes uploaded again answer with the same handle.",
399            )
400        })?;
401
402    serde_json::from_value(stored).map_err(|cause| {
403        Error::Json(Fault::new(
404            Code::unspecified(),
405            format!("The upload finished and its handle could not be read: {cause}"),
406        ))
407        .hint(
408            "Report this. The file is stored under the hash of its contents, \
409             so uploading the same bytes again answers with the handle.",
410        )
411    })
412}
413
414/// How many bytes a stored file holds, from the store's own record of it.
415///
416/// ```
417/// # use simpleplatform_sdk::prelude::*;
418/// # use simpleplatform_sdk::storage::DocumentHandle;
419/// # use simpleplatform_sdk::testing;
420/// # let _session = testing::install(|name, _params| {
421/// #     assert_eq!(name, "action:storage/stat");
422/// #     Ok(json!({ "size": 81_920 }))
423/// # });
424/// # let handle = DocumentHandle {
425/// #     file_hash: "9f86d081884c".into(),
426/// #     filename: "statement.pdf".into(),
427/// #     mime_type: "application/pdf".into(),
428/// #     size: 81_920,
429/// #     storage_path: "_staged/9f86d081884c".into(),
430/// # };
431/// assert_eq!(simple::storage::size(&handle)?, 81_920);
432/// # Ok::<(), Error>(())
433/// ```
434pub fn size(handle: &DocumentHandle) -> Result<u64, Error> {
435    check_handle(handle)?;
436
437    let answer = host::transport()?.call(STAT.to_string(), json!({ "handle": handle }))?;
438
439    answer
440        .get("size")
441        .and_then(Value::as_u64)
442        .filter(|size| *size < MAX_EXACT_SIZE)
443        .ok_or_else(|| {
444            Error::Host(Fault::new(
445                Code::unspecified(),
446                format!("{STAT} answered without a size: {answer}"),
447            ))
448        })
449}
450
451/// The whole of a stored file.
452///
453/// The size is asked for first, the buffer is allocated once at exactly that
454/// size, and every range is written by the host straight into its place in it.
455/// A file this action has no memory for is refused before anything is read,
456/// and a range answered short — a file that changed while it was read — is
457/// refused rather than handed over incomplete.
458pub fn read(handle: &DocumentHandle) -> Result<Vec<u8>, Error> {
459    let total = size(handle)?;
460
461    read_span(handle, 0, total)
462}
463
464/// Up to `length` bytes of a stored file, starting `offset` bytes in.
465///
466/// The size is asked for first, so the range is held at what the file has past
467/// `offset`: one that runs past the end answers with exactly the bytes up to
468/// the end, and one that starts at or past the end is refused before any range
469/// is read. What is left is read as [`read`] reads a whole file — in ranges of
470/// at most [`MAX_RANGE_BYTES`], into one buffer allocated once at that length,
471/// each range answered in full or refused.
472///
473/// Asking for the size first is also what makes a host that cannot read
474/// stored files refuse in its own words, before any range is asked of it.
475pub fn read_range(handle: &DocumentHandle, offset: u64, length: u64) -> Result<Vec<u8>, Error> {
476    check_handle(handle)?;
477
478    if length == 0 {
479        return Err(Error::invalid("A range needs at least one byte.")
480            .hint("Pass a length of one or more, or ask size() how large the file is."));
481    }
482
483    let total = size(handle)?;
484
485    if offset >= total {
486        return Err(past_end(offset, total));
487    }
488
489    read_span(handle, offset, length.min(total - offset))
490}
491
492/// Exactly `length` bytes from `offset`, which the file's size says are there.
493///
494/// The buffer is allocated once at `length` before any range is asked for, and
495/// every range must arrive whole.
496fn read_span(handle: &DocumentHandle, offset: u64, length: u64) -> Result<Vec<u8>, Error> {
497    let mut bytes = Vec::new();
498    let capacity = usize::try_from(length).map_err(|_too_large| too_large(length))?;
499    bytes
500        .try_reserve_exact(capacity)
501        .map_err(|_no_memory| too_large(length))?;
502
503    let transport = host::transport()?;
504    let mut done = 0;
505
506    while done < length {
507        let wanted = MAX_RANGE_BYTES.min(length - done);
508        let appended = transport.call_bytes(
509            READ.to_string(),
510            range(handle, offset + done, wanted),
511            &mut bytes,
512        )?;
513
514        if appended as u64 != wanted {
515            return Err(short(offset + done, wanted, appended));
516        }
517
518        done += wanted;
519    }
520
521    Ok(bytes)
522}
523
524/// The parameters of one range read.
525fn range(handle: &DocumentHandle, offset: u64, length: u64) -> Value {
526    json!({ "handle": handle, "offset": offset, "length": length })
527}
528
529/// A read this action cannot hold.
530fn too_large(length: u64) -> Error {
531    Error::failed(format!(
532        "Reading {length} bytes needs more memory than this action has."
533    ))
534    .hint("Raise the action's mem_limit, or read the file in parts with read_range.")
535}
536
537/// A range that starts where the file has nothing left to read.
538fn past_end(offset: u64, total: u64) -> Error {
539    Error::invalid(format!(
540        "The range starts at byte {offset}, at or past the end of the file, which is {total} bytes."
541    ))
542    .hint("Ask size() how large the file is, and start the range before its end.")
543}
544
545/// A range the host answered with fewer bytes than the file's size promised.
546fn short(offset: u64, length: u64, appended: usize) -> Error {
547    Error::Host(Fault::new(
548        Code::unspecified(),
549        format!(
550            "The file answered {appended} bytes for the {length} at offset {offset}, \
551             so it is not the size it was when the read began."
552        ),
553    ))
554    .hint("Read it again.")
555}
556
557/// Whether a handle names a stored file.
558fn check_handle(handle: &DocumentHandle) -> Result<(), Error> {
559    let named = [
560        ("storage_path", &handle.storage_path),
561        ("filename", &handle.filename),
562        ("file_hash", &handle.file_hash),
563    ];
564
565    for (member, value) in named {
566        if value.trim().is_empty() {
567            return Err(Error::invalid(format!("A document handle needs {member}."))
568                .hint("Pass the handle exactly as the :document field holds it."));
569        }
570    }
571
572    Ok(())
573}
574
575/// Whether a target names a field to attach to.
576fn check_target(target: &Target) -> Result<(), Error> {
577    let named = [
578        ("app_id", &target.app_id),
579        ("table_name", &target.table_name),
580        ("field_name", &target.field_name),
581    ];
582
583    for (member, value) in named {
584        if value.trim().is_empty() {
585            return Err(Error::invalid(format!("A storage target needs {member}."))
586                .hint("Name the app, the table, and the :document field the handle goes in."));
587        }
588    }
589
590    Ok(())
591}
592
593/// Whether a credential carries what its scheme needs.
594///
595/// The scheme itself is settled by [`Auth`] — there are two, and each holds its
596/// own members — so what is left to check is that the members say something.
597fn check_auth(auth: &Auth) -> Result<(), Error> {
598    match auth {
599        Auth::Basic { username, password } => {
600            if username.trim().is_empty() {
601                return Err(Error::invalid("Basic authentication needs a username.")
602                    .hint("Pass the username, or read the URL without a credential."));
603            }
604
605            if password.trim().is_empty() {
606                return Err(Error::invalid("Basic authentication needs a password.")
607                    .hint("Pass the password, or read the URL without a credential."));
608            }
609        }
610        Auth::Bearer { bearer_token } => {
611            if bearer_token.trim().is_empty() {
612                return Err(Error::invalid("Bearer authentication needs a token.")
613                    .hint("Pass the token, or read the URL without a credential."));
614            }
615        }
616    }
617
618    Ok(())
619}
620
621/// The symbols base64 spends, in their standard order.
622const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
623
624/// What fills a group that ran out of bytes.
625const PAD: char = '=';
626
627/// The bytes as base64: three bytes to four symbols, padded to a multiple of
628/// four, in the standard alphabet.
629fn encode(bytes: &[u8]) -> String {
630    let mut encoded = String::with_capacity(bytes.len().div_ceil(3) * 4);
631
632    for group in bytes.chunks(3) {
633        let mut packed = 0_u32;
634
635        // Left to right: the first byte takes the top eight of twenty-four bits,
636        // and a group of one or two leaves the rest zero.
637        for (index, byte) in group.iter().enumerate() {
638            packed |= u32::from(*byte) << (16 - 8 * index);
639        }
640
641        encoded.push(symbol(packed >> 18));
642        encoded.push(symbol(packed >> 12));
643        encoded.push(if group.len() > 1 {
644            symbol(packed >> 6)
645        } else {
646            PAD
647        });
648        encoded.push(if group.len() > 2 { symbol(packed) } else { PAD });
649    }
650
651    encoded
652}
653
654/// The symbol for the low six bits of `packed`.
655fn symbol(packed: u32) -> char {
656    // Six bits count to sixty-three and the alphabet holds sixty-four symbols,
657    // so there is always one to answer with.
658    char::from(ALPHABET[(packed & 0b11_1111) as usize])
659}
660
661#[cfg(test)]
662mod tests {
663    use serde_json::json;
664
665    use super::*;
666    use crate::testing;
667
668    /// What the host answers with for an upload that landed.
669    fn stored() -> Value {
670        json!({
671            "file_hash": "c3ab8ff13720e8ad9047dd39466b3c89",
672            "filename": "report.pdf",
673            "mime_type": "application/pdf",
674            "size": 6,
675            "storage_path": "documents/c3/ab/c3ab8ff13720e8ad9047dd39466b3c89",
676        })
677    }
678
679    fn target() -> Target {
680        Target::new("dev.simple.system", "documents", "attachment")
681    }
682
683    fn handle() -> DocumentHandle {
684        DocumentHandle {
685            file_hash: "9f86d081884c7d65".to_string(),
686            filename: "statement.pdf".to_string(),
687            mime_type: "application/pdf".to_string(),
688            size: 0,
689            storage_path: "_staged/9f86d081884c7d65".to_string(),
690        }
691    }
692
693    /// Every byte value, so bytes passed through any text decoding on the way
694    /// could not compare equal.
695    fn every_byte(count: usize) -> Vec<u8> {
696        (0..count).map(|i| ((i * 7 + 3) % 256) as u8).collect()
697    }
698
699    /// A session serving `file` the way the host does: its size from the store,
700    /// and each range as its bytes, up to the end of the file.
701    fn serving(file: Vec<u8>) -> testing::Session {
702        let size = file.len();
703
704        testing::install(move |name, _params| {
705            assert_eq!(name, "action:storage/stat");
706            Ok(json!({ "size": size }))
707        })
708        .with_bytes(move |name, params| {
709            assert_eq!(name, "action:storage/read");
710            let offset = params["offset"].as_u64().unwrap() as usize;
711            let length = params["length"].as_u64().unwrap() as usize;
712
713            if offset >= file.len() {
714                return Err(Error::invalid(
715                    "'offset' is at or past the end of the file.",
716                ));
717            }
718
719            Ok(file[offset..file.len().min(offset + length)].to_vec())
720        })
721    }
722
723    #[test]
724    fn a_read_answers_the_file_byte_for_byte() {
725        let file = every_byte(5_000);
726        let _session = serving(file.clone());
727
728        assert_eq!(read(&handle()).unwrap(), file);
729    }
730
731    #[test]
732    fn a_read_allocates_once_at_exactly_the_size_the_store_reports() {
733        let file = every_byte(40 * 1024 * 1024 + 17);
734        let _session = serving(file.clone());
735
736        let bytes = read(&handle()).unwrap();
737
738        assert_eq!(bytes.len(), file.len());
739        assert_eq!(bytes.capacity(), file.len());
740        assert!(bytes == file);
741    }
742
743    #[test]
744    fn a_large_read_asks_for_ranges_no_longer_than_the_host_answers() {
745        let file = every_byte(40 * 1024 * 1024 + 17);
746        let session = serving(file);
747
748        read(&handle()).unwrap();
749
750        let ranges: Vec<(u64, u64)> = session
751            .calls()
752            .iter()
753            .filter(|call| call.name == "action:storage/read")
754            .map(|call| {
755                (
756                    call.params["offset"].as_u64().unwrap(),
757                    call.params["length"].as_u64().unwrap(),
758                )
759            })
760            .collect();
761
762        let mib = 1024 * 1024;
763        assert_eq!(
764            ranges,
765            vec![
766                (0, 16 * mib),
767                (16 * mib, 16 * mib),
768                (32 * mib, 8 * mib + 17)
769            ]
770        );
771    }
772
773    #[test]
774    fn every_read_carries_the_handle_it_was_given() {
775        let session = serving(every_byte(10));
776
777        read(&handle()).unwrap();
778
779        for call in session.calls() {
780            assert_eq!(
781                call.params["handle"],
782                serde_json::to_value(handle()).unwrap()
783            );
784        }
785    }
786
787    #[test]
788    fn an_empty_file_is_read_without_asking_for_a_range() {
789        let session = serving(Vec::new());
790
791        assert_eq!(read(&handle()).unwrap(), Vec::<u8>::new());
792        assert_eq!(session.calls().len(), 1);
793    }
794
795    #[test]
796    fn a_range_answered_short_is_refused_rather_than_handed_over() {
797        let _session = testing::install(|_name, _params| Ok(json!({ "size": 100 })))
798            .with_bytes(|_name, _params| Ok(vec![0; 60]));
799
800        let error = read(&handle()).unwrap_err();
801
802        assert!(error.message().contains("60 bytes for the 100 at offset 0"));
803    }
804
805    #[test]
806    fn a_host_refusal_reaches_the_caller_with_its_own_message() {
807        let _session = testing::install(|_name, _params| Ok(json!({ "size": 10 }))).with_bytes(
808            |_name, _params| Err(Error::invalid("No stored file matches this handle.")),
809        );
810
811        let error = read(&handle()).unwrap_err();
812
813        assert!(error
814            .message()
815            .contains("action:storage/read failed: No stored file matches this handle."));
816    }
817
818    #[test]
819    fn a_file_larger_than_this_action_can_hold_is_refused_before_any_range() {
820        let largest = MAX_EXACT_SIZE - 1;
821        let session = testing::install(move |_name, _params| Ok(json!({ "size": largest })));
822
823        let error = read(&handle()).unwrap_err();
824
825        assert_eq!(
826            error.message(),
827            format!("Reading {largest} bytes needs more memory than this action has.")
828        );
829        assert_eq!(session.calls().len(), 1);
830
831        let error = read_range(&handle(), 0, largest).unwrap_err();
832
833        assert!(error
834            .message()
835            .contains("needs more memory than this action has"));
836        assert_eq!(
837            session.calls().len(),
838            2,
839            "one size lookup each, and no range"
840        );
841    }
842
843    #[test]
844    fn size_answers_what_the_store_reports() {
845        let _session = testing::install(|_name, _params| Ok(json!({ "size": 81_920 })));
846
847        assert_eq!(size(&handle()).unwrap(), 81_920);
848    }
849
850    #[test]
851    fn a_size_that_is_not_a_count_every_sdk_holds_exactly_is_refused() {
852        for answer in [
853            json!({}),
854            json!({ "size": -1 }),
855            json!({ "size": 1.5 }),
856            json!({ "size": "10" }),
857            json!({ "size": MAX_EXACT_SIZE }),
858            json!("10"),
859        ] {
860            let reply = answer.clone();
861            let _session = testing::install(move |_name, _params| Ok(reply.clone()));
862
863            let error = size(&handle()).unwrap_err();
864
865            assert_eq!(
866                error.message(),
867                format!("action:storage/stat answered without a size: {answer}")
868            );
869        }
870    }
871
872    #[test]
873    fn a_size_refusal_keeps_the_hosts_own_message() {
874        let _session = testing::install(|_name, _params| {
875            Err(Error::invalid("No stored file matches this handle."))
876        });
877
878        let error = size(&handle()).unwrap_err();
879
880        assert_eq!(
881            error.message(),
882            "action:storage/stat failed: No stored file matches this handle."
883        );
884    }
885
886    #[test]
887    fn a_range_answers_exactly_that_range_after_asking_for_the_size() {
888        let file = every_byte(10_000);
889        let session = serving(file.clone());
890
891        assert_eq!(
892            read_range(&handle(), 4_000, 1_500).unwrap(),
893            file[4_000..5_500]
894        );
895
896        let names: Vec<String> = session.calls().into_iter().map(|call| call.name).collect();
897
898        assert_eq!(names, ["action:storage/stat", "action:storage/read"]);
899    }
900
901    #[test]
902    fn a_range_running_past_the_end_answers_up_to_the_end() {
903        let file = every_byte(10_000);
904        let _session = serving(file.clone());
905
906        assert_eq!(read_range(&handle(), 9_990, 100).unwrap(), file[9_990..]);
907    }
908
909    #[test]
910    fn a_range_starting_at_or_past_the_end_is_refused_before_any_range() {
911        for offset in [10, 11, 20] {
912            let session = serving(every_byte(10));
913
914            let error = read_range(&handle(), offset, 1).unwrap_err();
915
916            assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
917            assert_eq!(
918                error.message(),
919                format!(
920                    "The range starts at byte {offset}, at or past the end of the file, which is 10 bytes."
921                )
922            );
923            assert_eq!(
924                error.fault().hint(),
925                "Ask size() how large the file is, and start the range before its end."
926            );
927            assert!(ranges(&session).is_empty());
928        }
929    }
930
931    /// The offset and length of every range the host was asked for, in order.
932    fn ranges(session: &testing::Session) -> Vec<(u64, u64)> {
933        session
934            .calls()
935            .iter()
936            .filter(|call| call.name == "action:storage/read")
937            .map(|call| {
938                (
939                    call.params["offset"].as_u64().unwrap(),
940                    call.params["length"].as_u64().unwrap(),
941                )
942            })
943            .collect()
944    }
945
946    /// A file's size, the offset and the length a range asks for, and the
947    /// offset and length of every range the host is asked for in turn.
948    type Case = (u64, u64, u64, Vec<(u64, u64)>);
949
950    // The same cases pin the Go and TypeScript SDKs. A range is held at what
951    // the file has past its offset, so the file ending exactly where a range
952    // of 16 MiB does never leads to asking for a range at the end.
953    #[test]
954    fn a_range_is_held_at_what_the_file_has_at_every_boundary() {
955        let mib: u64 = 1024 * 1024;
956        let largest = every_byte(32 * 1024 * 1024 + 5);
957
958        let cases: Vec<Case> = vec![
959            (16 * mib, 0, 16 * mib, vec![(0, 16 * mib)]),
960            (16 * mib, 0, 16 * mib + 1, vec![(0, 16 * mib)]),
961            (
962                16 * mib + 1,
963                0,
964                16 * mib + 1,
965                vec![(0, 16 * mib), (16 * mib, 1)],
966            ),
967            (
968                32 * mib + 5,
969                0,
970                32 * mib + 5,
971                vec![(0, 16 * mib), (16 * mib, 16 * mib), (32 * mib, 5)],
972            ),
973            (
974                32 * mib + 5,
975                0,
976                48 * mib,
977                vec![(0, 16 * mib), (16 * mib, 16 * mib), (32 * mib, 5)],
978            ),
979            (
980                32 * mib + 5,
981                16 * mib,
982                16 * mib + 1,
983                vec![(16 * mib, 16 * mib), (32 * mib, 1)],
984            ),
985            (32 * mib, 16 * mib, 16 * mib + 1, vec![(16 * mib, 16 * mib)]),
986            (10, 9, 5, vec![(9, 1)]),
987        ];
988
989        for (size, offset, length, asked) in cases {
990            let file = largest[..size as usize].to_vec();
991            let session = serving(file.clone());
992
993            let bytes = read_range(&handle(), offset, length).unwrap();
994            let end = size.min(offset + length) as usize;
995
996            assert_eq!(
997                bytes.len(),
998                end - offset as usize,
999                "{size} {offset} {length}"
1000            );
1001            assert_eq!(bytes.capacity(), bytes.len(), "{size} {offset} {length}");
1002            assert!(
1003                bytes == file[offset as usize..end],
1004                "{size} {offset} {length}"
1005            );
1006            assert_eq!(ranges(&session), asked, "{size} {offset} {length}");
1007            assert_eq!(session.calls()[0].name, "action:storage/stat");
1008        }
1009
1010        for (size, offset, length) in [
1011            (0, 0, 1),
1012            (10, 10, 1),
1013            (10, 11, 1),
1014            (10, 20, 32 * mib),
1015            (16 * mib, 16 * mib, 16 * mib + 1),
1016        ] {
1017            let session = serving(largest[..size as usize].to_vec());
1018
1019            let error = read_range(&handle(), offset, length).unwrap_err();
1020
1021            assert!(
1022                error.message().contains(&format!(
1023                    "at or past the end of the file, which is {size} bytes"
1024                )),
1025                "{size} {offset} {length}: {}",
1026                error.message()
1027            );
1028            assert!(ranges(&session).is_empty(), "{size} {offset} {length}");
1029        }
1030
1031        for size in [0, 16 * mib, 16 * mib + 1, 32 * mib + 5] {
1032            let file = largest[..size as usize].to_vec();
1033            let _session = serving(file.clone());
1034
1035            assert!(read(&handle()).unwrap() == file, "{size}");
1036        }
1037    }
1038
1039    // A host from before stored files could be read answers a byte read in
1040    // its JSON envelope, as though the envelope were the file. The size is
1041    // asked for first, which such a host refuses as a request it does not
1042    // know, so the envelope is never asked for and never handed over.
1043    #[test]
1044    fn a_host_that_cannot_read_stored_files_refuses_before_any_range() {
1045        let session = testing::install(|name, _params| {
1046            Err(Error::invalid(format!("Unknown request: {name}")))
1047        })
1048        .with_bytes(|_name, _params| {
1049            Ok(
1050                br#"{"ok":false,"error":{"message":"Unknown request: action:storage/read"}}"#
1051                    .to_vec(),
1052            )
1053        });
1054
1055        for error in [
1056            read_range(&handle(), 0, 64).unwrap_err(),
1057            read_range(&handle(), 0, 32 * 1024 * 1024).unwrap_err(),
1058            read(&handle()).unwrap_err(),
1059        ] {
1060            assert_eq!(
1061                error.message(),
1062                "action:storage/stat failed: Unknown request: action:storage/stat"
1063            );
1064        }
1065
1066        assert!(ranges(&session).is_empty());
1067    }
1068
1069    #[test]
1070    fn an_empty_range_is_refused_before_anything_is_sent() {
1071        let session = serving(every_byte(10));
1072
1073        assert!(read_range(&handle(), 0, 0).is_err());
1074        assert!(session.calls().is_empty());
1075    }
1076
1077    #[test]
1078    fn a_handle_missing_what_names_the_file_is_refused_before_anything_is_sent() {
1079        let session = serving(every_byte(10));
1080
1081        for broken in [
1082            DocumentHandle {
1083                storage_path: String::new(),
1084                ..handle()
1085            },
1086            DocumentHandle {
1087                filename: " ".to_string(),
1088                ..handle()
1089            },
1090            DocumentHandle {
1091                file_hash: String::new(),
1092                ..handle()
1093            },
1094        ] {
1095            assert_eq!(
1096                read(&broken).unwrap_err().code().as_str(),
1097                "INVALID_TOOL_INPUT"
1098            );
1099        }
1100
1101        assert!(session.calls().is_empty());
1102    }
1103
1104    #[test]
1105    fn a_session_with_no_bytes_reply_says_how_to_give_it_one() {
1106        let _session = testing::install(|_name, _params| Ok(json!({ "size": 10 })));
1107
1108        let error = read(&handle()).unwrap_err();
1109
1110        assert!(error.message().contains("was given no way to"));
1111    }
1112
1113    #[test]
1114    fn a_buffer_travels_base64_encoded_beside_its_name_and_type() {
1115        let session = testing::install(|_name, _params| Ok(stored()));
1116
1117        upload_buffer(b"foobar", "report.pdf", "application/pdf", target()).unwrap();
1118
1119        let calls = session.calls();
1120
1121        assert_eq!(calls.len(), 1);
1122        assert_eq!(calls[0].name, "action:storage/upload-external");
1123        assert_eq!(
1124            calls[0].params["source"],
1125            json!({
1126                "bytes": "Zm9vYmFy",
1127                "filename": "report.pdf",
1128                "mime_type": "application/pdf",
1129            })
1130        );
1131        assert_eq!(
1132            calls[0].params["target"],
1133            json!({
1134                "app_id": "dev.simple.system",
1135                "table_name": "documents",
1136                "field_name": "attachment",
1137            })
1138        );
1139    }
1140
1141    #[test]
1142    fn a_stored_file_answers_with_every_member_of_its_handle() {
1143        let _session = testing::install(|_name, _params| Ok(stored()));
1144
1145        let handle = upload_buffer(b"foobar", "report.pdf", "application/pdf", target()).unwrap();
1146
1147        assert_eq!(
1148            handle,
1149            DocumentHandle {
1150                file_hash: "c3ab8ff13720e8ad9047dd39466b3c89".to_string(),
1151                filename: "report.pdf".to_string(),
1152                mime_type: "application/pdf".to_string(),
1153                size: 6,
1154                storage_path: "documents/c3/ab/c3ab8ff13720e8ad9047dd39466b3c89".to_string(),
1155            }
1156        );
1157    }
1158
1159    #[test]
1160    fn an_empty_buffer_is_refused_before_anything_is_sent() {
1161        let session = testing::install(|_name, _params| Ok(stored()));
1162
1163        let error = upload_buffer(&[], "report.pdf", "application/pdf", target()).unwrap_err();
1164
1165        assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
1166        assert!(session.calls().is_empty());
1167    }
1168
1169    #[test]
1170    fn a_blank_name_or_type_is_refused_before_anything_is_sent() {
1171        let session = testing::install(|_name, _params| Ok(stored()));
1172
1173        for (filename, mime_type) in [("   ", "application/pdf"), ("report.pdf", "")] {
1174            let error = upload_buffer(b"foobar", filename, mime_type, target()).unwrap_err();
1175
1176            assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
1177        }
1178
1179        assert!(session.calls().is_empty());
1180    }
1181
1182    #[test]
1183    fn a_target_missing_a_member_names_the_one_it_is_missing() {
1184        let session = testing::install(|_name, _params| Ok(stored()));
1185
1186        let incomplete = [
1187            (Target::new("", "documents", "attachment"), "app_id"),
1188            (
1189                Target::new("dev.simple.system", " ", "attachment"),
1190                "table_name",
1191            ),
1192            (
1193                Target::new("dev.simple.system", "documents", ""),
1194                "field_name",
1195            ),
1196        ];
1197
1198        for (target, member) in incomplete {
1199            let error = upload_buffer(b"foobar", "report.pdf", "application/pdf", target)
1200                .expect_err("an incomplete target names nothing to attach to");
1201
1202            assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
1203            assert!(error.message().contains(member), "{}", error.message());
1204        }
1205
1206        assert!(session.calls().is_empty());
1207    }
1208
1209    #[test]
1210    fn a_url_upload_carries_the_url_alone_when_it_needs_no_credential() {
1211        let session = testing::install(|_name, _params| Ok(stored()));
1212
1213        upload_external(Source::url("https://example.com/report.pdf"), target()).unwrap();
1214
1215        assert_eq!(
1216            session.calls()[0].params["source"],
1217            json!({ "url": "https://example.com/report.pdf" }),
1218            "a source with no credential carries no auth member"
1219        );
1220    }
1221
1222    #[test]
1223    fn a_credential_travels_under_its_own_scheme() {
1224        let session = testing::install(|_name, _params| Ok(stored()));
1225
1226        upload_external(
1227            Source::url("https://example.com/report.pdf").with_auth(Auth::bearer("t-1234")),
1228            target(),
1229        )
1230        .unwrap();
1231
1232        upload_external(
1233            Source::url("https://example.com/report.pdf").with_auth(Auth::basic("ada", "s3cret")),
1234            target(),
1235        )
1236        .unwrap();
1237
1238        let calls = session.calls();
1239
1240        assert_eq!(
1241            calls[0].params["source"]["auth"],
1242            json!({ "type": "bearer", "bearer_token": "t-1234" })
1243        );
1244        assert_eq!(
1245            calls[1].params["source"]["auth"],
1246            json!({ "type": "basic", "username": "ada", "password": "s3cret" })
1247        );
1248    }
1249
1250    #[test]
1251    fn a_blank_url_is_refused_before_anything_is_sent() {
1252        let session = testing::install(|_name, _params| Ok(stored()));
1253
1254        let error = upload_external(Source::url("  "), target()).unwrap_err();
1255
1256        assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
1257        assert!(session.calls().is_empty());
1258    }
1259
1260    #[test]
1261    fn a_credential_with_nothing_in_it_is_refused_before_anything_is_sent() {
1262        let session = testing::install(|_name, _params| Ok(stored()));
1263
1264        let blank = [
1265            Auth::bearer("  "),
1266            Auth::basic("", "s3cret"),
1267            Auth::basic("ada", " "),
1268        ];
1269
1270        for auth in blank {
1271            let error = upload_external(
1272                Source::url("https://example.com/report.pdf").with_auth(auth),
1273                target(),
1274            )
1275            .expect_err("a credential that says nothing cannot read the URL");
1276
1277            assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
1278        }
1279
1280        assert!(session.calls().is_empty());
1281    }
1282
1283    #[test]
1284    fn a_refused_upload_keeps_the_hosts_own_message() {
1285        let _session = testing::install(|_name, _params| Err(Error::failed("the store is full")));
1286
1287        let error =
1288            upload_buffer(b"foobar", "report.pdf", "application/pdf", target()).unwrap_err();
1289
1290        assert!(matches!(error, Error::Host(_)));
1291        assert!(error.message().contains("the store is full"));
1292        assert!(!error.is_retryable());
1293    }
1294
1295    #[test]
1296    fn a_handle_that_cannot_be_read_is_reported_rather_than_returned() {
1297        let _session = testing::install(|_name, _params| Ok(json!({ "filename": "report.pdf" })));
1298
1299        let error =
1300            upload_buffer(b"foobar", "report.pdf", "application/pdf", target()).unwrap_err();
1301
1302        assert!(matches!(error, Error::Json(_)));
1303        assert!(error.message().contains("could not be read"));
1304    }
1305
1306    #[test]
1307    fn an_upload_with_no_host_to_send_it_to_says_so() {
1308        let error =
1309            upload_buffer(b"foobar", "report.pdf", "application/pdf", target()).unwrap_err();
1310
1311        assert!(error.message().contains("No host is installed"));
1312    }
1313
1314    // The standard test vectors, which pin the alphabet and the padding.
1315    #[test]
1316    fn bytes_encode_to_the_standard_alphabet_with_standard_padding() {
1317        assert_eq!(encode(b""), "");
1318        assert_eq!(encode(b"f"), "Zg==");
1319        assert_eq!(encode(b"fo"), "Zm8=");
1320        assert_eq!(encode(b"foo"), "Zm9v");
1321        assert_eq!(encode(b"foob"), "Zm9vYg==");
1322        assert_eq!(encode(b"fooba"), "Zm9vYmE=");
1323        assert_eq!(encode(b"foobar"), "Zm9vYmFy");
1324    }
1325
1326    #[test]
1327    fn the_last_two_symbols_of_the_alphabet_are_plus_and_slash() {
1328        assert_eq!(encode(&[0xfb, 0xff]), "+/8=");
1329        assert_eq!(encode(&[0xff, 0xff, 0xff]), "////");
1330        assert_eq!(encode(&[0x00, 0x00, 0x00]), "AAAA");
1331    }
1332
1333    #[test]
1334    fn every_byte_value_survives_the_encoding() {
1335        let every: Vec<u8> = (0..=255).collect();
1336        let encoded = encode(&every);
1337
1338        assert_eq!(encoded.len(), 344, "256 bytes fill 86 groups of four");
1339        assert!(encoded.ends_with("/w=="));
1340        assert!(encoded
1341            .chars()
1342            .all(|symbol| symbol == PAD || ALPHABET.contains(&(symbol as u8))));
1343    }
1344
1345    #[test]
1346    fn a_credential_is_not_written_out_by_a_debug_rendering() {
1347        let source =
1348            Source::url("https://example.com/report.pdf").with_auth(Auth::bearer("t-1234"));
1349
1350        let rendered = format!("{source:?}");
1351
1352        assert!(!rendered.contains("t-1234"));
1353        assert!(rendered.contains("Bearer"));
1354
1355        let basic = format!("{:?}", Auth::basic("ada", "s3cret"));
1356
1357        assert!(!basic.contains("s3cret"));
1358        assert!(basic.contains("ada"));
1359    }
1360}