Skip to main content

simpleplatform_sdk/
storage.rs

1//! Putting a file into the platform's store.
2//!
3//! Two ways in — bytes this action already holds, and a file behind a URL — and
4//! one thing out of both: a [`DocumentHandle`], which is the value a `:document`
5//! field holds.
6//!
7//! ```
8//! # use simpleplatform_sdk::prelude::*;
9//! # use simpleplatform_sdk::testing;
10//! use simpleplatform_sdk::storage::Target;
11//!
12//! # let _session = testing::install(|_name, _params| {
13//! #     Ok(json!({
14//! #         "file_hash": "c3ab8ff13720e8ad9047dd39466b3c89",
15//! #         "filename": "report.pdf",
16//! #         "mime_type": "application/pdf",
17//! #         "size": 6,
18//! #         "storage_path": "documents/c3/ab/c3ab8ff13720e8ad9047dd39466b3c89",
19//! #     }))
20//! # });
21//! let target = Target::new("dev.simple.system", "documents", "attachment");
22//! let handle = simple::storage::upload_buffer(b"foobar", "report.pdf", "application/pdf", target)?;
23//!
24//! assert_eq!(handle.filename, "report.pdf");
25//! assert_eq!(handle.size, 6);
26//! # Ok::<(), Error>(())
27//! ```
28//!
29//! # The handle is the point
30//!
31//! An upload stores the file and answers with a handle to it. Attaching that
32//! handle to a record is a separate step, and the one that changes tenant data:
33//! write the handle into the `:document` field with [`crate::graphql::mutate`]
34//! once you have it. So the two halves stay separable — the file is in the store
35//! whether or not the record was written, and the store is the only thing
36//! [`upload_buffer`] and [`upload_external`] touch.
37//!
38//! # The same bytes are the same file
39//!
40//! The store is addressed by the SHA-256 of the contents, which is the
41//! `file_hash` on the handle. Identical bytes land under the identical hash and
42//! are kept once, so uploading a file the store already holds costs a hash and
43//! answers with the handle that was already there.
44//!
45//! # Why the bytes are encoded here
46//!
47//! The call travels as JSON, and JSON carries text. So a buffer is base64 on the
48//! wire — encoded once, at this boundary, in the same standard alphabet with the
49//! same padding every SDK uses, and decoded by the platform before the file is
50//! stored. An action passes `&[u8]` and never sees the encoding.
51
52use std::fmt;
53
54use serde::{Deserialize, Serialize};
55use serde_json::{json, Value};
56
57use crate::codes::Code;
58use crate::error::{Error, Fault};
59use crate::host;
60
61/// The host action that stores a file and answers with its handle.
62const UPLOAD_EXTERNAL: &str = "action:storage/upload-external";
63
64/// A stored file, as a `:document` field holds it.
65///
66/// This is a pointer to the contents rather than the contents: it is what an
67/// upload answers with, what a record stores, and what identifies the file
68/// afterwards.
69#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
70pub struct DocumentHandle {
71    /// The SHA-256 of the contents, which is the file's identity in the store.
72    pub file_hash: String,
73    /// The name the file is stored under.
74    pub filename: String,
75    /// The media type of the contents, such as `application/pdf`.
76    pub mime_type: String,
77    /// The size of the contents in bytes.
78    pub size: u64,
79    /// Where the store keeps the file.
80    pub storage_path: String,
81}
82
83/// The record field a stored file is destined for.
84#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
85pub struct Target {
86    /// The app that owns the table.
87    pub app_id: String,
88    /// The table the record lives in.
89    pub table_name: String,
90    /// The `:document` field the handle is stored in.
91    pub field_name: String,
92}
93
94impl Target {
95    /// A target named by its app, its table and its field.
96    ///
97    /// ```
98    /// use simpleplatform_sdk::storage::Target;
99    ///
100    /// let target = Target::new("dev.simple.system", "documents", "attachment");
101    ///
102    /// assert_eq!(target.field_name, "attachment");
103    /// ```
104    pub fn new(
105        app_id: impl Into<String>,
106        table_name: impl Into<String>,
107        field_name: impl Into<String>,
108    ) -> Target {
109        Target {
110            app_id: app_id.into(),
111            table_name: table_name.into(),
112            field_name: field_name.into(),
113        }
114    }
115}
116
117/// The credential a URL is read with.
118///
119/// It is `#[non_exhaustive]`: a `match` on it needs a `_` arm, so a scheme added
120/// later does not break an action that was already written.
121#[non_exhaustive]
122#[derive(Clone, Serialize)]
123#[serde(tag = "type", rename_all = "lowercase")]
124pub enum Auth {
125    /// A username and a password.
126    Basic {
127        /// The username.
128        username: String,
129        /// The password.
130        password: String,
131    },
132    /// A bearer token.
133    Bearer {
134        /// The token.
135        bearer_token: String,
136    },
137}
138
139impl Auth {
140    /// Read the URL with a bearer token.
141    ///
142    /// ```
143    /// use simpleplatform_sdk::storage::Auth;
144    ///
145    /// let auth = Auth::bearer("t-1234");
146    ///
147    /// assert!(matches!(auth, Auth::Bearer { .. }));
148    /// ```
149    pub fn bearer(bearer_token: impl Into<String>) -> Auth {
150        Auth::Bearer {
151            bearer_token: bearer_token.into(),
152        }
153    }
154
155    /// Read the URL with a username and a password.
156    pub fn basic(username: impl Into<String>, password: impl Into<String>) -> Auth {
157        Auth::Basic {
158            username: username.into(),
159            password: password.into(),
160        }
161    }
162}
163
164impl fmt::Debug for Auth {
165    /// The scheme, and never the secret.
166    ///
167    /// A credential is written by hand and read by a host, and the one place it
168    /// has no business appearing is a log line. So this renders the scheme, and
169    /// the username that names the account, and stops there.
170    ///
171    /// ```
172    /// use simpleplatform_sdk::storage::Auth;
173    ///
174    /// assert_eq!(format!("{:?}", Auth::bearer("t-1234")), "Bearer { .. }");
175    /// ```
176    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
177        match self {
178            Auth::Basic { username, .. } => formatter
179                .debug_struct("Basic")
180                .field("username", username)
181                .finish_non_exhaustive(),
182            Auth::Bearer { .. } => formatter.debug_struct("Bearer").finish_non_exhaustive(),
183        }
184    }
185}
186
187/// The file an external upload reads.
188#[derive(Clone, Debug, Serialize)]
189pub struct Source {
190    /// The URL the file is read from.
191    pub url: String,
192    /// The credential the URL is read with, for a URL that needs one.
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub auth: Option<Auth>,
195}
196
197impl Source {
198    /// A file at this URL, read without a credential.
199    pub fn url(url: impl Into<String>) -> Source {
200        Source {
201            url: url.into(),
202            auth: None,
203        }
204    }
205
206    /// The same file, read with this credential.
207    ///
208    /// ```
209    /// use simpleplatform_sdk::storage::{Auth, Source};
210    ///
211    /// let source = Source::url("https://example.com/report.pdf").with_auth(Auth::bearer("t-1234"));
212    ///
213    /// assert!(source.auth.is_some());
214    /// ```
215    pub fn with_auth(mut self, auth: Auth) -> Source {
216        self.auth = Some(auth);
217        self
218    }
219}
220
221/// Store bytes this action already holds.
222///
223/// The buffer, the name and the media type are all required, and each is checked
224/// before anything is sent: an empty buffer, a blank name, a blank media type or
225/// an incomplete [`Target`] is refused where it costs nothing.
226///
227/// ```
228/// # use simpleplatform_sdk::prelude::*;
229/// # use simpleplatform_sdk::testing;
230/// use simpleplatform_sdk::storage::Target;
231///
232/// # let _session = testing::install(|_name, params| {
233/// #     assert_eq!(params["source"]["bytes"], json!("Zm9vYmFy"));
234/// #     Ok(json!({
235/// #         "file_hash": "c3ab8ff137",
236/// #         "filename": "notes.txt",
237/// #         "mime_type": "text/plain",
238/// #         "size": 6,
239/// #         "storage_path": "documents/c3/ab/c3ab8ff137",
240/// #     }))
241/// # });
242/// let handle = simple::storage::upload_buffer(
243///     b"foobar",
244///     "notes.txt",
245///     "text/plain",
246///     Target::new("dev.simple.system", "documents", "attachment"),
247/// )?;
248///
249/// assert_eq!(handle.file_hash, "c3ab8ff137");
250/// # Ok::<(), Error>(())
251/// ```
252pub fn upload_buffer(
253    bytes: &[u8],
254    filename: &str,
255    mime_type: &str,
256    target: Target,
257) -> Result<DocumentHandle, Error> {
258    if bytes.is_empty() {
259        return Err(
260            Error::invalid("An upload needs at least one byte, and this buffer is empty.")
261                .hint("Pass the file's contents."),
262        );
263    }
264
265    if filename.trim().is_empty() {
266        return Err(Error::invalid("An upload needs a filename.")
267            .hint("Pass the name the file is stored under, such as report.pdf."));
268    }
269
270    if mime_type.trim().is_empty() {
271        return Err(Error::invalid("An upload needs a media type.")
272            .hint("Pass the type of the contents, such as application/pdf."));
273    }
274
275    check_target(&target)?;
276
277    let source = json!({
278        "bytes": encode(bytes),
279        "filename": filename,
280        "mime_type": mime_type,
281    });
282
283    send(source, target)
284}
285
286/// Store a file the platform reads from a URL.
287///
288/// The URL and any credential on it are checked before anything is sent, as is
289/// the [`Target`].
290///
291/// ```
292/// # use simpleplatform_sdk::prelude::*;
293/// # use simpleplatform_sdk::testing;
294/// use simpleplatform_sdk::storage::{Auth, Source, Target};
295///
296/// # let _session = testing::install(|_name, params| {
297/// #     assert_eq!(params["source"]["auth"]["type"], json!("bearer"));
298/// #     Ok(json!({
299/// #         "file_hash": "9f86d081884c",
300/// #         "filename": "statement.pdf",
301/// #         "mime_type": "application/pdf",
302/// #         "size": 81_920,
303/// #         "storage_path": "documents/9f/86/9f86d081884c",
304/// #     }))
305/// # });
306/// let handle = simple::storage::upload_external(
307///     Source::url("https://example.com/statement.pdf").with_auth(Auth::bearer("t-1234")),
308///     Target::new("dev.simple.system", "documents", "attachment"),
309/// )?;
310///
311/// assert_eq!(handle.size, 81_920);
312/// # Ok::<(), Error>(())
313/// ```
314pub fn upload_external(source: Source, target: Target) -> Result<DocumentHandle, Error> {
315    if source.url.trim().is_empty() {
316        return Err(
317            Error::invalid("An external upload needs a URL to read from.")
318                .hint("Pass the address of the file, or use upload_buffer for bytes you hold."),
319        );
320    }
321
322    if let Some(auth) = &source.auth {
323        check_auth(auth)?;
324    }
325
326    check_target(&target)?;
327
328    send(serde_json::to_value(&source)?, target)
329}
330
331/// One upload, one round trip, one handle.
332fn send(source: Value, target: Target) -> Result<DocumentHandle, Error> {
333    let params = json!({ "source": source, "target": serde_json::to_value(&target)? });
334
335    let stored = host::transport()?
336        .call(UPLOAD_EXTERNAL.to_string(), params)
337        .map_err(|cause| {
338            Error::Host(Fault::new(Code::unspecified(), cause.message())).hint(
339                "Nothing was attached to the record. The store is addressed by content, \
340                 so the same bytes uploaded again answer with the same handle.",
341            )
342        })?;
343
344    serde_json::from_value(stored).map_err(|cause| {
345        Error::Json(Fault::new(
346            Code::unspecified(),
347            format!("The upload finished and its handle could not be read: {cause}"),
348        ))
349        .hint(
350            "Report this. The file is stored under the hash of its contents, \
351             so uploading the same bytes again answers with the handle.",
352        )
353    })
354}
355
356/// Whether a target names a field to attach to.
357fn check_target(target: &Target) -> Result<(), Error> {
358    let named = [
359        ("app_id", &target.app_id),
360        ("table_name", &target.table_name),
361        ("field_name", &target.field_name),
362    ];
363
364    for (member, value) in named {
365        if value.trim().is_empty() {
366            return Err(Error::invalid(format!("A storage target needs {member}."))
367                .hint("Name the app, the table, and the :document field the handle goes in."));
368        }
369    }
370
371    Ok(())
372}
373
374/// Whether a credential carries what its scheme needs.
375///
376/// The scheme itself is settled by [`Auth`] — there are two, and each holds its
377/// own members — so what is left to check is that the members say something.
378fn check_auth(auth: &Auth) -> Result<(), Error> {
379    match auth {
380        Auth::Basic { username, password } => {
381            if username.trim().is_empty() {
382                return Err(Error::invalid("Basic authentication needs a username.")
383                    .hint("Pass the username, or read the URL without a credential."));
384            }
385
386            if password.trim().is_empty() {
387                return Err(Error::invalid("Basic authentication needs a password.")
388                    .hint("Pass the password, or read the URL without a credential."));
389            }
390        }
391        Auth::Bearer { bearer_token } => {
392            if bearer_token.trim().is_empty() {
393                return Err(Error::invalid("Bearer authentication needs a token.")
394                    .hint("Pass the token, or read the URL without a credential."));
395            }
396        }
397    }
398
399    Ok(())
400}
401
402/// The symbols base64 spends, in their standard order.
403const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
404
405/// What fills a group that ran out of bytes.
406const PAD: char = '=';
407
408/// The bytes as base64: three bytes to four symbols, padded to a multiple of
409/// four, in the standard alphabet.
410fn encode(bytes: &[u8]) -> String {
411    let mut encoded = String::with_capacity(bytes.len().div_ceil(3) * 4);
412
413    for group in bytes.chunks(3) {
414        let mut packed = 0_u32;
415
416        // Left to right: the first byte takes the top eight of twenty-four bits,
417        // and a group of one or two leaves the rest zero.
418        for (index, byte) in group.iter().enumerate() {
419            packed |= u32::from(*byte) << (16 - 8 * index);
420        }
421
422        encoded.push(symbol(packed >> 18));
423        encoded.push(symbol(packed >> 12));
424        encoded.push(if group.len() > 1 {
425            symbol(packed >> 6)
426        } else {
427            PAD
428        });
429        encoded.push(if group.len() > 2 { symbol(packed) } else { PAD });
430    }
431
432    encoded
433}
434
435/// The symbol for the low six bits of `packed`.
436fn symbol(packed: u32) -> char {
437    // Six bits count to sixty-three and the alphabet holds sixty-four symbols,
438    // so there is always one to answer with.
439    char::from(ALPHABET[(packed & 0b11_1111) as usize])
440}
441
442#[cfg(test)]
443mod tests {
444    use serde_json::json;
445
446    use super::*;
447    use crate::testing;
448
449    /// What the host answers with for an upload that landed.
450    fn stored() -> Value {
451        json!({
452            "file_hash": "c3ab8ff13720e8ad9047dd39466b3c89",
453            "filename": "report.pdf",
454            "mime_type": "application/pdf",
455            "size": 6,
456            "storage_path": "documents/c3/ab/c3ab8ff13720e8ad9047dd39466b3c89",
457        })
458    }
459
460    fn target() -> Target {
461        Target::new("dev.simple.system", "documents", "attachment")
462    }
463
464    #[test]
465    fn a_buffer_travels_base64_encoded_beside_its_name_and_type() {
466        let session = testing::install(|_name, _params| Ok(stored()));
467
468        upload_buffer(b"foobar", "report.pdf", "application/pdf", target()).unwrap();
469
470        let calls = session.calls();
471
472        assert_eq!(calls.len(), 1);
473        assert_eq!(calls[0].name, "action:storage/upload-external");
474        assert_eq!(
475            calls[0].params["source"],
476            json!({
477                "bytes": "Zm9vYmFy",
478                "filename": "report.pdf",
479                "mime_type": "application/pdf",
480            })
481        );
482        assert_eq!(
483            calls[0].params["target"],
484            json!({
485                "app_id": "dev.simple.system",
486                "table_name": "documents",
487                "field_name": "attachment",
488            })
489        );
490    }
491
492    #[test]
493    fn a_stored_file_answers_with_every_member_of_its_handle() {
494        let _session = testing::install(|_name, _params| Ok(stored()));
495
496        let handle = upload_buffer(b"foobar", "report.pdf", "application/pdf", target()).unwrap();
497
498        assert_eq!(
499            handle,
500            DocumentHandle {
501                file_hash: "c3ab8ff13720e8ad9047dd39466b3c89".to_string(),
502                filename: "report.pdf".to_string(),
503                mime_type: "application/pdf".to_string(),
504                size: 6,
505                storage_path: "documents/c3/ab/c3ab8ff13720e8ad9047dd39466b3c89".to_string(),
506            }
507        );
508    }
509
510    #[test]
511    fn an_empty_buffer_is_refused_before_anything_is_sent() {
512        let session = testing::install(|_name, _params| Ok(stored()));
513
514        let error = upload_buffer(&[], "report.pdf", "application/pdf", target()).unwrap_err();
515
516        assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
517        assert!(session.calls().is_empty());
518    }
519
520    #[test]
521    fn a_blank_name_or_type_is_refused_before_anything_is_sent() {
522        let session = testing::install(|_name, _params| Ok(stored()));
523
524        for (filename, mime_type) in [("   ", "application/pdf"), ("report.pdf", "")] {
525            let error = upload_buffer(b"foobar", filename, mime_type, target()).unwrap_err();
526
527            assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
528        }
529
530        assert!(session.calls().is_empty());
531    }
532
533    #[test]
534    fn a_target_missing_a_member_names_the_one_it_is_missing() {
535        let session = testing::install(|_name, _params| Ok(stored()));
536
537        let incomplete = [
538            (Target::new("", "documents", "attachment"), "app_id"),
539            (
540                Target::new("dev.simple.system", " ", "attachment"),
541                "table_name",
542            ),
543            (
544                Target::new("dev.simple.system", "documents", ""),
545                "field_name",
546            ),
547        ];
548
549        for (target, member) in incomplete {
550            let error = upload_buffer(b"foobar", "report.pdf", "application/pdf", target)
551                .expect_err("an incomplete target names nothing to attach to");
552
553            assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
554            assert!(error.message().contains(member), "{}", error.message());
555        }
556
557        assert!(session.calls().is_empty());
558    }
559
560    #[test]
561    fn a_url_upload_carries_the_url_alone_when_it_needs_no_credential() {
562        let session = testing::install(|_name, _params| Ok(stored()));
563
564        upload_external(Source::url("https://example.com/report.pdf"), target()).unwrap();
565
566        assert_eq!(
567            session.calls()[0].params["source"],
568            json!({ "url": "https://example.com/report.pdf" }),
569            "a source with no credential carries no auth member"
570        );
571    }
572
573    #[test]
574    fn a_credential_travels_under_its_own_scheme() {
575        let session = testing::install(|_name, _params| Ok(stored()));
576
577        upload_external(
578            Source::url("https://example.com/report.pdf").with_auth(Auth::bearer("t-1234")),
579            target(),
580        )
581        .unwrap();
582
583        upload_external(
584            Source::url("https://example.com/report.pdf").with_auth(Auth::basic("ada", "s3cret")),
585            target(),
586        )
587        .unwrap();
588
589        let calls = session.calls();
590
591        assert_eq!(
592            calls[0].params["source"]["auth"],
593            json!({ "type": "bearer", "bearer_token": "t-1234" })
594        );
595        assert_eq!(
596            calls[1].params["source"]["auth"],
597            json!({ "type": "basic", "username": "ada", "password": "s3cret" })
598        );
599    }
600
601    #[test]
602    fn a_blank_url_is_refused_before_anything_is_sent() {
603        let session = testing::install(|_name, _params| Ok(stored()));
604
605        let error = upload_external(Source::url("  "), target()).unwrap_err();
606
607        assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
608        assert!(session.calls().is_empty());
609    }
610
611    #[test]
612    fn a_credential_with_nothing_in_it_is_refused_before_anything_is_sent() {
613        let session = testing::install(|_name, _params| Ok(stored()));
614
615        let blank = [
616            Auth::bearer("  "),
617            Auth::basic("", "s3cret"),
618            Auth::basic("ada", " "),
619        ];
620
621        for auth in blank {
622            let error = upload_external(
623                Source::url("https://example.com/report.pdf").with_auth(auth),
624                target(),
625            )
626            .expect_err("a credential that says nothing cannot read the URL");
627
628            assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
629        }
630
631        assert!(session.calls().is_empty());
632    }
633
634    #[test]
635    fn a_refused_upload_keeps_the_hosts_own_message() {
636        let _session = testing::install(|_name, _params| Err(Error::failed("the store is full")));
637
638        let error =
639            upload_buffer(b"foobar", "report.pdf", "application/pdf", target()).unwrap_err();
640
641        assert!(matches!(error, Error::Host(_)));
642        assert!(error.message().contains("the store is full"));
643        assert!(!error.is_retryable());
644    }
645
646    #[test]
647    fn a_handle_that_cannot_be_read_is_reported_rather_than_returned() {
648        let _session = testing::install(|_name, _params| Ok(json!({ "filename": "report.pdf" })));
649
650        let error =
651            upload_buffer(b"foobar", "report.pdf", "application/pdf", target()).unwrap_err();
652
653        assert!(matches!(error, Error::Json(_)));
654        assert!(error.message().contains("could not be read"));
655    }
656
657    #[test]
658    fn an_upload_with_no_host_to_send_it_to_says_so() {
659        let error =
660            upload_buffer(b"foobar", "report.pdf", "application/pdf", target()).unwrap_err();
661
662        assert!(error.message().contains("No host is installed"));
663    }
664
665    // The standard test vectors, which pin the alphabet and the padding.
666    #[test]
667    fn bytes_encode_to_the_standard_alphabet_with_standard_padding() {
668        assert_eq!(encode(b""), "");
669        assert_eq!(encode(b"f"), "Zg==");
670        assert_eq!(encode(b"fo"), "Zm8=");
671        assert_eq!(encode(b"foo"), "Zm9v");
672        assert_eq!(encode(b"foob"), "Zm9vYg==");
673        assert_eq!(encode(b"fooba"), "Zm9vYmE=");
674        assert_eq!(encode(b"foobar"), "Zm9vYmFy");
675    }
676
677    #[test]
678    fn the_last_two_symbols_of_the_alphabet_are_plus_and_slash() {
679        assert_eq!(encode(&[0xfb, 0xff]), "+/8=");
680        assert_eq!(encode(&[0xff, 0xff, 0xff]), "////");
681        assert_eq!(encode(&[0x00, 0x00, 0x00]), "AAAA");
682    }
683
684    #[test]
685    fn every_byte_value_survives_the_encoding() {
686        let every: Vec<u8> = (0..=255).collect();
687        let encoded = encode(&every);
688
689        assert_eq!(encoded.len(), 344, "256 bytes fill 86 groups of four");
690        assert!(encoded.ends_with("/w=="));
691        assert!(encoded
692            .chars()
693            .all(|symbol| symbol == PAD || ALPHABET.contains(&(symbol as u8))));
694    }
695
696    #[test]
697    fn a_credential_is_not_written_out_by_a_debug_rendering() {
698        let source =
699            Source::url("https://example.com/report.pdf").with_auth(Auth::bearer("t-1234"));
700
701        let rendered = format!("{source:?}");
702
703        assert!(!rendered.contains("t-1234"));
704        assert!(rendered.contains("Bearer"));
705
706        let basic = format!("{:?}", Auth::basic("ada", "s3cret"));
707
708        assert!(!basic.contains("s3cret"));
709        assert!(basic.contains("ada"));
710    }
711}