Skip to main content

simpleplatform_sdk/
ai.rs

1//! Asking the AI engine for an answer.
2//!
3//! Four operations read something and answer with it — [`extract`] pulls
4//! structured data out against a schema you give, [`summarize`] writes prose,
5//! [`transcribe`] turns audio or video into words, and [`transcribe_pages`]
6//! reads the pages of a PDF that have no text of their own from their images.
7//! Three more hold the tenant's face collection: [`enroll_face`],
8//! [`search_face`] and [`delete_face`].
9//!
10//! ```
11//! # use simpleplatform_sdk::prelude::*;
12//! # use simpleplatform_sdk::ai::{Execution, Options};
13//! # use simpleplatform_sdk::testing;
14//! #[derive(Deserialize)]
15//! struct Invoice {
16//!     number: String,
17//!     total: f64,
18//! }
19//!
20//! # let _session = testing::install(|_name, _params| {
21//! #     Ok(json!({
22//! #         "data": { "number": "INV-8", "total": 240.0 },
23//! #         "metadata": { "input_tokens": 812, "output_tokens": 24 }
24//! #     }))
25//! # });
26//! let schema = json!({
27//!     "type": "object",
28//!     "properties": {
29//!         "number": { "type": "string" },
30//!         "total": { "type": "number" }
31//!     },
32//!     "required": ["number", "total"]
33//! });
34//!
35//! let read: Execution<Invoice> = simple::ai::extract(
36//!     json!({ "file_hash": "9f2c…", "mime_type": "application/pdf" }),
37//!     "Read the invoice number and the total.",
38//!     schema,
39//!     Options::default(),
40//! )?;
41//!
42//! assert_eq!(read.data.number, "INV-8");
43//! assert_eq!(read.metadata.input_tokens, 812);
44//! # Ok::<(), Error>(())
45//! ```
46//!
47//! # The shape, and why it is this one
48//!
49//! **What an operation must have is an argument; what it may have is a field.**
50//! An input, a prompt and a schema are the operation — leave one out and there
51//! is nothing to run — so they are arguments, and the type system asks for them.
52//! Everything else has a working default, so it lives in [`Options`], which
53//! derives one:
54//!
55//! ```
56//! # use simpleplatform_sdk::ai::{Model, Options};
57//! # use std::time::Duration;
58//! let options = Options {
59//!     model: Some(Model::Large),
60//!     timeout: Some(Duration::from_secs(90)),
61//!     ..Default::default()
62//! };
63//! ```
64//!
65//! Nobody passes six `None`s, and a member added to [`Options`] later does not
66//! disturb a call that was written with `..Default::default()`.
67//!
68//! **The answer type is yours.** [`extract`] is generic over what it decodes
69//! into, exactly as [`crate::graphql::query`] is, so an extraction lands in your
70//! own struct rather than in a `Value` you then have to pick apart. Where the
71//! answer's shape is settled it is named for you instead: [`summarize`] answers
72//! with prose, and [`transcribe`] answers with a [`Transcript`], because this
73//! module wrote the schema that produced it.
74//!
75//! **Every answer carries what it cost.** An operation answers with an
76//! [`Execution`] — the data and the [`Metadata`] beside it — so token counts are
77//! there to record without a second call.
78//!
79//! # Asking for part of a file, or for its text
80//!
81//! A file reference may say how the file should travel and which of its pages
82//! should. [`DocumentInput`] is that reference with those two beside it, and
83//! [`Delivery`] and [`Pages`] are what they say. A plain handle asks for nothing
84//! and sends the whole file the way its type is sent by default.
85//!
86//! What was asked for is not always what happened, so every answer says how
87//! each file travelled, in [`Metadata::delivery`]: as the document, as its text
88//! or as an image, which pages were read from their images, and, when text was
89//! asked for and the document went instead, which pages could not be read.
90//!
91//! # Files that have not been uploaded yet
92//!
93//! A document handle that is still pending is uploaded to ephemeral storage
94//! before the operation runs, and the operation is given the handle that upload
95//! answered with. A handle that is already stored is passed straight through,
96//! and so is a list of handles, item by item. Nothing in a call site changes:
97//! pass the handle you were given.
98
99use std::fmt;
100use std::time::Duration;
101
102use serde::de::DeserializeOwned;
103use serde::{Deserialize, Serialize};
104use serde_json::{json, Map, Value};
105
106use crate::codes::Code;
107use crate::error::{Error, Fault};
108use crate::host;
109use crate::storage::DocumentHandle;
110
111/// The primitive that runs an AI operation.
112const ORCHESTRATOR: &str = "logic:dev.simple.system/ai-orchestrator";
113
114/// The host action that puts a pending file into ephemeral storage.
115const UPLOAD_EPHEMERAL: &str = "action:documents/upload-ephemeral";
116
117/// The host action that adds a face to the tenant's collection.
118const FACE_ENROLL: &str = "action:ai/face/enroll";
119
120/// The host action that searches the tenant's collection.
121const FACE_SEARCH: &str = "action:ai/face/search";
122
123/// The host action that removes faces from the tenant's collection.
124const FACE_DELETE: &str = "action:ai/face/delete";
125
126/// The size of model an operation runs on.
127///
128/// It is `#[non_exhaustive]`: a `match` on it needs a `_` arm, so a size added
129/// later does not break an action that was already written.
130#[non_exhaustive]
131#[derive(Clone, Copy, Debug, PartialEq, Eq)]
132pub enum Model {
133    /// The quickest and cheapest.
134    Lite,
135    /// The middle of the range.
136    Medium,
137    /// The one to reach for when the answer has to be right.
138    Large,
139    /// The largest available.
140    Xl,
141}
142
143impl Model {
144    /// The size exactly as it travels on the wire.
145    pub fn as_str(&self) -> &str {
146        match self {
147            Model::Lite => "lite",
148            Model::Medium => "medium",
149            Model::Large => "large",
150            Model::Xl => "xl",
151        }
152    }
153}
154
155impl fmt::Display for Model {
156    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
157        formatter.write_str(self.as_str())
158    }
159}
160
161/// How a document reaches the model.
162///
163/// Both answers are sayable, so a call can write the default down rather than
164/// leaving it to the absence of a value.
165///
166/// It is `#[non_exhaustive]`: a `match` on it needs a `_` arm, so a way added
167/// later does not break an action that was already written.
168#[non_exhaustive]
169#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
170#[serde(rename_all = "lowercase")]
171pub enum Delivery {
172    /// The document itself, which is how a PDF travels unless you say
173    /// otherwise.
174    Document,
175    /// The text the platform reads out of the document.
176    Text,
177}
178
179/// The pages of a PDF to use, counted from 1 with both ends included.
180///
181/// The two ends are one value here because the engine refuses one without the
182/// other. A half-written range cannot be expressed, so it cannot be sent.
183///
184/// What this type does not decide is still the engine's: a `first` below 1, a
185/// `last` before `first`, or a range that runs past the end of the document is
186/// refused there, before the file is read or anything is spent.
187#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
188pub struct Pages {
189    /// The first page to use, counted from 1.
190    pub first_page: u32,
191    /// The last page to use, included.
192    pub last_page: u32,
193}
194
195impl Pages {
196    /// Pages `first` to `last`, counted from 1 with both ends included.
197    pub fn new(first: u32, last: u32) -> Pages {
198        Pages {
199            first_page: first,
200            last_page: last,
201        }
202    }
203}
204
205/// A stored file handed to an AI operation, together with what should be done
206/// with it.
207///
208/// A PDF travels to the model as a PDF, because the model reads one: its
209/// tables, its layout and its figures survive the trip, and none of them
210/// survive being turned into text. Asking for text is therefore opt-in.
211///
212/// ```
213/// # use simpleplatform_sdk::prelude::*;
214/// # use simpleplatform_sdk::ai::{self, Delivery, DocumentInput, Execution, Pages};
215/// # use simpleplatform_sdk::testing;
216/// # let _session = testing::install(|_name, _params| {
217/// #     Ok(json!({ "data": "The first three pages sign the contract off." }))
218/// # });
219/// let contract = DocumentInput {
220///     file: DocumentHandle {
221///         file_hash: "9f2c…".into(),
222///         filename: "contract.pdf".into(),
223///         mime_type: "application/pdf".into(),
224///         size: 1_048_576,
225///         storage_path: "acme/documents/9f/2c/9f2c….pdf".into(),
226///     },
227///     pages: Some(Pages::new(1, 3)),
228///     deliver_as: Some(Delivery::Text),
229/// };
230///
231/// let read: Execution<String> =
232///     ai::summarize(json!(contract), "Say what it commits us to.", Options::default())?;
233/// # Ok::<(), Error>(())
234/// ```
235///
236/// The pages and the delivery are two questions, asked in that order. The pages
237/// say WHICH DOCUMENT the call is about: they are taken out of the PDF first,
238/// and everything after that is about them and nothing else. The delivery then
239/// says how that document should reach the model.
240///
241/// Refused by the engine, before the file is read or anything is spent:
242///
243/// * [`pages`](DocumentInput::pages) on anything that is not a PDF;
244/// * a range that runs past the end of the document, or one whose last page
245///   precedes its first;
246/// * [`Delivery::Text`] on an image, which is not read as text;
247/// * [`Delivery::Document`] on a Word, Excel, PowerPoint, CSV, RTF or text
248///   file, which only ever travels as text.
249///
250/// A page with no usable text of its own — a scan, or text that reads as noise
251/// — is read from its image instead, and that text stands in its place, marked
252/// `[Transcribed from the page image.]` under its page label. Only if that
253/// reading fails are the pages asked for sent as a PDF, so an answer is never
254/// built on text with a hole in it. The answer's [`Metadata::delivery`] says
255/// which happened.
256///
257/// Pages asked for as text arrive numbered from 1, because by then they are a
258/// document of their own. A line above them says which pages of which document
259/// they were, rather than the numbers being rewritten inside the text: a page
260/// number printed in a header or a cross-reference cannot be told apart from a
261/// page label, so editing them would corrupt the document's own words.
262#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
263pub struct DocumentInput {
264    /// The stored file, as the upload or the record that holds it answered
265    /// with.
266    #[serde(flatten)]
267    pub file: DocumentHandle,
268
269    /// Which pages of the PDF to send. Unset sends the whole document.
270    #[serde(default, flatten)]
271    pub pages: Option<Pages>,
272
273    /// How the document reaches the model. Unset sends it the way its type is
274    /// sent by default.
275    ///
276    #[serde(default, skip_serializing_if = "Option::is_none")]
277    pub deliver_as: Option<Delivery>,
278}
279
280/// What any operation may be told, beyond the inputs that define it.
281///
282/// Every member has a default, so a call that wants one thing changed says that
283/// thing and nothing else:
284///
285/// ```
286/// # use simpleplatform_sdk::ai::{Model, Options};
287/// let options = Options {
288///     model: Some(Model::Large),
289///     reasoning: false,
290///     ..Default::default()
291/// };
292/// ```
293#[derive(Clone, Debug)]
294pub struct Options {
295    /// Which size of model to run on. Unset leaves the choice to the platform.
296    pub model: Option<Model>,
297
298    /// The role, or the standing instruction, the model holds for the whole
299    /// task. Unset sends none.
300    pub system_prompt: Option<String>,
301
302    /// How far the model may range, from `0.0` for the most settled answer to
303    /// `1.0` for the most inventive. Unset leaves the model's own default.
304    pub temperature: Option<f64>,
305
306    /// Whether the model works the answer out step by step, and reports how.
307    /// On unless you turn it off.
308    pub reasoning: bool,
309
310    /// The most tokens the reasoning may spend. Unset leaves the platform's
311    /// default, and it is read only while `reasoning` is on.
312    pub reasoning_budget: Option<u32>,
313
314    /// Whether to do the work again rather than answer from a cached result for
315    /// the same input, prompt, schema, model and options.
316    pub regenerate: bool,
317
318    /// How long to wait for the answer. Unset leaves the platform's default.
319    pub timeout: Option<Duration>,
320}
321
322impl Default for Options {
323    /// Reasoning on, and every other choice left to the platform.
324    fn default() -> Options {
325        Options {
326            model: None,
327            system_prompt: None,
328            temperature: None,
329            reasoning: true,
330            reasoning_budget: None,
331            regenerate: false,
332            timeout: None,
333        }
334    }
335}
336
337/// Whether to tell the speakers apart, and what to call them.
338#[derive(Clone, Debug, Default, PartialEq, Eq)]
339pub enum Participants {
340    /// Do not tell the speakers apart.
341    #[default]
342    Ignore,
343
344    /// Tell the speakers apart, and label them `Participant 1`,
345    /// `Participant 2`, and so on.
346    Detect,
347
348    /// Tell the speakers apart under the names given. Naming nobody asks for
349    /// the same thing as [`Participants::Detect`].
350    Named(Vec<String>),
351}
352
353impl Participants {
354    /// Whether the speakers are to be told apart at all.
355    fn wanted(&self) -> bool {
356        !matches!(self, Participants::Ignore)
357    }
358
359    /// The names to identify, as one list, or nothing when none were given.
360    fn names(&self) -> Option<String> {
361        match self {
362            Participants::Named(names) if !names.is_empty() => Some(names.join(", ")),
363            _ => None,
364        }
365    }
366}
367
368/// What a transcription is to produce.
369///
370/// At least one of `include_transcript` and `summarize` has to be set, since
371/// between them they are the whole of what a transcription answers with.
372#[derive(Clone, Debug, Default)]
373pub struct TranscribeOptions {
374    /// Ask for the words that were said.
375    pub include_transcript: bool,
376
377    /// Ask for `[MM:SS]` stamps through the transcript. Read only while
378    /// `include_transcript` is set.
379    pub include_timestamps: bool,
380
381    /// Ask for a summary of what was said.
382    pub summarize: bool,
383
384    /// Whether to tell the speakers apart, and what to call them.
385    pub participants: Participants,
386}
387
388/// What a transcription answered with.
389///
390/// A member that was not asked for is empty, and every member survives an
391/// answer that leaves it out.
392#[derive(Clone, Debug, Default, Deserialize)]
393#[serde(default)]
394pub struct Transcript {
395    /// The language that was heard, as an ISO 639-1 code such as `en` or `es`.
396    pub language: String,
397
398    /// The words that were said, when they were asked for.
399    pub transcript: Option<String>,
400
401    /// The summary, when it was asked for.
402    pub summary: Option<String>,
403
404    /// The speakers that were told apart, when they were asked for.
405    pub participants: Vec<String>,
406}
407
408/// What [`transcribe_pages`] may be told, beyond the PDF and the pages.
409///
410/// There is no prompt, model or reasoning switch, which is why this is not
411/// [`Options`]: the instructions are the platform's, and every page is read at
412/// the platform's own accuracy-first effort.
413#[derive(Clone, Debug, Default)]
414pub struct TranscribePagesOptions {
415    /// Whether to run the operation again rather than answer from its kept
416    /// result. A page already read is still served from its kept read, which
417    /// is tied to this version of the file and to the platform's instructions.
418    pub regenerate: bool,
419
420    /// How long to wait for the answer. Unset leaves the platform's default.
421    pub timeout: Option<Duration>,
422}
423
424/// One page's transcription, or why it has none.
425///
426/// Pages are numbered in the original document, whatever range was asked for.
427/// A page either has its text or has a reason, never both, so a caller cannot
428/// mistake a partial read for the page.
429#[derive(Clone, Debug, PartialEq, Eq)]
430pub enum PageTranscription {
431    /// The page was read from its image.
432    Read {
433        /// The page's number in the original document.
434        page: u32,
435
436        /// The page's text: verbatim markdown, in reading order, tables as
437        /// markdown tables, `[illegible]` where a word could not be read. An
438        /// empty string is a read that found the page blank.
439        text: String,
440    },
441
442    /// The page could not be read.
443    Unread {
444        /// The page's number in the original document.
445        page: u32,
446
447        /// Why, in the platform's words. No partial text is given.
448        error: String,
449    },
450}
451
452impl PageTranscription {
453    /// The page's number in the original document, read or not.
454    pub fn page(&self) -> u32 {
455        match self {
456            PageTranscription::Read { page, .. } | PageTranscription::Unread { page, .. } => *page,
457        }
458    }
459
460    /// The page's text, when it was read.
461    pub fn text(&self) -> Option<&str> {
462        match self {
463            PageTranscription::Read { text, .. } => Some(text),
464            PageTranscription::Unread { .. } => None,
465        }
466    }
467}
468
469/// What an operation answered, and what it cost.
470#[derive(Clone, Debug)]
471pub struct Execution<T> {
472    /// The answer itself, decoded into the type the call asked for.
473    pub data: T,
474
475    /// What the run spent, and how it reasoned.
476    pub metadata: Metadata,
477}
478
479/// What a run spent, how it reasoned, and how its files reached the model.
480#[derive(Clone, Debug, Default, PartialEq, Eq)]
481pub struct Metadata {
482    /// Tokens in what was sent.
483    pub input_tokens: u64,
484
485    /// Tokens in what came back.
486    pub output_tokens: u64,
487
488    /// Tokens spent reasoning, when the run reported them.
489    pub reasoning_tokens: Option<u64>,
490
491    /// The reasoning itself, when the run reported it.
492    pub reasoning: Option<String>,
493
494    /// How each file the run carried reached the model, in the order the files
495    /// were given. Empty when the run carried no file, and on an answer kept
496    /// from before the platform reported it.
497    pub delivery: Vec<FileDelivery>,
498}
499
500/// How one file an operation carried reached the model.
501///
502/// A file asked for as text is not always sent as text. A page with no text of
503/// its own is read from its image, and when that reading fails the document is
504/// sent instead, so the answer is never built on text with a hole in it. This
505/// is how a caller finds out which happened, without reading a platform log.
506///
507/// Page numbers are the original document's, whatever range was cut from it,
508/// so a page named here is the page a person would turn to.
509///
510/// ```
511/// # use simpleplatform_sdk::prelude::*;
512/// # use simpleplatform_sdk::ai::{DeliveredAs, Options, Pages};
513/// # use simpleplatform_sdk::testing;
514/// # let _session = testing::install(|_name, _params| {
515/// #     Ok(json!({
516/// #         "data": "It commits us to a fixed price.",
517/// #         "metadata": { "delivery": [{
518/// #             "filename": "contract.pdf",
519/// #             "delivered_as": "document",
520/// #             "first_page": 40,
521/// #             "last_page": 52,
522/// #             "transcribed_pages": [],
523/// #             "fallback": {
524/// #                 "reason": "transcription_failed",
525/// #                 "pages": [44],
526/// #                 "message": "page 44: the page image could not be read"
527/// #             }
528/// #         }] }
529/// #     }))
530/// # });
531/// let read = simple::ai::summarize(
532///     json!({
533///         "file_hash": "9f2c…",
534///         "filename": "contract.pdf",
535///         "mime_type": "application/pdf",
536///         "first_page": 40,
537///         "last_page": 52,
538///         "deliver_as": "text"
539///     }),
540///     "Say what it commits us to.",
541///     Options::default(),
542/// )?;
543///
544/// for file in &read.metadata.delivery {
545///     if let Some(fallback) = &file.fallback {
546///         println!("{} was read as a PDF: {}", file.filename, fallback.message);
547///     }
548/// }
549///
550/// let contract = &read.metadata.delivery[0];
551///
552/// assert_eq!(contract.delivered_as, DeliveredAs::Document);
553/// assert_eq!(contract.pages, Some(Pages::new(40, 52)));
554/// # Ok::<(), Error>(())
555/// ```
556#[derive(Clone, Debug, PartialEq, Eq)]
557pub struct FileDelivery {
558    /// The file's name.
559    pub filename: String,
560
561    /// How the file travelled.
562    pub delivered_as: DeliveredAs,
563
564    /// The range the file was cut to, when one was asked for. Unset when the
565    /// whole file travelled.
566    pub pages: Option<Pages>,
567
568    /// The pages whose text was read from their images rather than from their
569    /// text layer. Empty when none were.
570    pub transcribed_pages: Vec<u32>,
571
572    /// Set only when text was asked for and the document travelled instead:
573    /// which pages could not be read, and why.
574    pub fallback: Option<DeliveryFallback>,
575}
576
577/// How a file reached the model.
578///
579/// This is what happened, where [`Delivery`] is what was asked for, and it has
580/// a way that cannot be asked for: an image only ever travels as one.
581///
582/// It is `#[non_exhaustive]`: a `match` on it needs a `_` arm, so a way added
583/// later does not break an action that was already written.
584#[non_exhaustive]
585#[derive(Clone, Copy, Debug, PartialEq, Eq)]
586pub enum DeliveredAs {
587    /// The document itself, which the model reads with its layout, tables and
588    /// figures.
589    Document,
590    /// The text read out of the document, with any page that had none read from
591    /// its image.
592    Text,
593    /// An image.
594    Image,
595}
596
597/// Why a file asked for as text travelled as the document instead.
598#[derive(Clone, Debug, Default, PartialEq, Eq)]
599pub struct DeliveryFallback {
600    /// Why, as the platform names it: `transcription_failed` when pages with no
601    /// text of their own could not be read from their images. It is kept as
602    /// the platform's word rather than narrowed to a list, so a reason added
603    /// later still reaches you instead of hiding that a fallback happened.
604    pub reason: String,
605
606    /// The pages that could not be read, numbered in the original document.
607    pub pages: Vec<u32>,
608
609    /// What went wrong, page by page, in the platform's words.
610    pub message: String,
611}
612
613/// How far a face search reaches.
614#[derive(Clone, Debug, Default)]
615pub struct FaceSearch {
616    /// The most matches to answer with. Unset leaves the platform's default.
617    pub max_faces: Option<u32>,
618
619    /// The similarity a match has to reach, as a percentage from `0.0` to
620    /// `100.0`. Unset leaves the platform's default.
621    pub similarity_threshold: Option<f64>,
622}
623
624/// Read structured data out of an input, against a schema you give.
625///
626/// The schema is the contract the answer is held to, and the type parameter is
627/// what that answer is decoded into — so give the model the schema of the struct
628/// you want back.
629///
630/// ```
631/// # use simpleplatform_sdk::prelude::*;
632/// # use simpleplatform_sdk::ai::{Execution, Options};
633/// # use simpleplatform_sdk::testing;
634/// #[derive(Deserialize)]
635/// struct Sentiment {
636///     score: f64,
637/// }
638///
639/// # let _session = testing::install(|_name, _params| {
640/// #     Ok(json!({ "data": { "score": 0.8 }, "metadata": { "output_tokens": 6 } }))
641/// # });
642/// let schema = json!({
643///     "type": "object",
644///     "properties": { "score": { "type": "number" } },
645///     "required": ["score"]
646/// });
647///
648/// let read: Execution<Sentiment> = simple::ai::extract(
649///     json!("The delivery arrived early and intact."),
650///     "Score the sentiment from -1.0 to 1.0.",
651///     schema,
652///     Options::default(),
653/// )?;
654///
655/// assert_eq!(read.data.score, 0.8);
656/// # Ok::<(), Error>(())
657/// ```
658pub fn extract<T: DeserializeOwned>(
659    input: Value,
660    prompt: &str,
661    schema: Value,
662    options: Options,
663) -> Result<Execution<T>, Error> {
664    refuse_empty(
665        &input,
666        "extract needs something to read.",
667        "Pass the text, the document handle, or the object the extraction is to read.",
668    )?;
669    refuse_empty_prompt(prompt, "extract")?;
670
671    if !schema.is_object() {
672        return Err(
673            Error::invalid("extract needs a JSON Schema object describing the answer.").hint(
674                "Pass a schema object naming the type and the properties the answer is to have.",
675            ),
676        );
677    }
678
679    run("extract", input, Some(prompt), Some(schema), &options)
680}
681
682/// Write prose about an input.
683///
684/// ```
685/// # use simpleplatform_sdk::prelude::*;
686/// # use simpleplatform_sdk::ai::Options;
687/// # use simpleplatform_sdk::testing;
688/// # let _session = testing::install(|_name, _params| {
689/// #     Ok(json!({ "data": "A refund was agreed.", "metadata": { "output_tokens": 5 } }))
690/// # });
691/// let written = simple::ai::summarize(
692///     json!({ "file_hash": "3a91…", "mime_type": "application/pdf" }),
693///     "Summarise what was agreed, in one sentence.",
694///     Options::default(),
695/// )?;
696///
697/// assert_eq!(written.data, "A refund was agreed.");
698/// # Ok::<(), Error>(())
699/// ```
700pub fn summarize(input: Value, prompt: &str, options: Options) -> Result<Execution<String>, Error> {
701    refuse_empty(
702        &input,
703        "summarize needs something to read.",
704        "Pass the text, the document handle, or the object the summary is to be written from.",
705    )?;
706    refuse_empty_prompt(prompt, "summarize")?;
707
708    run("summarize", input, Some(prompt), None, &options)
709}
710
711/// Turn audio or video into words.
712///
713/// `document` is the handle of an uploaded audio or video file. The prompt and
714/// the schema are written here, from what `wanted` asks for, which is why the
715/// answer is a [`Transcript`] rather than a type you have to name.
716///
717/// ```
718/// # use simpleplatform_sdk::prelude::*;
719/// # use simpleplatform_sdk::ai::{Options, Participants, TranscribeOptions};
720/// # use simpleplatform_sdk::testing;
721/// # let _session = testing::install(|_name, _params| {
722/// #     Ok(json!({
723/// #         "data": {
724/// #             "language": "en",
725/// #             "transcript": "Customer: My order is late.\nAgent: I have refunded it.",
726/// #             "participants": ["Customer", "Agent"]
727/// #         },
728/// #         "metadata": { "input_tokens": 4_200, "output_tokens": 96 }
729/// #     }))
730/// # });
731/// let heard = simple::ai::transcribe(
732///     json!({ "file_hash": "c07b…", "mime_type": "audio/mpeg" }),
733///     TranscribeOptions {
734///         include_transcript: true,
735///         participants: Participants::Named(vec!["Customer".into(), "Agent".into()]),
736///         ..Default::default()
737///     },
738///     Options::default(),
739/// )?;
740///
741/// assert_eq!(heard.data.language, "en");
742/// assert_eq!(heard.data.participants, ["Customer", "Agent"]);
743/// # Ok::<(), Error>(())
744/// ```
745pub fn transcribe(
746    document: Value,
747    wanted: TranscribeOptions,
748    options: Options,
749) -> Result<Execution<Transcript>, Error> {
750    refuse_unless_media(&document)?;
751
752    if !wanted.include_transcript && !wanted.summarize {
753        return Err(
754            Error::invalid("transcribe needs to be told what to produce.")
755                .hint("Set include_transcript, summarize, or both."),
756        );
757    }
758
759    let prompt = transcribe_prompt(&wanted);
760
761    run(
762        "extract",
763        document,
764        Some(&prompt),
765        Some(transcribe_schema(&wanted)),
766        &options,
767    )
768}
769
770/// Read the pages of a PDF that have no usable text of their own — scans,
771/// image-only exhibits, text that reads as noise — from their images.
772///
773/// Every such page is read once, and it is the same transcription an
774/// [`extract`] or [`summarize`] that asked for [`Delivery::Text`] is given in
775/// the page's place. A quote on an image page can therefore be checked against
776/// the very text the answer was built on. Pages the platform reads as text are
777/// neither read nor returned.
778///
779/// A page is not transcribed a second time to check the first: that would be
780/// the same model reading the same image again, doubling the cost of every
781/// scanned page without being independent of the first read. An independent
782/// check reads the pages a second way, as the document itself
783/// ([`Delivery::Document`]), and compares the answers.
784///
785/// Transcriptions are kept per version of the file and page, and shared with
786/// every other operation, so a page already read for an [`extract`] or
787/// [`summarize`] that asked for text is not read again.
788///
789/// `pages` narrows the reading to a range, counted from 1 with both ends
790/// included, and the answer still numbers pages in the original document. The
791/// file is taken as a [`DocumentHandle`] rather than a [`DocumentInput`]
792/// because the pages are answered as text, so a delivery has no meaning here
793/// and cannot be sent. Anything that is not a PDF is refused before the host
794/// is asked.
795///
796/// ```
797/// # use simpleplatform_sdk::prelude::*;
798/// # use simpleplatform_sdk::ai::{PageTranscription, Pages, TranscribePagesOptions};
799/// # use simpleplatform_sdk::testing;
800/// # let _session = testing::install(|_name, _params| {
801/// #     Ok(json!({
802/// #         "data": { "pages": [
803/// #             { "page": 41, "text": "## Scope of Work\n\nThe Contractor shall furnish…" },
804/// #             { "page": 44, "error": "the page image could not be read" }
805/// #         ] },
806/// #         "metadata": { "input_tokens": 0, "output_tokens": 0 }
807/// #     }))
808/// # });
809/// # let contract = DocumentHandle {
810/// #     file_hash: "9f2c…".into(),
811/// #     filename: "contract.pdf".into(),
812/// #     mime_type: "application/pdf".into(),
813/// #     size: 1_048_576,
814/// #     storage_path: "acme/documents/9f/2c/9f2c….pdf".into(),
815/// # };
816/// let quote = "The Contractor shall furnish";
817///
818/// let read = simple::ai::transcribe_pages(
819///     &contract,
820///     Some(Pages::new(40, 52)),
821///     TranscribePagesOptions::default(),
822/// )?;
823///
824/// for page in &read.data {
825///     match page {
826///         PageTranscription::Read { page, text } => {
827///             println!("page {page} carries the quote: {}", text.contains(quote));
828///         }
829///         PageTranscription::Unread { page, error } => {
830///             println!("page {page} could not be read: {error}");
831///         }
832///     }
833/// }
834///
835/// assert!(read.data[0].text().is_some_and(|text| text.contains(quote)));
836/// assert_eq!(read.data[1].page(), 44);
837/// # Ok::<(), Error>(())
838/// ```
839pub fn transcribe_pages(
840    document: &DocumentHandle,
841    pages: Option<Pages>,
842    options: TranscribePagesOptions,
843) -> Result<Execution<Vec<PageTranscription>>, Error> {
844    refuse_unless_pdf(document)?;
845
846    let input = json!(DocumentInput {
847        file: document.clone(),
848        pages,
849        deliver_as: None,
850    });
851
852    // The same universal options every other caller of the operation sends,
853    // since the whole map is part of the key a kept result is found under.
854    let options = Options {
855        regenerate: options.regenerate,
856        timeout: options.timeout,
857        ..Options::default()
858    };
859
860    let read: Execution<Value> = run("transcribe", input, None, None, &options)?;
861
862    Ok(Execution {
863        data: page_transcriptions(&read.data),
864        metadata: read.metadata,
865    })
866}
867
868/// Add a face to the tenant's collection, under the subject it belongs to.
869///
870/// `image` is either a base64 image or a document handle. The answer is the id
871/// of the enrolled face, which is what [`delete_face`] takes.
872///
873/// ```
874/// # use simpleplatform_sdk::prelude::*;
875/// # use simpleplatform_sdk::testing;
876/// # let _session = testing::install(|_name, _params| Ok(json!({ "face_id": "FACE-1" })));
877/// let face_id = simple::ai::enroll_face("EMP-42", json!("iVBORw0KGgo…"))?;
878///
879/// assert_eq!(face_id, "FACE-1");
880/// # Ok::<(), Error>(())
881/// ```
882pub fn enroll_face(subject_id: &str, image: Value) -> Result<String, Error> {
883    if subject_id.trim().is_empty() {
884        return Err(
885            Error::invalid("enroll_face needs the subject the face belongs to.")
886                .hint("Pass the id the face is to be found under, such as an employee or user id."),
887        );
888    }
889
890    refuse_empty(
891        &image,
892        "enroll_face needs an image of the face.",
893        "Pass a base64 image, or the handle of an uploaded one.",
894    )?;
895
896    let image = upload_pending(image)?;
897
898    let enrolled = host::transport()?
899        .call(
900            FACE_ENROLL.to_string(),
901            json!({ "image": image, "subject_id": subject_id }),
902        )
903        .map_err(|cause| {
904            cause.hint(
905                "Search the subject before enrolling the same face again, and report the failure.",
906            )
907        })?;
908
909    enrolled
910        .get("face_id")
911        .and_then(Value::as_str)
912        .map(str::to_string)
913        .ok_or_else(|| {
914            Error::Json(Fault::new(
915                Code::unspecified(),
916                "The enrolment answered without the id of the face it enrolled.",
917            ))
918            .hint("Search the subject to establish what was enrolled before enrolling again.")
919        })
920}
921
922/// Find the faces in the tenant's collection that match an image.
923///
924/// `image` is either a base64 image or a document handle. The matches are
925/// decoded into the type the call asks for; name your own to read them, or ask
926/// for a [`Value`] to keep them whole.
927///
928/// ```
929/// # use simpleplatform_sdk::prelude::*;
930/// # use simpleplatform_sdk::ai::FaceSearch;
931/// # use simpleplatform_sdk::testing;
932/// # let _session = testing::install(|_name, _params| Ok(json!([{ "subject_id": "EMP-42" }])));
933/// let found: Value = simple::ai::search_face(
934///     json!("iVBORw0KGgo…"),
935///     FaceSearch {
936///         max_faces: Some(3),
937///         similarity_threshold: Some(92.5),
938///     },
939/// )?;
940///
941/// assert_eq!(found[0]["subject_id"], json!("EMP-42"));
942/// # Ok::<(), Error>(())
943/// ```
944pub fn search_face<T: DeserializeOwned>(image: Value, options: FaceSearch) -> Result<T, Error> {
945    refuse_empty(
946        &image,
947        "search_face needs an image to search with.",
948        "Pass a base64 image, or the handle of an uploaded one.",
949    )?;
950
951    let image = upload_pending(image)?;
952
953    let mut narrowing = Map::new();
954
955    if let Some(max_faces) = options.max_faces {
956        narrowing.insert("max_faces".to_string(), Value::from(max_faces));
957    }
958
959    if let Some(threshold) = options
960        .similarity_threshold
961        .filter(|value| value.is_finite())
962    {
963        narrowing.insert("similarity_threshold".to_string(), Value::from(threshold));
964    }
965
966    let found = host::transport()?
967        .call(
968            FACE_SEARCH.to_string(),
969            json!({ "image": image, "options": narrowing }),
970        )
971        .map_err(|cause| {
972            cause.hint("Nothing in the collection was changed. Report the search failure.")
973        })?;
974
975    serde_json::from_value(found).map_err(|cause| {
976        Error::Json(Fault::new(
977            Code::unspecified(),
978            format!("The matches could not be read into the type this action asked for: {cause}"),
979        ))
980        .hint("Read the matches as a Value, or name a type that matches what the search answers.")
981    })
982}
983
984/// Remove faces from the tenant's collection, by the ids they were enrolled
985/// under.
986///
987/// The answer is the ids the platform reports as removed.
988///
989/// ```
990/// # use simpleplatform_sdk::prelude::*;
991/// # use simpleplatform_sdk::testing;
992/// # let _session = testing::install(|_name, _params| Ok(json!({ "deleted": ["FACE-1"] })));
993/// let removed = simple::ai::delete_face(&["FACE-1".to_string()])?;
994///
995/// assert_eq!(removed, ["FACE-1"]);
996/// # Ok::<(), Error>(())
997/// ```
998pub fn delete_face(face_ids: &[String]) -> Result<Vec<String>, Error> {
999    if face_ids.is_empty() {
1000        return Err(Error::invalid("delete_face needs at least one face id.")
1001            .hint("Pass the ids enroll_face answered with."));
1002    }
1003
1004    let removed = host::transport()?
1005        .call(FACE_DELETE.to_string(), json!({ "face_ids": face_ids }))
1006        .map_err(|cause| {
1007            cause.hint(
1008                "Treat the removal as unfinished: search the collection to establish what is \
1009                 still in it before deleting again.",
1010            )
1011        })?;
1012
1013    let listed = match &removed {
1014        Value::Array(ids) => Some(ids),
1015        other => other.get("deleted").and_then(Value::as_array),
1016    };
1017
1018    Ok(listed
1019        .map(|ids| {
1020            ids.iter()
1021                .filter_map(Value::as_str)
1022                .map(str::to_string)
1023                .collect()
1024        })
1025        .unwrap_or_default())
1026}
1027
1028/// One operation: upload what is pending, ask the primitive, read the answer.
1029fn run<T: DeserializeOwned>(
1030    operation: &str,
1031    input: Value,
1032    prompt: Option<&str>,
1033    schema: Option<Value>,
1034    options: &Options,
1035) -> Result<Execution<T>, Error> {
1036    let input = upload_pending(input)?;
1037
1038    let answered = host::transport()?
1039        .call(
1040            ORCHESTRATOR.to_string(),
1041            payload(operation, input, prompt, schema, options),
1042        )
1043        .map_err(|cause| {
1044            cause.hint(
1045                "No tenant record was changed. Review the input, the prompt and the schema, \
1046                 and report the failure.",
1047            )
1048        })?;
1049
1050    // The metadata is read on its own and never decides the outcome, so a run
1051    // that answered is reported as one whatever it said about its own cost.
1052    let metadata = metadata_of(answered.get("metadata"));
1053    let data = answered.get("data").cloned().unwrap_or(Value::Null);
1054
1055    let data = serde_json::from_value(data).map_err(|cause| {
1056        Error::Json(Fault::new(
1057            Code::unspecified(),
1058            format!("The {operation} answer could not be read into the type this action asked for: {cause}"),
1059        ))
1060        .hint(
1061            "The operation ran. Read the answer as a Value, or name a type that matches the \
1062             schema, rather than running it again.",
1063        )
1064    })?;
1065
1066    Ok(Execution { data, metadata })
1067}
1068
1069/// Everything the primitive is told about one operation.
1070///
1071/// A choice that was not made is left out, so what arrives is what the call
1072/// asked for and the platform's own defaults fill the rest.
1073fn payload(
1074    operation: &str,
1075    input: Value,
1076    prompt: Option<&str>,
1077    schema: Option<Value>,
1078    options: &Options,
1079) -> Value {
1080    // The engine reads `reasoning` and `reasoning_budget` from here, and the
1081    // whole map is part of the key a cached result is found under.
1082    let mut universal = Map::new();
1083
1084    universal.insert("reasoning".to_string(), Value::Bool(options.reasoning));
1085
1086    if let Some(budget) = options.reasoning_budget {
1087        universal.insert("reasoning_budget".to_string(), Value::from(budget));
1088    }
1089
1090    if let Some(temperature) = options.temperature.filter(|value| value.is_finite()) {
1091        universal.insert("temperature".to_string(), Value::from(temperature));
1092    }
1093
1094    let mut payload = Map::new();
1095
1096    payload.insert("operation".to_string(), Value::from(operation));
1097    payload.insert("input".to_string(), input);
1098    payload.insert("options".to_string(), Value::Object(universal));
1099
1100    // An operation whose instructions are the platform's is sent none, rather
1101    // than an empty one it would have to tell apart from a prompt.
1102    if let Some(prompt) = prompt {
1103        payload.insert("prompt".to_string(), Value::from(prompt));
1104    }
1105    payload.insert("regenerate".to_string(), Value::Bool(options.regenerate));
1106
1107    if let Some(schema) = schema {
1108        payload.insert("schema".to_string(), schema);
1109    }
1110
1111    if let Some(model) = options.model {
1112        payload.insert("model".to_string(), Value::from(model.as_str()));
1113    }
1114
1115    if let Some(system_prompt) = &options.system_prompt {
1116        payload.insert(
1117            "systemPrompt".to_string(),
1118            Value::from(system_prompt.as_str()),
1119        );
1120    }
1121
1122    if let Some(timeout) = options.timeout {
1123        payload.insert("timeout".to_string(), Value::from(milliseconds(timeout)));
1124    }
1125
1126    Value::Object(payload)
1127}
1128
1129/// A duration as the milliseconds the wire carries.
1130fn milliseconds(timeout: Duration) -> u64 {
1131    u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX)
1132}
1133
1134/// What a run reported about itself.
1135fn metadata_of(metadata: Option<&Value>) -> Metadata {
1136    let Some(metadata) = metadata else {
1137        return Metadata::default();
1138    };
1139
1140    Metadata {
1141        input_tokens: count(metadata, "input_tokens").unwrap_or(0),
1142        output_tokens: count(metadata, "output_tokens").unwrap_or(0),
1143        reasoning_tokens: count(metadata, "reasoning_tokens"),
1144        reasoning: metadata
1145            .get("reasoning")
1146            .and_then(Value::as_str)
1147            .map(str::to_string),
1148        delivery: delivery_of(metadata.get("delivery")),
1149    }
1150}
1151
1152/// One token count, when it is there and is a count.
1153fn count(metadata: &Value, key: &str) -> Option<u64> {
1154    metadata.get(key).and_then(Value::as_u64)
1155}
1156
1157/// How each file travelled, as far as the report can be read.
1158///
1159/// The report explains an answer; it never decides whether there is one. An
1160/// entry that is not an object, names no file, or names a way of travelling
1161/// this SDK does not know is left out rather than failing a run that answered.
1162fn delivery_of(delivery: Option<&Value>) -> Vec<FileDelivery> {
1163    delivery
1164        .and_then(Value::as_array)
1165        .map(|files| files.iter().filter_map(file_delivery).collect())
1166        .unwrap_or_default()
1167}
1168
1169/// One file's delivery, when the entry can be read as one.
1170///
1171/// A range is kept only when both ends are there, because a range with one end
1172/// is not one the engine could have cut. A key sent as `null` reads as absent.
1173fn file_delivery(file: &Value) -> Option<FileDelivery> {
1174    let delivered_as = match file.get("delivered_as").and_then(Value::as_str)? {
1175        "document" => DeliveredAs::Document,
1176        "text" => DeliveredAs::Text,
1177        "image" => DeliveredAs::Image,
1178        _ => return None,
1179    };
1180
1181    let first_page = file.get("first_page").and_then(page_number);
1182    let last_page = file.get("last_page").and_then(page_number);
1183
1184    Some(FileDelivery {
1185        filename: file.get("filename").and_then(Value::as_str)?.to_string(),
1186        delivered_as,
1187        pages: first_page
1188            .zip(last_page)
1189            .map(|(first, last)| Pages::new(first, last)),
1190        transcribed_pages: page_numbers(file.get("transcribed_pages")),
1191        fallback: file
1192            .get("fallback")
1193            .filter(|fallback| fallback.is_object())
1194            .map(delivery_fallback),
1195    })
1196}
1197
1198/// A fallback, read leniently: that the document went instead of its text is
1199/// the fact that matters, so a member missing from the report does not lose it.
1200fn delivery_fallback(fallback: &Value) -> DeliveryFallback {
1201    let text = |key: &str| {
1202        fallback
1203            .get(key)
1204            .and_then(Value::as_str)
1205            .unwrap_or_default()
1206            .to_string()
1207    };
1208
1209    DeliveryFallback {
1210        reason: text("reason"),
1211        pages: page_numbers(fallback.get("pages")),
1212        message: text("message"),
1213    }
1214}
1215
1216/// A page number, when this is one: a whole number from 1.
1217fn page_number(value: &Value) -> Option<u32> {
1218    value
1219        .as_u64()
1220        .and_then(|number| u32::try_from(number).ok())
1221        .filter(|page| *page > 0)
1222}
1223
1224/// The page numbers in a list, leaving out anything that is not one.
1225fn page_numbers(pages: Option<&Value>) -> Vec<u32> {
1226    pages
1227        .and_then(Value::as_array)
1228        .map(|pages| pages.iter().filter_map(page_number).collect())
1229        .unwrap_or_default()
1230}
1231
1232/// Why a page the platform answered for without its text is unread.
1233const NO_TRANSCRIPTION: &str = "no transcription was returned for this page";
1234
1235/// Each page a transcription answered for, in the shape a caller can rely on.
1236///
1237/// An answer without a list of pages has none. An entry that names no page is
1238/// left out, since there is no page it could be the transcription of.
1239fn page_transcriptions(data: &Value) -> Vec<PageTranscription> {
1240    data.get("pages")
1241        .and_then(Value::as_array)
1242        .map(|pages| pages.iter().filter_map(page_transcription).collect())
1243        .unwrap_or_default()
1244}
1245
1246/// One page's transcription: its text or its error, never both.
1247///
1248/// The platform's page is mapped rather than passed through. An error wins
1249/// over any text, so no partial text is handed on, and a page that carries no
1250/// text is answered as unread rather than as text that is not there.
1251fn page_transcription(entry: &Value) -> Option<PageTranscription> {
1252    let page = entry.get("page").and_then(page_number)?;
1253    let error = entry.get("error").and_then(Value::as_str);
1254    let text = entry.get("text").and_then(Value::as_str);
1255
1256    Some(match (error, text) {
1257        (None, Some(text)) => PageTranscription::Read {
1258            page,
1259            text: text.to_string(),
1260        },
1261        (error, _) => PageTranscription::Unread {
1262            page,
1263            error: error.unwrap_or(NO_TRANSCRIPTION).to_string(),
1264        },
1265    })
1266}
1267
1268/// Put whatever is still pending into ephemeral storage, and answer with what
1269/// the operation is to be given instead.
1270///
1271/// A handle that is already stored, and anything that is not a handle at all,
1272/// comes back as it went in.
1273fn upload_pending(value: Value) -> Result<Value, Error> {
1274    if is_pending(&value) {
1275        let stored = host::transport()?
1276            .call(UPLOAD_EPHEMERAL.to_string(), value.clone())
1277            .map_err(|cause| {
1278                cause.hint(
1279                    "The file was not uploaded and the operation did not run. Upload the file, \
1280                     then pass the handle that upload answered with.",
1281                )
1282            })?;
1283
1284        return Ok(stored_handle(value, stored));
1285    }
1286
1287    if let Value::Array(items) = value {
1288        let mut uploaded = Vec::with_capacity(items.len());
1289
1290        for item in items {
1291            uploaded.push(upload_pending(item)?);
1292        }
1293
1294        return Ok(Value::Array(uploaded));
1295    }
1296
1297    Ok(value)
1298}
1299
1300/// The handle the operation reads, once a pending file has been stored.
1301///
1302/// The upload answers with the stored file's own reference: where it is, what
1303/// it hashes to, its name, its type and its size. Every one of those keys
1304/// describes the file, so every one of them is taken from the upload and none
1305/// is carried over. A value from before the upload would name a file that is
1306/// no longer the one being read.
1307///
1308/// Everything else on the handle is the caller's: how the file is to be sent,
1309/// which of its pages, and any key added to a file reference later. Those
1310/// describe the request, not the file, and the upload knows nothing about
1311/// them, so they are carried over. A caller reads a pending file and a stored
1312/// one the same way.
1313///
1314/// `pending` is the one key that is neither. It said the bytes had not been
1315/// stored yet. They have been now, so it is dropped. A handle that still
1316/// called itself pending would be uploaded a second time on the next pass.
1317fn stored_handle(handle: Value, stored: Value) -> Value {
1318    match (handle, stored) {
1319        (Value::Object(mut handle), Value::Object(stored)) => {
1320            handle.remove("pending");
1321            handle.extend(stored);
1322
1323            Value::Object(handle)
1324        }
1325        (_, stored) => stored,
1326    }
1327}
1328
1329/// Whether this is a document handle whose file is still to be uploaded.
1330///
1331/// It is one when it says it is pending and carries the hash of the file it
1332/// stands for, since that hash is what the upload is asked for by.
1333fn is_pending(value: &Value) -> bool {
1334    value.get("pending").and_then(Value::as_bool) == Some(true) && has_file_hash(value)
1335}
1336
1337/// Whether this carries the hash of a file.
1338fn has_file_hash(value: &Value) -> bool {
1339    value
1340        .get("file_hash")
1341        .and_then(Value::as_str)
1342        .is_some_and(|hash| !hash.is_empty())
1343}
1344
1345/// Refuse a value that names nothing, in the words of whoever needed it.
1346fn refuse_empty(value: &Value, message: &str, hint: &str) -> Result<(), Error> {
1347    let empty = match value {
1348        Value::Null => true,
1349        Value::String(text) => text.is_empty(),
1350        _ => false,
1351    };
1352
1353    if empty {
1354        return Err(Error::invalid(message).hint(hint));
1355    }
1356
1357    Ok(())
1358}
1359
1360/// Refuse an operation that has been given nothing to do.
1361fn refuse_empty_prompt(prompt: &str, operation: &str) -> Result<(), Error> {
1362    if prompt.trim().is_empty() {
1363        return Err(Error::invalid(format!("{operation} needs a prompt."))
1364            .hint("Say what the operation is to produce, in a sentence the model can act on."));
1365    }
1366
1367    Ok(())
1368}
1369
1370/// Refuse anything that is not the handle of an audio or video file.
1371fn refuse_unless_media(document: &Value) -> Result<(), Error> {
1372    if !has_file_hash(document) {
1373        return Err(
1374            Error::invalid("transcribe needs a document handle to work on.").hint(
1375                "Pass the handle of the uploaded file, the one carrying file_hash and mime_type.",
1376            ),
1377        );
1378    }
1379
1380    let media = document
1381        .get("mime_type")
1382        .and_then(Value::as_str)
1383        .map(str::to_lowercase)
1384        .is_some_and(|mime| mime.starts_with("audio/") || mime.starts_with("video/"));
1385
1386    if !media {
1387        return Err(Error::invalid("transcribe works on audio and video.")
1388            .hint("Pass a handle whose mime_type begins audio/ or video/."));
1389    }
1390
1391    Ok(())
1392}
1393
1394/// Refuse anything that is not the handle of a stored PDF.
1395///
1396/// The type is compared without its parameters and without regard to case, so
1397/// `Application/PDF; version=1.7` is a PDF.
1398fn refuse_unless_pdf(document: &DocumentHandle) -> Result<(), Error> {
1399    if document.file_hash.is_empty() {
1400        return Err(
1401            Error::invalid("transcribe_pages needs the handle of a stored file.").hint(
1402                "Pass the handle the upload or the record answered with, the one carrying \
1403                 file_hash.",
1404            ),
1405        );
1406    }
1407
1408    let essence = document.mime_type.split(';').next().unwrap_or_default();
1409
1410    if !essence.trim().eq_ignore_ascii_case("application/pdf") {
1411        return Err(Error::invalid("transcribe_pages reads the pages of a PDF.")
1412            .hint("Pass the handle of a PDF. Read any other file with extract or summarize."));
1413    }
1414
1415    Ok(())
1416}
1417
1418/// The schema a transcription is held to, written from what was asked for.
1419fn transcribe_schema(wanted: &TranscribeOptions) -> Value {
1420    let identify = wanted.participants.wanted();
1421
1422    let mut properties = Map::new();
1423    let mut required = vec![Value::from("language")];
1424
1425    properties.insert(
1426        "language".to_string(),
1427        json!({
1428            "description": "The detected language of the audio (ISO 639-1 code, e.g., \"en\", \"es\")",
1429            "type": "string"
1430        }),
1431    );
1432
1433    if wanted.include_transcript {
1434        let description = match (identify, wanted.include_timestamps) {
1435            (true, true) => {
1436                "The full transcript with participant labels and timestamps. \
1437                 Format: [MM:SS] Participant Name: text"
1438            }
1439            (true, false) => {
1440                "The full transcript with participant labels. Format: Participant Name: text"
1441            }
1442            (false, true) => "The full transcript with timestamps. Format: [MM:SS] text",
1443            (false, false) => "The full transcript of the audio",
1444        };
1445
1446        properties.insert(
1447            "transcript".to_string(),
1448            json!({ "description": description, "type": "string" }),
1449        );
1450        required.push(Value::from("transcript"));
1451    }
1452
1453    if wanted.summarize {
1454        let description = if identify {
1455            "A concise summary of the audio content, including key points from each participant"
1456        } else {
1457            "A concise summary of the audio content"
1458        };
1459
1460        properties.insert(
1461            "summary".to_string(),
1462            json!({ "description": description, "type": "string" }),
1463        );
1464        required.push(Value::from("summary"));
1465    }
1466
1467    if identify {
1468        properties.insert(
1469            "participants".to_string(),
1470            json!({
1471                "description": "List of identified participants in the audio",
1472                "items": { "type": "string" },
1473                "type": "array"
1474            }),
1475        );
1476        required.push(Value::from("participants"));
1477    }
1478
1479    json!({ "properties": properties, "required": required, "type": "object" })
1480}
1481
1482/// The prompt a transcription runs on, written from what was asked for.
1483fn transcribe_prompt(wanted: &TranscribeOptions) -> String {
1484    let identify = wanted.participants.wanted();
1485    let names = wanted.participants.names();
1486
1487    let mut prompt = String::from("Analyze this audio/video file and provide:\n");
1488
1489    if wanted.include_transcript {
1490        if identify {
1491            match &names {
1492                Some(names) => {
1493                    prompt.push_str("- A complete transcript identifying these participants: ");
1494                    prompt.push_str(names);
1495                    prompt.push_str(". ");
1496                }
1497                None => prompt.push_str(
1498                    "- A complete transcript with participant identification \
1499                     (label participants as Participant 1, Participant 2, etc.). ",
1500                ),
1501            }
1502
1503            if wanted.include_timestamps {
1504                prompt.push_str(
1505                    "Include timestamps in [MM:SS] format before each participant segment.\n",
1506                );
1507            } else {
1508                prompt.push_str("Format each line as \"Participant Name: text\".\n");
1509            }
1510        } else if wanted.include_timestamps {
1511            prompt.push_str(
1512                "- A complete transcript with timestamps in [MM:SS] format before each segment\n",
1513            );
1514        } else {
1515            prompt.push_str("- A complete transcript of all spoken content\n");
1516        }
1517    }
1518
1519    if wanted.summarize {
1520        if identify {
1521            prompt.push_str("- A concise summary highlighting key points from each participant\n");
1522        } else {
1523            prompt.push_str("- A concise summary of the main points and key information\n");
1524        }
1525    }
1526
1527    if identify {
1528        match &names {
1529            Some(names) => {
1530                prompt.push_str("- Identify and distinguish between these participants: ");
1531                prompt.push_str(names);
1532                prompt.push('\n');
1533            }
1534            None => prompt.push_str("- Identify and list all distinct participants in the audio\n"),
1535        }
1536    }
1537
1538    prompt.push_str("- The detected language code (ISO 639-1 format)\n");
1539
1540    prompt
1541}
1542
1543#[cfg(test)]
1544mod tests {
1545    use serde_json::json;
1546
1547    use super::*;
1548    use crate::testing;
1549
1550    /// What the primitive answers when a test does not care what came back.
1551    fn answered() -> Value {
1552        json!({ "data": { "ok": true }, "metadata": { "input_tokens": 1, "output_tokens": 2 } })
1553    }
1554
1555    /// A stored file, as an upload answers with one.
1556    fn stored() -> DocumentHandle {
1557        DocumentHandle {
1558            file_hash: "ffce20f1".to_string(),
1559            filename: "contract.pdf".to_string(),
1560            mime_type: "application/pdf".to_string(),
1561            size: 1_234,
1562            storage_path: "acme/documents/ffce20f1".to_string(),
1563        }
1564    }
1565
1566    #[test]
1567    fn a_document_input_travels_as_the_handle_with_the_keys_the_engine_reads() {
1568        let sent = json!(DocumentInput {
1569            file: stored(),
1570            pages: Some(Pages::new(12, 30)),
1571            deliver_as: Some(Delivery::Text),
1572        });
1573
1574        assert_eq!(sent["storage_path"], json!("acme/documents/ffce20f1"));
1575        assert_eq!(sent["file_hash"], json!("ffce20f1"));
1576        assert_eq!(sent["mime_type"], json!("application/pdf"));
1577        assert_eq!(
1578            sent["first_page"],
1579            json!(12),
1580            "a range travels as the two keys the engine reads, not as a nested one"
1581        );
1582        assert_eq!(sent["last_page"], json!(30));
1583        assert_eq!(sent["deliver_as"], json!("text"));
1584    }
1585
1586    #[test]
1587    fn a_document_input_that_asks_for_nothing_carries_neither_key() {
1588        let sent = json!(DocumentInput {
1589            file: stored(),
1590            ..Default::default()
1591        });
1592
1593        assert_eq!(sent["filename"], json!("contract.pdf"));
1594        assert_eq!(
1595            sent.get("deliver_as"),
1596            None,
1597            "asking for nothing says nothing, so the default stays the engine's"
1598        );
1599        assert_eq!(sent.get("first_page"), None);
1600        assert_eq!(sent.get("last_page"), None);
1601    }
1602
1603    #[test]
1604    fn an_extraction_asks_the_primitive_and_answers_with_the_type_it_was_asked_for() {
1605        #[derive(Debug, Deserialize, PartialEq)]
1606        struct Invoice {
1607            number: String,
1608            total: f64,
1609        }
1610
1611        let session = testing::install(|name, params| {
1612            assert_eq!(name, ORCHESTRATOR);
1613            assert_eq!(params["operation"], json!("extract"));
1614            assert_eq!(params["prompt"], json!("Read it."));
1615            assert_eq!(params["input"], json!("INV-8, 240.00"));
1616            assert_eq!(params["schema"]["type"], json!("object"));
1617
1618            Ok(json!({
1619                "data": { "number": "INV-8", "total": 240.0 },
1620                "metadata": { "input_tokens": 11, "output_tokens": 3 }
1621            }))
1622        });
1623
1624        let read: Execution<Invoice> = extract(
1625            json!("INV-8, 240.00"),
1626            "Read it.",
1627            json!({ "type": "object" }),
1628            Options::default(),
1629        )
1630        .unwrap();
1631
1632        assert_eq!(
1633            read.data,
1634            Invoice {
1635                number: "INV-8".to_string(),
1636                total: 240.0
1637            }
1638        );
1639        assert_eq!(read.metadata.input_tokens, 11);
1640        assert_eq!(read.metadata.output_tokens, 3);
1641        assert_eq!(session.calls().len(), 1);
1642    }
1643
1644    #[test]
1645    fn a_default_call_reasons_does_not_regenerate_and_leaves_the_rest_to_the_platform() {
1646        let session = testing::install(|_name, _params| Ok(answered()));
1647
1648        let _: Execution<Value> = extract(
1649            json!("text"),
1650            "Read it.",
1651            json!({ "type": "object" }),
1652            Options::default(),
1653        )
1654        .unwrap();
1655
1656        let sent = session.calls()[0].params.clone();
1657
1658        assert_eq!(sent["options"], json!({ "reasoning": true }));
1659        assert_eq!(sent["regenerate"], json!(false));
1660        assert!(sent.get("model").is_none());
1661        assert!(sent.get("systemPrompt").is_none());
1662        assert!(sent.get("timeout").is_none());
1663    }
1664
1665    #[test]
1666    fn every_option_that_was_set_travels_with_the_operation() {
1667        let session = testing::install(|_name, _params| Ok(answered()));
1668
1669        let _: Execution<Value> = extract(
1670            json!("text"),
1671            "Read it.",
1672            json!({ "type": "object" }),
1673            Options {
1674                model: Some(Model::Xl),
1675                system_prompt: Some("You are an auditor.".to_string()),
1676                temperature: Some(0.25),
1677                reasoning: false,
1678                reasoning_budget: Some(2_048),
1679                regenerate: true,
1680                timeout: Some(Duration::from_secs(90)),
1681            },
1682        )
1683        .unwrap();
1684
1685        let sent = session.calls()[0].params.clone();
1686
1687        assert_eq!(sent["model"], json!("xl"));
1688        assert_eq!(sent["systemPrompt"], json!("You are an auditor."));
1689        assert_eq!(sent["regenerate"], json!(true));
1690        assert_eq!(sent["timeout"], json!(90_000));
1691        assert_eq!(
1692            sent["options"],
1693            json!({ "reasoning": false, "reasoning_budget": 2_048, "temperature": 0.25 })
1694        );
1695    }
1696
1697    #[test]
1698    fn a_summary_answers_with_the_prose_and_carries_no_schema() {
1699        let session = testing::install(|_name, params| {
1700            assert_eq!(params["operation"], json!("summarize"));
1701            assert!(params.get("schema").is_none());
1702
1703            Ok(json!({ "data": "A refund was agreed.", "metadata": { "output_tokens": 5 } }))
1704        });
1705
1706        let written = summarize(json!("A long thread."), "One sentence.", Options::default())
1707            .expect("a summary answers with prose");
1708
1709        assert_eq!(written.data, "A refund was agreed.");
1710        assert_eq!(written.metadata.output_tokens, 5);
1711        assert_eq!(session.calls().len(), 1);
1712    }
1713
1714    #[test]
1715    fn a_pending_document_is_uploaded_before_the_operation_reads_it() {
1716        let session = testing::install(|name, _params| {
1717            if name == UPLOAD_EPHEMERAL {
1718                return Ok(json!({ "file_hash": "abc", "storage_path": "ephemeral/abc" }));
1719            }
1720
1721            Ok(answered())
1722        });
1723
1724        let _: Execution<Value> = extract(
1725            json!({ "file_hash": "abc", "pending": true }),
1726            "Read it.",
1727            json!({ "type": "object" }),
1728            Options::default(),
1729        )
1730        .unwrap();
1731
1732        let calls = session.calls();
1733
1734        assert_eq!(calls.len(), 2);
1735        assert_eq!(calls[0].name, UPLOAD_EPHEMERAL);
1736        assert_eq!(calls[1].name, ORCHESTRATOR);
1737        assert_eq!(
1738            calls[1].params["input"],
1739            json!({ "file_hash": "abc", "storage_path": "ephemeral/abc" }),
1740            "the operation reads the handle the upload answered with"
1741        );
1742    }
1743
1744    #[test]
1745    fn a_pending_document_is_read_the_way_the_caller_asked_for_it() {
1746        let session = testing::install(|name, _params| {
1747            if name == UPLOAD_EPHEMERAL {
1748                return Ok(json!({
1749                    "file_hash": "ffce20f1",
1750                    "filename": "contract.pdf",
1751                    "mime_type": "application/pdf",
1752                    "size": 1_234,
1753                    "storage_path": "acme/ephemeral/ffce20f1"
1754                }));
1755            }
1756
1757            Ok(answered())
1758        });
1759
1760        let _: Execution<Value> = extract(
1761            json!({
1762                "file_hash": "staged",
1763                "first_page": 12,
1764                "last_page": 30,
1765                "pending": true,
1766                "deliver_as": "text",
1767                "storage_path": "acme/staged/staged"
1768            }),
1769            "Read it.",
1770            json!({ "type": "object" }),
1771            Options::default(),
1772        )
1773        .unwrap();
1774
1775        let calls = session.calls();
1776        let sent = calls[1].params["input"].clone();
1777
1778        assert_eq!(
1779            sent["deliver_as"],
1780            json!("text"),
1781            "the caller said how the file was to be sent"
1782        );
1783        assert_eq!(
1784            sent["first_page"],
1785            json!(12),
1786            "the caller asked for a range"
1787        );
1788        assert_eq!(sent["last_page"], json!(30), "the caller asked for a range");
1789        assert_eq!(
1790            sent["file_hash"],
1791            json!("ffce20f1"),
1792            "the file is the one the upload stored"
1793        );
1794        assert_eq!(
1795            sent["storage_path"],
1796            json!("acme/ephemeral/ffce20f1"),
1797            "the file is read from where the upload put it"
1798        );
1799        assert_eq!(sent["filename"], json!("contract.pdf"));
1800        assert_eq!(sent["mime_type"], json!("application/pdf"));
1801        assert_eq!(sent["size"], json!(1_234));
1802        assert!(
1803            sent.get("pending").is_none(),
1804            "the file is stored, so the handle may not still call itself pending"
1805        );
1806    }
1807
1808    #[test]
1809    fn a_document_that_is_already_stored_is_read_where_it_is() {
1810        let session = testing::install(|_name, _params| Ok(answered()));
1811
1812        let handle = json!({ "file_hash": "abc", "mime_type": "application/pdf" });
1813
1814        let _: Execution<Value> = extract(
1815            handle.clone(),
1816            "Read it.",
1817            json!({ "type": "object" }),
1818            Options::default(),
1819        )
1820        .unwrap();
1821
1822        let calls = session.calls();
1823
1824        assert_eq!(calls.len(), 1, "nothing was uploaded");
1825        assert_eq!(calls[0].params["input"], handle);
1826    }
1827
1828    #[test]
1829    fn a_list_of_documents_is_uploaded_item_by_item() {
1830        let session = testing::install(|name, params| {
1831            if name == UPLOAD_EPHEMERAL {
1832                let hash = params["file_hash"].as_str().unwrap_or_default();
1833
1834                return Ok(json!({
1835                    "file_hash": hash,
1836                    "storage_path": format!("ephemeral/{hash}")
1837                }));
1838            }
1839
1840            Ok(answered())
1841        });
1842
1843        let _: Execution<Value> = extract(
1844            json!([
1845                { "file_hash": "one", "pending": true },
1846                { "file_hash": "two", "mime_type": "image/png" },
1847                { "file_hash": "three", "pending": true }
1848            ]),
1849            "Read them.",
1850            json!({ "type": "object" }),
1851            Options::default(),
1852        )
1853        .unwrap();
1854
1855        let calls = session.calls();
1856
1857        assert_eq!(calls.len(), 3, "two uploads and the operation");
1858        assert_eq!(
1859            calls[2].params["input"],
1860            json!([
1861                { "file_hash": "one", "storage_path": "ephemeral/one" },
1862                { "file_hash": "two", "mime_type": "image/png" },
1863                { "file_hash": "three", "storage_path": "ephemeral/three" }
1864            ])
1865        );
1866    }
1867
1868    #[test]
1869    fn a_handle_is_pending_only_while_it_says_so_and_carries_a_hash_to_upload_by() {
1870        assert!(is_pending(&json!({ "file_hash": "abc", "pending": true })));
1871        assert!(!is_pending(&json!({ "file_hash": "abc" })));
1872        assert!(!is_pending(&json!({ "pending": true })));
1873        assert!(!is_pending(&json!({ "file_hash": "", "pending": true })));
1874        assert!(!is_pending(
1875            &json!({ "file_hash": "abc", "pending": "yes" })
1876        ));
1877        assert!(!is_pending(&json!("some words")));
1878        assert!(!is_pending(&json!([
1879            { "file_hash": "abc", "pending": true }
1880        ])));
1881    }
1882
1883    #[test]
1884    fn an_answer_that_reports_nothing_about_its_cost_is_still_an_answer() {
1885        let _session = testing::install(|_name, _params| Ok(json!({ "data": "done" })));
1886
1887        let written = summarize(json!("text"), "One sentence.", Options::default()).unwrap();
1888
1889        assert_eq!(written.data, "done");
1890        assert_eq!(written.metadata, Metadata::default());
1891    }
1892
1893    #[test]
1894    fn reasoning_reaches_the_caller_when_the_run_reported_it() {
1895        let _session = testing::install(|_name, _params| {
1896            Ok(json!({
1897                "data": "done",
1898                "metadata": {
1899                    "input_tokens": 9,
1900                    "output_tokens": 4,
1901                    "reasoning": "Read the total from the last line.",
1902                    "reasoning_tokens": 120
1903                }
1904            }))
1905        });
1906
1907        let written = summarize(json!("text"), "One sentence.", Options::default()).unwrap();
1908
1909        assert_eq!(written.metadata.reasoning_tokens, Some(120));
1910        assert_eq!(
1911            written.metadata.reasoning.as_deref(),
1912            Some("Read the total from the last line.")
1913        );
1914    }
1915
1916    #[test]
1917    fn a_run_reports_how_each_file_travelled_in_the_order_the_files_were_given() {
1918        let _session = testing::install(|_name, _params| {
1919            Ok(json!({
1920                "data": "done",
1921                "metadata": {
1922                    "input_tokens": 9,
1923                    "output_tokens": 4,
1924                    "delivery": [
1925                        {
1926                            "filename": "contract.pdf",
1927                            "delivered_as": "text",
1928                            "first_page": 40,
1929                            "last_page": 52,
1930                            "transcribed_pages": [41, 44],
1931                            "fallback": null
1932                        },
1933                        {
1934                            "filename": "drawings.pdf",
1935                            "delivered_as": "document",
1936                            "first_page": null,
1937                            "last_page": null,
1938                            "transcribed_pages": [],
1939                            "fallback": {
1940                                "reason": "transcription_failed",
1941                                "pages": [3, 7],
1942                                "message": "page 3: timed out; page 7: blank answer"
1943                            }
1944                        },
1945                        {
1946                            "filename": "site.jpg",
1947                            "delivered_as": "image",
1948                            "first_page": null,
1949                            "last_page": null,
1950                            "transcribed_pages": [],
1951                            "fallback": null
1952                        }
1953                    ]
1954                }
1955            }))
1956        });
1957
1958        let written = summarize(json!("text"), "One sentence.", Options::default()).unwrap();
1959
1960        assert_eq!(
1961            written.metadata.delivery,
1962            vec![
1963                FileDelivery {
1964                    filename: "contract.pdf".to_string(),
1965                    delivered_as: DeliveredAs::Text,
1966                    pages: Some(Pages::new(40, 52)),
1967                    transcribed_pages: vec![41, 44],
1968                    fallback: None,
1969                },
1970                FileDelivery {
1971                    filename: "drawings.pdf".to_string(),
1972                    delivered_as: DeliveredAs::Document,
1973                    pages: None,
1974                    transcribed_pages: Vec::new(),
1975                    fallback: Some(DeliveryFallback {
1976                        reason: "transcription_failed".to_string(),
1977                        pages: vec![3, 7],
1978                        message: "page 3: timed out; page 7: blank answer".to_string(),
1979                    }),
1980                },
1981                FileDelivery {
1982                    filename: "site.jpg".to_string(),
1983                    delivered_as: DeliveredAs::Image,
1984                    pages: None,
1985                    transcribed_pages: Vec::new(),
1986                    fallback: None,
1987                },
1988            ]
1989        );
1990        assert_eq!(
1991            written.metadata.input_tokens, 9,
1992            "the members a result already had read as they did"
1993        );
1994    }
1995
1996    #[test]
1997    fn an_answer_without_a_delivery_report_carries_none() {
1998        for metadata in [
1999            json!({ "input_tokens": 1 }),
2000            json!({ "delivery": null }),
2001            json!({ "delivery": "document" }),
2002            json!({ "delivery": { "filename": "contract.pdf", "delivered_as": "text" } }),
2003        ] {
2004            let _session = testing::install(move |_name, _params| {
2005                Ok(json!({ "data": "done", "metadata": metadata.clone() }))
2006            });
2007
2008            let written = summarize(json!("text"), "One sentence.", Options::default())
2009                .expect("a report that cannot be read never fails the run");
2010
2011            assert!(written.metadata.delivery.is_empty());
2012        }
2013    }
2014
2015    #[test]
2016    fn a_delivery_entry_that_cannot_be_read_is_left_out_and_the_rest_are_kept() {
2017        let delivery = delivery_of(Some(&json!([
2018            "contract.pdf",
2019            { "filename": "no-way.pdf" },
2020            { "filename": "audio.mp3", "delivered_as": "audio" },
2021            { "delivered_as": "text" },
2022            { "filename": 7, "delivered_as": "text" },
2023            {
2024                "filename": "kept.pdf",
2025                "delivered_as": "text",
2026                "first_page": 4,
2027                "transcribed_pages": ["5", -1, 0, 6, 4_294_967_296_u64, null]
2028            }
2029        ])));
2030
2031        assert_eq!(
2032            delivery,
2033            vec![FileDelivery {
2034                filename: "kept.pdf".to_string(),
2035                delivered_as: DeliveredAs::Text,
2036                pages: None,
2037                transcribed_pages: vec![6],
2038                fallback: None,
2039            }],
2040            "a range with one end is not a range, and only whole pages from 1 are pages"
2041        );
2042    }
2043
2044    #[test]
2045    fn a_fallback_that_is_reported_is_never_lost_to_a_missing_member() {
2046        let delivery = delivery_of(Some(&json!([
2047            {
2048                "filename": "contract.pdf",
2049                "delivered_as": "document",
2050                "fallback": { "reason": "transcription_failed" }
2051            },
2052            {
2053                "filename": "drawings.pdf",
2054                "delivered_as": "document",
2055                "fallback": "transcription_failed"
2056            }
2057        ])));
2058
2059        assert_eq!(
2060            delivery[0].fallback,
2061            Some(DeliveryFallback {
2062                reason: "transcription_failed".to_string(),
2063                pages: Vec::new(),
2064                message: String::new(),
2065            })
2066        );
2067        assert_eq!(
2068            delivery[1].fallback, None,
2069            "a fallback that is not an object says nothing that can be read"
2070        );
2071        assert!(delivery[0].transcribed_pages.is_empty());
2072    }
2073
2074    #[test]
2075    fn an_operation_with_nothing_to_do_is_refused_before_anything_is_sent() {
2076        let session = testing::install(|_name, _params| Ok(answered()));
2077
2078        let missing_input = extract::<Value>(
2079            json!(null),
2080            "Read it.",
2081            json!({ "type": "object" }),
2082            Options::default(),
2083        )
2084        .unwrap_err();
2085
2086        let missing_prompt = extract::<Value>(
2087            json!("text"),
2088            "   ",
2089            json!({ "type": "object" }),
2090            Options::default(),
2091        )
2092        .unwrap_err();
2093
2094        let missing_schema =
2095            extract::<Value>(json!("text"), "Read it.", json!([]), Options::default()).unwrap_err();
2096
2097        let missing_summary_input =
2098            summarize(json!(""), "One sentence.", Options::default()).unwrap_err();
2099
2100        for refusal in [
2101            &missing_input,
2102            &missing_prompt,
2103            &missing_schema,
2104            &missing_summary_input,
2105        ] {
2106            assert_eq!(refusal.code().as_str(), "INVALID_TOOL_INPUT");
2107        }
2108
2109        assert!(session.calls().is_empty());
2110    }
2111
2112    #[test]
2113    fn a_transcription_writes_its_own_schema_and_prompt() {
2114        let session = testing::install(|_name, _params| {
2115            Ok(json!({
2116                "data": { "language": "en", "transcript": "Hello." },
2117                "metadata": { "input_tokens": 400, "output_tokens": 12 }
2118            }))
2119        });
2120
2121        let heard = transcribe(
2122            json!({ "file_hash": "abc", "mime_type": "AUDIO/MPEG" }),
2123            TranscribeOptions {
2124                include_transcript: true,
2125                ..Default::default()
2126            },
2127            Options::default(),
2128        )
2129        .expect("an uppercase mime type is the same mime type");
2130
2131        assert_eq!(heard.data.language, "en");
2132        assert_eq!(heard.data.transcript.as_deref(), Some("Hello."));
2133        assert_eq!(heard.data.summary, None);
2134        assert!(heard.data.participants.is_empty());
2135
2136        let sent = session.calls()[0].params.clone();
2137
2138        assert_eq!(sent["operation"], json!("extract"));
2139        assert_eq!(
2140            sent["schema"]["required"],
2141            json!(["language", "transcript"])
2142        );
2143        assert_eq!(sent["schema"]["properties"]["transcript"]["type"], "string");
2144        assert_eq!(
2145            sent["prompt"],
2146            json!(
2147                "Analyze this audio/video file and provide:\n\
2148                 - A complete transcript of all spoken content\n\
2149                 - The detected language code (ISO 639-1 format)\n"
2150            )
2151        );
2152    }
2153
2154    #[test]
2155    fn named_participants_reach_both_the_prompt_and_the_schema() {
2156        let session = testing::install(|_name, _params| {
2157            Ok(json!({ "data": { "language": "en", "participants": ["Customer", "Agent"] } }))
2158        });
2159
2160        let heard = transcribe(
2161            json!({ "file_hash": "abc", "mime_type": "video/mp4" }),
2162            TranscribeOptions {
2163                include_transcript: true,
2164                include_timestamps: true,
2165                summarize: true,
2166                participants: Participants::Named(vec![
2167                    "Customer".to_string(),
2168                    "Agent".to_string(),
2169                ]),
2170            },
2171            Options::default(),
2172        )
2173        .unwrap();
2174
2175        assert_eq!(heard.data.participants, ["Customer", "Agent"]);
2176
2177        let sent = session.calls()[0].params.clone();
2178        let prompt = sent["prompt"].as_str().unwrap_or_default().to_string();
2179
2180        assert_eq!(
2181            sent["schema"]["required"],
2182            json!(["language", "transcript", "summary", "participants"])
2183        );
2184        assert_eq!(
2185            sent["schema"]["properties"]["participants"]["items"]["type"],
2186            json!("string")
2187        );
2188        assert!(prompt.contains("identifying these participants: Customer, Agent. "));
2189        assert!(prompt.contains("Include timestamps in [MM:SS] format"));
2190        assert!(
2191            prompt.contains("- A concise summary highlighting key points from each participant\n")
2192        );
2193        assert!(prompt
2194            .contains("- Identify and distinguish between these participants: Customer, Agent\n"));
2195    }
2196
2197    #[test]
2198    fn detected_participants_are_labelled_rather_than_named() {
2199        let session =
2200            testing::install(|_name, _params| Ok(json!({ "data": { "language": "en" } })));
2201
2202        let _ = transcribe(
2203            json!({ "file_hash": "abc", "mime_type": "audio/wav" }),
2204            TranscribeOptions {
2205                include_transcript: true,
2206                participants: Participants::Detect,
2207                ..Default::default()
2208            },
2209            Options::default(),
2210        )
2211        .unwrap();
2212
2213        let prompt = session.calls()[0].params["prompt"]
2214            .as_str()
2215            .unwrap_or_default()
2216            .to_string();
2217
2218        assert!(prompt.contains("label participants as Participant 1, Participant 2, etc."));
2219        assert!(prompt.contains("Format each line as \"Participant Name: text\".\n"));
2220        assert!(prompt.contains("- Identify and list all distinct participants in the audio\n"));
2221    }
2222
2223    #[test]
2224    fn naming_nobody_asks_for_the_same_thing_as_detecting() {
2225        let wanted = |participants| TranscribeOptions {
2226            include_transcript: true,
2227            participants,
2228            ..Default::default()
2229        };
2230
2231        assert_eq!(
2232            transcribe_prompt(&wanted(Participants::Named(Vec::new()))),
2233            transcribe_prompt(&wanted(Participants::Detect))
2234        );
2235        assert_eq!(
2236            transcribe_schema(&wanted(Participants::Named(Vec::new()))),
2237            transcribe_schema(&wanted(Participants::Detect))
2238        );
2239    }
2240
2241    #[test]
2242    fn a_transcription_that_cannot_run_is_refused_before_anything_is_sent() {
2243        let session = testing::install(|_name, _params| Ok(answered()));
2244
2245        let audio = json!({ "file_hash": "abc", "mime_type": "audio/mpeg" });
2246
2247        let nothing_wanted = transcribe(
2248            audio.clone(),
2249            TranscribeOptions::default(),
2250            Options::default(),
2251        )
2252        .unwrap_err();
2253
2254        let not_media = transcribe(
2255            json!({ "file_hash": "abc", "mime_type": "application/pdf" }),
2256            TranscribeOptions {
2257                include_transcript: true,
2258                ..Default::default()
2259            },
2260            Options::default(),
2261        )
2262        .unwrap_err();
2263
2264        let not_a_handle = transcribe(
2265            json!("some words"),
2266            TranscribeOptions {
2267                summarize: true,
2268                ..Default::default()
2269            },
2270            Options::default(),
2271        )
2272        .unwrap_err();
2273
2274        for refusal in [&nothing_wanted, &not_media, &not_a_handle] {
2275            assert_eq!(refusal.code().as_str(), "INVALID_TOOL_INPUT");
2276        }
2277
2278        assert!(session.calls().is_empty());
2279    }
2280
2281    #[test]
2282    fn a_page_transcription_asks_for_the_transcribe_operation_with_the_range_and_no_prompt() {
2283        let session = testing::install(|_name, _params| {
2284            Ok(json!({
2285                "data": {
2286                    "pages": [
2287                        { "page": 41, "text": "Scope of Work" },
2288                        { "page": 42, "error": "the answer carries no pages" }
2289                    ]
2290                },
2291                "metadata": {
2292                    "delivery": null,
2293                    "input_tokens": 0,
2294                    "output_tokens": 0,
2295                    "reasoning_tokens": 0
2296                }
2297            }))
2298        });
2299
2300        let read = transcribe_pages(
2301            &stored(),
2302            Some(Pages::new(40, 52)),
2303            TranscribePagesOptions::default(),
2304        )
2305        .unwrap();
2306
2307        let calls = session.calls();
2308
2309        assert_eq!(calls.len(), 1);
2310        assert_eq!(calls[0].name, ORCHESTRATOR);
2311        assert_eq!(
2312            calls[0].params,
2313            json!({
2314                "operation": "transcribe",
2315                "input": {
2316                    "file_hash": "ffce20f1",
2317                    "filename": "contract.pdf",
2318                    "mime_type": "application/pdf",
2319                    "size": 1_234,
2320                    "storage_path": "acme/documents/ffce20f1",
2321                    "first_page": 40,
2322                    "last_page": 52
2323                },
2324                "options": { "reasoning": true },
2325                "regenerate": false
2326            }),
2327            "no prompt, schema, model or delivery: the instructions are the platform's"
2328        );
2329
2330        assert_eq!(
2331            read.data,
2332            vec![
2333                PageTranscription::Read {
2334                    page: 41,
2335                    text: "Scope of Work".to_string()
2336                },
2337                PageTranscription::Unread {
2338                    page: 42,
2339                    error: "the answer carries no pages".to_string()
2340                },
2341            ]
2342        );
2343        assert_eq!(read.metadata.input_tokens, 0);
2344        assert!(read.metadata.delivery.is_empty());
2345    }
2346
2347    #[test]
2348    fn a_page_transcription_carries_only_the_whole_file_and_the_options_it_was_given() {
2349        let session = testing::install(|_name, _params| Ok(json!({ "data": { "pages": [] } })));
2350
2351        let read = transcribe_pages(
2352            &stored(),
2353            None,
2354            TranscribePagesOptions {
2355                regenerate: true,
2356                timeout: Some(Duration::from_secs(90)),
2357            },
2358        )
2359        .unwrap();
2360
2361        let sent = session.calls()[0].params.clone();
2362
2363        assert!(read.data.is_empty());
2364        assert!(sent["input"].get("first_page").is_none());
2365        assert!(sent["input"].get("last_page").is_none());
2366        assert_eq!(sent["regenerate"], json!(true));
2367        assert_eq!(sent["timeout"], json!(90_000));
2368        assert_eq!(sent["options"], json!({ "reasoning": true }));
2369    }
2370
2371    #[test]
2372    fn a_page_with_no_text_is_answered_as_unread_never_as_text_that_is_not_there() {
2373        let _session = testing::install(|_name, _params| {
2374            Ok(json!({
2375                "data": {
2376                    "pages": [
2377                        { "page": 7, "text": "" },
2378                        { "page": 8, "error": "the model timed out", "text": "Scope of" },
2379                        { "page": 9 },
2380                        { "page": 10, "transcriptions": ["Scope of Work", "Scope of Work"] },
2381                        { "page": 11, "error": null, "text": "Clause 4" },
2382                        { "page": 12, "error": 500, "text": "Clause 5" },
2383                        { "page": 13, "text": null },
2384                        { "text": "a page with no number" },
2385                        { "page": "14", "text": "a number that is a string" },
2386                        "page 15"
2387                    ]
2388                }
2389            }))
2390        });
2391
2392        let read = transcribe_pages(&stored(), None, TranscribePagesOptions::default()).unwrap();
2393
2394        let unread = |page: u32, error: &str| PageTranscription::Unread {
2395            page,
2396            error: error.to_string(),
2397        };
2398        let read_as = |page: u32, text: &str| PageTranscription::Read {
2399            page,
2400            text: text.to_string(),
2401        };
2402
2403        assert_eq!(
2404            read.data,
2405            vec![
2406                read_as(7, ""),
2407                unread(8, "the model timed out"),
2408                unread(9, NO_TRANSCRIPTION),
2409                unread(10, NO_TRANSCRIPTION),
2410                read_as(11, "Clause 4"),
2411                read_as(12, "Clause 5"),
2412                unread(13, NO_TRANSCRIPTION),
2413            ]
2414        );
2415        assert_eq!(read.data[1].text(), None, "an error wins over partial text");
2416        assert_eq!(read.data[0].text(), Some(""), "a blank page is a read");
2417        assert_eq!(read.data[2].page(), 9);
2418    }
2419
2420    #[test]
2421    fn an_answer_without_a_list_of_pages_has_no_pages() {
2422        for data in [
2423            json!(null),
2424            json!({}),
2425            json!({ "pages": "none" }),
2426            json!([]),
2427        ] {
2428            let _session =
2429                testing::install(move |_name, _params| Ok(json!({ "data": data.clone() })));
2430
2431            let read =
2432                transcribe_pages(&stored(), None, TranscribePagesOptions::default()).unwrap();
2433
2434            assert!(read.data.is_empty());
2435        }
2436    }
2437
2438    #[test]
2439    fn a_page_transcription_of_anything_but_a_pdf_is_refused_before_the_host_is_asked() {
2440        let session = testing::install(|_name, _params| Ok(json!({ "data": { "pages": [] } })));
2441
2442        let spreadsheet = transcribe_pages(
2443            &DocumentHandle {
2444                filename: "rates.csv".to_string(),
2445                mime_type: "text/csv".to_string(),
2446                ..stored()
2447            },
2448            None,
2449            TranscribePagesOptions::default(),
2450        )
2451        .unwrap_err();
2452
2453        let unstored = transcribe_pages(
2454            &DocumentHandle {
2455                file_hash: String::new(),
2456                ..stored()
2457            },
2458            None,
2459            TranscribePagesOptions::default(),
2460        )
2461        .unwrap_err();
2462
2463        for refusal in [&spreadsheet, &unstored] {
2464            assert_eq!(refusal.code().as_str(), "INVALID_TOOL_INPUT");
2465        }
2466
2467        assert!(spreadsheet.message().contains("reads the pages of a PDF"));
2468        assert!(session.calls().is_empty());
2469
2470        transcribe_pages(
2471            &DocumentHandle {
2472                mime_type: "Application/PDF; version=1.7".to_string(),
2473                ..stored()
2474            },
2475            None,
2476            TranscribePagesOptions::default(),
2477        )
2478        .expect("a PDF type with parameters or in capitals is still a PDF");
2479
2480        assert_eq!(session.calls().len(), 1);
2481    }
2482
2483    #[test]
2484    fn an_enrolment_answers_with_the_id_the_face_was_filed_under() {
2485        let session = testing::install(|name, params| {
2486            assert_eq!(name, FACE_ENROLL);
2487            assert_eq!(params["subject_id"], json!("EMP-42"));
2488            assert_eq!(params["image"], json!("iVBORw0KGgo"));
2489
2490            Ok(json!({ "face_id": "FACE-1" }))
2491        });
2492
2493        assert_eq!(
2494            enroll_face("EMP-42", json!("iVBORw0KGgo")).unwrap(),
2495            "FACE-1"
2496        );
2497        assert_eq!(session.calls().len(), 1);
2498    }
2499
2500    #[test]
2501    fn an_enrolment_uploads_a_pending_image_first() {
2502        let session = testing::install(|name, _params| {
2503            if name == UPLOAD_EPHEMERAL {
2504                return Ok(json!({ "file_hash": "abc", "storage_path": "ephemeral/abc" }));
2505            }
2506
2507            Ok(json!({ "face_id": "FACE-1" }))
2508        });
2509
2510        enroll_face("EMP-42", json!({ "file_hash": "abc", "pending": true })).unwrap();
2511
2512        let calls = session.calls();
2513
2514        assert_eq!(calls[0].name, UPLOAD_EPHEMERAL);
2515        assert_eq!(
2516            calls[1].params["image"],
2517            json!({ "file_hash": "abc", "storage_path": "ephemeral/abc" })
2518        );
2519    }
2520
2521    #[test]
2522    fn an_enrolment_without_a_subject_or_an_image_is_refused() {
2523        let session = testing::install(|_name, _params| Ok(json!({ "face_id": "FACE-1" })));
2524
2525        assert_eq!(
2526            enroll_face("  ", json!("image"))
2527                .unwrap_err()
2528                .code()
2529                .as_str(),
2530            "INVALID_TOOL_INPUT"
2531        );
2532        assert_eq!(
2533            enroll_face("EMP-42", json!(null))
2534                .unwrap_err()
2535                .code()
2536                .as_str(),
2537            "INVALID_TOOL_INPUT"
2538        );
2539        assert!(session.calls().is_empty());
2540    }
2541
2542    #[test]
2543    fn an_enrolment_that_answers_without_an_id_is_reported_rather_than_guessed_at() {
2544        let _session = testing::install(|_name, _params| Ok(json!({ "enrolled": true })));
2545
2546        let error = enroll_face("EMP-42", json!("image")).unwrap_err();
2547
2548        assert!(matches!(error, Error::Json(_)));
2549        assert_eq!(error.code().as_str(), "ACTION_FAILED");
2550    }
2551
2552    #[test]
2553    fn a_search_carries_only_the_narrowing_it_was_given() {
2554        let session = testing::install(|name, params| {
2555            assert_eq!(name, FACE_SEARCH);
2556
2557            Ok(json!({ "matches": params["options"].clone() }))
2558        });
2559
2560        let wide: Value = search_face(json!("image"), FaceSearch::default()).unwrap();
2561
2562        let narrow: Value = search_face(
2563            json!("image"),
2564            FaceSearch {
2565                max_faces: Some(3),
2566                similarity_threshold: Some(92.5),
2567            },
2568        )
2569        .unwrap();
2570
2571        assert_eq!(wide["matches"], json!({}), "the platform's own defaults");
2572        assert_eq!(
2573            narrow["matches"],
2574            json!({ "max_faces": 3, "similarity_threshold": 92.5 })
2575        );
2576        assert_eq!(session.calls().len(), 2);
2577    }
2578
2579    #[test]
2580    fn a_search_answers_with_the_type_the_call_asked_for() {
2581        #[derive(Debug, Deserialize, PartialEq)]
2582        struct Match {
2583            similarity: f64,
2584            subject_id: String,
2585        }
2586
2587        let _session = testing::install(|_name, _params| {
2588            Ok(json!([{ "similarity": 99.1, "subject_id": "EMP-42" }]))
2589        });
2590
2591        let found: Vec<Match> = search_face(json!("image"), FaceSearch::default()).unwrap();
2592
2593        assert_eq!(
2594            found,
2595            vec![Match {
2596                similarity: 99.1,
2597                subject_id: "EMP-42".to_string()
2598            }]
2599        );
2600    }
2601
2602    #[test]
2603    fn a_search_with_no_image_is_refused_before_anything_is_sent() {
2604        let session = testing::install(|_name, _params| Ok(json!([])));
2605
2606        let error = search_face::<Value>(json!(null), FaceSearch::default()).unwrap_err();
2607
2608        assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
2609        assert!(session.calls().is_empty());
2610    }
2611
2612    #[test]
2613    fn a_removal_reports_the_ids_the_platform_says_it_removed() {
2614        let session = testing::install(|name, params| {
2615            assert_eq!(name, FACE_DELETE);
2616            assert_eq!(params["face_ids"], json!(["FACE-1", "FACE-2"]));
2617
2618            Ok(json!({ "deleted": ["FACE-1"] }))
2619        });
2620
2621        let removed = delete_face(&["FACE-1".to_string(), "FACE-2".to_string()]).unwrap();
2622
2623        assert_eq!(removed, ["FACE-1"]);
2624        assert_eq!(session.calls().len(), 1);
2625    }
2626
2627    #[test]
2628    fn a_removal_reads_a_bare_list_of_ids_as_well() {
2629        let _session = testing::install(|_name, _params| Ok(json!(["FACE-1", "FACE-2"])));
2630
2631        assert_eq!(
2632            delete_face(&["FACE-1".to_string()]).unwrap(),
2633            ["FACE-1", "FACE-2"]
2634        );
2635    }
2636
2637    #[test]
2638    fn a_removal_that_names_nothing_removed_answers_with_nothing() {
2639        let _session = testing::install(|_name, _params| Ok(json!({ "ok": true })));
2640
2641        assert!(delete_face(&["FACE-1".to_string()]).unwrap().is_empty());
2642    }
2643
2644    #[test]
2645    fn a_removal_with_no_ids_is_refused_before_anything_is_sent() {
2646        let session = testing::install(|_name, _params| Ok(json!([])));
2647
2648        let error = delete_face(&[]).unwrap_err();
2649
2650        assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
2651        assert!(session.calls().is_empty());
2652    }
2653
2654    #[test]
2655    fn what_the_host_said_survives_into_every_failure() {
2656        let _session = testing::install(|_name, _params| Err(Error::failed("the engine is busy")));
2657
2658        let operation = summarize(json!("text"), "One sentence.", Options::default()).unwrap_err();
2659        let enrolment = enroll_face("EMP-42", json!("image")).unwrap_err();
2660        let search = search_face::<Value>(json!("image"), FaceSearch::default()).unwrap_err();
2661        let removal = delete_face(&["FACE-1".to_string()]).unwrap_err();
2662
2663        for failure in [&operation, &enrolment, &search, &removal] {
2664            assert!(failure.message().contains("the engine is busy"));
2665            assert!(!failure.fault().hint().is_empty(), "and says what to do");
2666        }
2667    }
2668
2669    #[test]
2670    fn an_answer_that_does_not_fit_the_type_is_reported_as_having_run() {
2671        let _session = testing::install(|_name, _params| Ok(json!({ "data": { "total": 240.0 } })));
2672
2673        let error = summarize(json!("text"), "One sentence.", Options::default()).unwrap_err();
2674
2675        assert!(matches!(error, Error::Json(_)));
2676        assert_eq!(error.code().as_str(), "ACTION_FAILED");
2677        assert!(error
2678            .fault()
2679            .hint()
2680            .contains("rather than running it again"));
2681    }
2682
2683    #[test]
2684    fn a_timeout_travels_as_the_milliseconds_the_wire_carries() {
2685        assert_eq!(milliseconds(Duration::from_secs(30)), 30_000);
2686        assert_eq!(milliseconds(Duration::from_millis(1)), 1);
2687        assert_eq!(milliseconds(Duration::MAX), u64::MAX);
2688    }
2689
2690    #[test]
2691    fn every_model_names_itself_as_the_wire_spells_it() {
2692        assert_eq!(Model::Lite.to_string(), "lite");
2693        assert_eq!(Model::Medium.to_string(), "medium");
2694        assert_eq!(Model::Large.to_string(), "large");
2695        assert_eq!(Model::Xl.to_string(), "xl");
2696    }
2697}