Skip to main content

simpleplatform_sdk/
ai.rs

1//! Asking the AI engine for an answer.
2//!
3//! Four operations read something and answer with it — [`extract`] pulls
4//! structured data out against a schema you give, [`summarize`] writes prose,
5//! [`transcribe`] turns audio or video into words, and [`transcribe_pages`]
6//! reads the pages of a PDF that have no text of their own from their images.
7//! Three more hold the tenant's face collection: [`enroll_face`],
8//! [`search_face`] and [`delete_face`].
9//!
10//! ```
11//! # use simpleplatform_sdk::prelude::*;
12//! # use simpleplatform_sdk::ai::{Execution, Options};
13//! # use simpleplatform_sdk::testing;
14//! #[derive(Deserialize)]
15//! struct Invoice {
16//!     number: String,
17//!     total: f64,
18//! }
19//!
20//! # let _session = testing::install(|_name, _params| {
21//! #     Ok(json!({
22//! #         "data": { "number": "INV-8", "total": 240.0 },
23//! #         "metadata": { "input_tokens": 812, "output_tokens": 24 }
24//! #     }))
25//! # });
26//! let schema = json!({
27//!     "type": "object",
28//!     "properties": {
29//!         "number": { "type": "string" },
30//!         "total": { "type": "number" }
31//!     },
32//!     "required": ["number", "total"]
33//! });
34//!
35//! let read: Execution<Invoice> = simple::ai::extract(
36//!     json!({ "file_hash": "9f2c…", "mime_type": "application/pdf" }),
37//!     "Read the invoice number and the total.",
38//!     schema,
39//!     Options::default(),
40//! )?;
41//!
42//! assert_eq!(read.data.number, "INV-8");
43//! assert_eq!(read.metadata.input_tokens, 812);
44//! # Ok::<(), Error>(())
45//! ```
46//!
47//! # The shape, and why it is this one
48//!
49//! **What an operation must have is an argument; what it may have is a field.**
50//! An input, a prompt and a schema are the operation — leave one out and there
51//! is nothing to run — so they are arguments, and the type system asks for them.
52//! Everything else has a working default, so it lives in [`Options`], which
53//! derives one:
54//!
55//! ```
56//! # use simpleplatform_sdk::ai::{Model, Options};
57//! # use std::time::Duration;
58//! let options = Options {
59//!     model: Some(Model::Large),
60//!     timeout: Some(Duration::from_secs(90)),
61//!     ..Default::default()
62//! };
63//! ```
64//!
65//! Nobody passes six `None`s, and a member added to [`Options`] later does not
66//! disturb a call that was written with `..Default::default()`.
67//!
68//! **The answer type is yours.** [`extract`] is generic over what it decodes
69//! into, exactly as [`crate::graphql::query`] is, so an extraction lands in your
70//! own struct rather than in a `Value` you then have to pick apart. Where the
71//! answer's shape is settled it is named for you instead: [`summarize`] answers
72//! with prose, and [`transcribe`] answers with a [`Transcript`], because this
73//! module wrote the schema that produced it.
74//!
75//! **Every answer carries what it cost.** An operation answers with an
76//! [`Execution`] — the data and the [`Metadata`] beside it — so token counts are
77//! there to record without a second call.
78//!
79//! # Asking for part of a file, or for its text
80//!
81//! A file reference may say how the file should travel and which of its pages
82//! should. [`DocumentInput`] is that reference with those two beside it, and
83//! [`Delivery`] and [`Pages`] are what they say. A plain handle asks for nothing
84//! and sends the whole file the way its type is sent by default.
85//!
86//! What was asked for is not always what happened, so every answer says how
87//! each file travelled, in [`Metadata::delivery`]: as the document, as its text
88//! or as an image, which pages were read from their images, and, when text was
89//! asked for and the document went instead, which pages could not be read.
90//!
91//! # Files that have not been uploaded yet
92//!
93//! A document handle that is still pending is uploaded to ephemeral storage
94//! before the operation runs, and the operation is given the handle that upload
95//! answered with. A handle that is already stored is passed straight through,
96//! and so is a list of handles, item by item. Nothing in a call site changes:
97//! pass the handle you were given.
98
99use std::fmt;
100use std::time::Duration;
101
102use serde::de::DeserializeOwned;
103use serde::{Deserialize, Serialize};
104use serde_json::{json, Map, Value};
105
106use crate::codes::Code;
107use crate::error::{Error, Fault};
108use crate::host;
109use crate::storage::DocumentHandle;
110
111/// The primitive that runs an AI operation.
112const ORCHESTRATOR: &str = "logic:dev.simple.system/ai-orchestrator";
113
114/// The host action that puts a pending file into ephemeral storage.
115const UPLOAD_EPHEMERAL: &str = "action:documents/upload-ephemeral";
116
117/// The host action that adds a face to the tenant's collection.
118const FACE_ENROLL: &str = "action:ai/face/enroll";
119
120/// The host action that searches the tenant's collection.
121const FACE_SEARCH: &str = "action:ai/face/search";
122
123/// The host action that removes faces from the tenant's collection.
124const FACE_DELETE: &str = "action:ai/face/delete";
125
126/// The size of model an operation runs on.
127///
128/// It is `#[non_exhaustive]`: a `match` on it needs a `_` arm, so a size added
129/// later does not break an action that was already written.
130#[non_exhaustive]
131#[derive(Clone, Copy, Debug, PartialEq, Eq)]
132pub enum Model {
133    /// The quickest and cheapest.
134    Lite,
135    /// The middle of the range.
136    Medium,
137    /// The one to reach for when the answer has to be right.
138    Large,
139    /// The largest available.
140    Xl,
141}
142
143impl Model {
144    /// The size exactly as it travels on the wire.
145    pub fn as_str(&self) -> &str {
146        match self {
147            Model::Lite => "lite",
148            Model::Medium => "medium",
149            Model::Large => "large",
150            Model::Xl => "xl",
151        }
152    }
153}
154
155impl fmt::Display for Model {
156    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
157        formatter.write_str(self.as_str())
158    }
159}
160
161/// How a document reaches the model.
162///
163/// Both answers are sayable, so a call can write the default down rather than
164/// leaving it to the absence of a value.
165///
166/// It is `#[non_exhaustive]`: a `match` on it needs a `_` arm, so a way added
167/// later does not break an action that was already written.
168#[non_exhaustive]
169#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
170#[serde(rename_all = "lowercase")]
171pub enum Delivery {
172    /// The document itself, which is how a PDF travels unless you say
173    /// otherwise.
174    Document,
175    /// The text the platform reads out of the document.
176    Text,
177}
178
179/// The pages of a PDF to use, counted from 1 with both ends included.
180///
181/// The two ends are one value here because the engine refuses one without the
182/// other. A half-written range cannot be expressed, so it cannot be sent.
183///
184/// What this type does not decide is still the engine's: a `first` below 1, a
185/// `last` before `first`, or a range that runs past the end of the document is
186/// refused there, before the file is read or anything is spent.
187#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
188pub struct Pages {
189    /// The first page to use, counted from 1.
190    pub first_page: u32,
191    /// The last page to use, included.
192    pub last_page: u32,
193}
194
195impl Pages {
196    /// Pages `first` to `last`, counted from 1 with both ends included.
197    pub fn new(first: u32, last: u32) -> Pages {
198        Pages {
199            first_page: first,
200            last_page: last,
201        }
202    }
203}
204
205/// A stored file handed to an AI operation, together with what should be done
206/// with it.
207///
208/// A PDF travels to the model as a PDF, because the model reads one: its
209/// tables, its layout and its figures survive the trip, and none of them
210/// survive being turned into text. Asking for text is therefore opt-in.
211///
212/// ```
213/// # use simpleplatform_sdk::prelude::*;
214/// # use simpleplatform_sdk::ai::{self, Delivery, DocumentInput, Execution, Pages};
215/// # use simpleplatform_sdk::testing;
216/// # let _session = testing::install(|_name, _params| {
217/// #     Ok(json!({ "data": "The first three pages sign the contract off." }))
218/// # });
219/// let contract = DocumentInput {
220///     file: DocumentHandle {
221///         file_hash: "9f2c…".into(),
222///         filename: "contract.pdf".into(),
223///         mime_type: "application/pdf".into(),
224///         size: 1_048_576,
225///         storage_path: "acme/documents/9f/2c/9f2c….pdf".into(),
226///     },
227///     pages: Some(Pages::new(1, 3)),
228///     deliver_as: Some(Delivery::Text),
229/// };
230///
231/// let read: Execution<String> =
232///     ai::summarize(json!(contract), "Say what it commits us to.", Options::default())?;
233/// # Ok::<(), Error>(())
234/// ```
235///
236/// The pages and the delivery are two questions, asked in that order. The pages
237/// say WHICH DOCUMENT the call is about: they are taken out of the PDF first,
238/// and everything after that is about them and nothing else. The delivery then
239/// says how that document should reach the model.
240///
241/// Refused by the engine, before the file is read or anything is spent:
242///
243/// * [`pages`](DocumentInput::pages) on anything that is not a PDF;
244/// * a range that runs past the end of the document, or one whose last page
245///   precedes its first;
246/// * [`Delivery::Text`] on an image, which is not read as text;
247/// * [`Delivery::Document`] on a Word, Excel, PowerPoint, CSV, RTF or text
248///   file, which only ever travels as text.
249///
250/// A page with no usable text of its own — a scan, or text that reads as noise
251/// — is read from its image instead, and that text stands in its place, marked
252/// `[Transcribed from the page image.]` under its page label. Only if that
253/// reading fails are the pages asked for sent as a PDF, so an answer is never
254/// built on text with a hole in it. The answer's [`Metadata::delivery`] says
255/// which happened.
256///
257/// Pages asked for as text arrive numbered from 1, because by then they are a
258/// document of their own. A line above them says which pages of which document
259/// they were, rather than the numbers being rewritten inside the text: a page
260/// number printed in a header or a cross-reference cannot be told apart from a
261/// page label, so editing them would corrupt the document's own words.
262#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
263pub struct DocumentInput {
264    /// The stored file, as the upload or the record that holds it answered
265    /// with.
266    #[serde(flatten)]
267    pub file: DocumentHandle,
268
269    /// Which pages of the PDF to send. Unset sends the whole document.
270    #[serde(default, flatten)]
271    pub pages: Option<Pages>,
272
273    /// How the document reaches the model. Unset sends it the way its type is
274    /// sent by default.
275    ///
276    #[serde(default, skip_serializing_if = "Option::is_none")]
277    pub deliver_as: Option<Delivery>,
278}
279
280/// What any operation may be told, beyond the inputs that define it.
281///
282/// Every member has a default, so a call that wants one thing changed says that
283/// thing and nothing else:
284///
285/// ```
286/// # use simpleplatform_sdk::ai::{Model, Options};
287/// let options = Options {
288///     model: Some(Model::Large),
289///     reasoning: false,
290///     ..Default::default()
291/// };
292/// ```
293#[derive(Clone, Debug)]
294pub struct Options {
295    /// Which size of model to run on. Unset leaves the choice to the platform.
296    pub model: Option<Model>,
297
298    /// The role, or the standing instruction, the model holds for the whole
299    /// task. Unset sends none.
300    pub system_prompt: Option<String>,
301
302    /// How far the model may range, from `0.0` for the most settled answer to
303    /// `1.0` for the most inventive. Unset leaves the model's own default.
304    pub temperature: Option<f64>,
305
306    /// Whether the model works the answer out step by step, and reports how.
307    /// On unless you turn it off.
308    pub reasoning: bool,
309
310    /// The most tokens the reasoning may spend. Unset leaves the platform's
311    /// default, and it is read only while `reasoning` is on.
312    pub reasoning_budget: Option<u32>,
313
314    /// Whether to do the work again rather than answer from a cached result for
315    /// the same input, prompt, schema, model and options.
316    pub regenerate: bool,
317
318    /// How long to wait for the answer. Unset leaves the platform's default.
319    pub timeout: Option<Duration>,
320}
321
322impl Default for Options {
323    /// Reasoning on, and every other choice left to the platform.
324    fn default() -> Options {
325        Options {
326            model: None,
327            system_prompt: None,
328            temperature: None,
329            reasoning: true,
330            reasoning_budget: None,
331            regenerate: false,
332            timeout: None,
333        }
334    }
335}
336
337/// Whether to tell the speakers apart, and what to call them.
338#[derive(Clone, Debug, Default, PartialEq, Eq)]
339pub enum Participants {
340    /// Do not tell the speakers apart.
341    #[default]
342    Ignore,
343
344    /// Tell the speakers apart, and label them `Participant 1`,
345    /// `Participant 2`, and so on.
346    Detect,
347
348    /// Tell the speakers apart under the names given. Naming nobody asks for
349    /// the same thing as [`Participants::Detect`].
350    Named(Vec<String>),
351}
352
353impl Participants {
354    /// Whether the speakers are to be told apart at all.
355    fn wanted(&self) -> bool {
356        !matches!(self, Participants::Ignore)
357    }
358
359    /// The names to identify, as one list, or nothing when none were given.
360    fn names(&self) -> Option<String> {
361        match self {
362            Participants::Named(names) if !names.is_empty() => Some(names.join(", ")),
363            _ => None,
364        }
365    }
366}
367
368/// What a transcription is to produce.
369///
370/// At least one of `include_transcript` and `summarize` has to be set, since
371/// between them they are the whole of what a transcription answers with.
372#[derive(Clone, Debug, Default)]
373pub struct TranscribeOptions {
374    /// Ask for the words that were said.
375    pub include_transcript: bool,
376
377    /// Ask for `[MM:SS]` stamps through the transcript. Read only while
378    /// `include_transcript` is set.
379    pub include_timestamps: bool,
380
381    /// Ask for a summary of what was said.
382    pub summarize: bool,
383
384    /// Whether to tell the speakers apart, and what to call them.
385    pub participants: Participants,
386}
387
388/// What a transcription answered with.
389///
390/// A member that was not asked for is empty, and every member survives an
391/// answer that leaves it out.
392#[derive(Clone, Debug, Default, Deserialize)]
393#[serde(default)]
394pub struct Transcript {
395    /// The language that was heard, as an ISO 639-1 code such as `en` or `es`.
396    pub language: String,
397
398    /// The words that were said, when they were asked for.
399    pub transcript: Option<String>,
400
401    /// The summary, when it was asked for.
402    pub summary: Option<String>,
403
404    /// The speakers that were told apart, when they were asked for.
405    pub participants: Vec<String>,
406}
407
408/// What [`transcribe_pages`] may be told, beyond the PDF and the pages.
409///
410/// There is no prompt, model or reasoning switch, which is why this is not
411/// [`Options`]: the instructions are the platform's, and every page is read at
412/// the platform's own accuracy-first effort.
413#[derive(Clone, Debug, Default)]
414pub struct TranscribePagesOptions {
415    /// Whether to run the operation again rather than answer from its kept
416    /// result. A page already read is still served from its kept read, which
417    /// is tied to this version of the file and to the platform's instructions.
418    pub regenerate: bool,
419
420    /// How long to wait for the answer. Unset leaves the platform's default.
421    pub timeout: Option<Duration>,
422}
423
424/// One page's transcription, or why it has none.
425///
426/// Pages are numbered in the original document, whatever range was asked for.
427/// A page either has its text or has a reason, never both, so a caller cannot
428/// mistake a partial read for the page.
429#[derive(Clone, Debug, PartialEq, Eq)]
430pub enum PageTranscription {
431    /// The page was read from its image.
432    Read {
433        /// The page's number in the original document.
434        page: u32,
435
436        /// The page's text: verbatim markdown, in reading order, tables as
437        /// markdown tables, `[illegible]` where a word could not be read. An
438        /// empty string is a read that found the page blank.
439        text: String,
440    },
441
442    /// The page could not be read.
443    Unread {
444        /// The page's number in the original document.
445        page: u32,
446
447        /// Why, in the platform's words. No partial text is given.
448        error: String,
449    },
450}
451
452impl PageTranscription {
453    /// The page's number in the original document, read or not.
454    pub fn page(&self) -> u32 {
455        match self {
456            PageTranscription::Read { page, .. } | PageTranscription::Unread { page, .. } => *page,
457        }
458    }
459
460    /// The page's text, when it was read.
461    pub fn text(&self) -> Option<&str> {
462        match self {
463            PageTranscription::Read { text, .. } => Some(text),
464            PageTranscription::Unread { .. } => None,
465        }
466    }
467}
468
469/// What an operation answered, and what it cost.
470#[derive(Clone, Debug)]
471pub struct Execution<T> {
472    /// The answer itself, decoded into the type the call asked for.
473    pub data: T,
474
475    /// What the run spent, and how it reasoned.
476    pub metadata: Metadata,
477}
478
479/// What a run spent, how it reasoned, and how its files reached the model.
480#[derive(Clone, Debug, Default, PartialEq, Eq)]
481pub struct Metadata {
482    /// Tokens in what was sent.
483    pub input_tokens: u64,
484
485    /// Tokens in what came back.
486    pub output_tokens: u64,
487
488    /// Tokens spent reasoning, when the run reported them.
489    pub reasoning_tokens: Option<u64>,
490
491    /// The reasoning itself, when the run reported it.
492    pub reasoning: Option<String>,
493
494    /// How each file the run carried reached the model, in the order the files
495    /// were given. Empty when the run carried no file, and on an answer kept
496    /// from before the platform reported it.
497    pub delivery: Vec<FileDelivery>,
498}
499
500/// How one file an operation carried reached the model.
501///
502/// A file asked for as text is not always sent as text. A page with no text of
503/// its own is read from its image, and when that reading fails the document is
504/// sent instead, so the answer is never built on text with a hole in it. This
505/// is how a caller finds out which happened, without reading a platform log.
506///
507/// Page numbers are the original document's, whatever range was cut from it,
508/// so a page named here is the page a person would turn to.
509///
510/// ```
511/// # use simpleplatform_sdk::prelude::*;
512/// # use simpleplatform_sdk::ai::{DeliveredAs, Options, Pages};
513/// # use simpleplatform_sdk::testing;
514/// # let _session = testing::install(|_name, _params| {
515/// #     Ok(json!({
516/// #         "data": "It commits us to a fixed price.",
517/// #         "metadata": { "delivery": [{
518/// #             "filename": "contract.pdf",
519/// #             "delivered_as": "document",
520/// #             "first_page": 40,
521/// #             "last_page": 52,
522/// #             "transcribed_pages": [],
523/// #             "fallback": {
524/// #                 "reason": "transcription_failed",
525/// #                 "pages": [44],
526/// #                 "message": "page 44: the page image could not be read"
527/// #             }
528/// #         }] }
529/// #     }))
530/// # });
531/// let read = simple::ai::summarize(
532///     json!({
533///         "file_hash": "9f2c…",
534///         "filename": "contract.pdf",
535///         "mime_type": "application/pdf",
536///         "first_page": 40,
537///         "last_page": 52,
538///         "deliver_as": "text"
539///     }),
540///     "Say what it commits us to.",
541///     Options::default(),
542/// )?;
543///
544/// for file in &read.metadata.delivery {
545///     if let Some(fallback) = &file.fallback {
546///         println!("{} was read as a PDF: {}", file.filename, fallback.message);
547///     }
548/// }
549///
550/// let contract = &read.metadata.delivery[0];
551///
552/// assert_eq!(contract.delivered_as, DeliveredAs::Document);
553/// assert_eq!(contract.pages, Some(Pages::new(40, 52)));
554/// # Ok::<(), Error>(())
555/// ```
556#[derive(Clone, Debug, PartialEq, Eq)]
557pub struct FileDelivery {
558    /// The file's name.
559    pub filename: String,
560
561    /// How the file travelled.
562    pub delivered_as: DeliveredAs,
563
564    /// The range the file was cut to, when one was asked for. Unset when the
565    /// whole file travelled.
566    pub pages: Option<Pages>,
567
568    /// The pages whose text was read from their images rather than from their
569    /// text layer. Empty when none were.
570    pub transcribed_pages: Vec<u32>,
571
572    /// Set only when text was asked for and the document travelled instead:
573    /// which pages could not be read, and why.
574    pub fallback: Option<DeliveryFallback>,
575}
576
577/// How a file reached the model.
578///
579/// This is what happened, where [`Delivery`] is what was asked for, and it has
580/// a way that cannot be asked for: an image only ever travels as one.
581///
582/// It is `#[non_exhaustive]`: a `match` on it needs a `_` arm, so a way added
583/// later does not break an action that was already written.
584#[non_exhaustive]
585#[derive(Clone, Copy, Debug, PartialEq, Eq)]
586pub enum DeliveredAs {
587    /// The document itself, which the model reads with its layout, tables and
588    /// figures.
589    Document,
590    /// The text read out of the document, with any page that had none read from
591    /// its image.
592    Text,
593    /// An image.
594    Image,
595}
596
597/// Why a file asked for as text travelled as the document instead.
598#[derive(Clone, Debug, Default, PartialEq, Eq)]
599pub struct DeliveryFallback {
600    /// Why, as the platform names it: `transcription_failed` when pages with no
601    /// text of their own could not be read from their images. It is kept as
602    /// the platform's word rather than narrowed to a list, so a reason added
603    /// later still reaches you instead of hiding that a fallback happened.
604    pub reason: String,
605
606    /// The pages that could not be read, numbered in the original document.
607    pub pages: Vec<u32>,
608
609    /// What went wrong, page by page, in the platform's words.
610    pub message: String,
611}
612
613/// How far a face search reaches.
614#[derive(Clone, Debug, Default)]
615pub struct FaceSearch {
616    /// The most matches to answer with. Unset leaves the platform's default.
617    pub max_faces: Option<u32>,
618
619    /// The similarity a match has to reach, as a percentage from `0.0` to
620    /// `100.0`. Unset leaves the platform's default.
621    pub similarity_threshold: Option<f64>,
622}
623
624/// Read structured data out of an input, against a schema you give.
625///
626/// The schema is the contract the answer is held to, and the type parameter is
627/// what that answer is decoded into — so give the model the schema of the struct
628/// you want back.
629///
630/// ```
631/// # use simpleplatform_sdk::prelude::*;
632/// # use simpleplatform_sdk::ai::{Execution, Options};
633/// # use simpleplatform_sdk::testing;
634/// #[derive(Deserialize)]
635/// struct Sentiment {
636///     score: f64,
637/// }
638///
639/// # let _session = testing::install(|_name, _params| {
640/// #     Ok(json!({ "data": { "score": 0.8 }, "metadata": { "output_tokens": 6 } }))
641/// # });
642/// let schema = json!({
643///     "type": "object",
644///     "properties": { "score": { "type": "number" } },
645///     "required": ["score"]
646/// });
647///
648/// let read: Execution<Sentiment> = simple::ai::extract(
649///     json!("The delivery arrived early and intact."),
650///     "Score the sentiment from -1.0 to 1.0.",
651///     schema,
652///     Options::default(),
653/// )?;
654///
655/// assert_eq!(read.data.score, 0.8);
656/// # Ok::<(), Error>(())
657/// ```
658pub fn extract<T: DeserializeOwned>(
659    input: Value,
660    prompt: &str,
661    schema: Value,
662    options: Options,
663) -> Result<Execution<T>, Error> {
664    refuse_empty(
665        &input,
666        "extract needs something to read.",
667        "Pass the text, the document handle, or the object the extraction is to read.",
668    )?;
669    refuse_empty_prompt(prompt, "extract")?;
670
671    if !schema.is_object() {
672        return Err(
673            Error::invalid("extract needs a JSON Schema object describing the answer.").hint(
674                "Pass a schema object naming the type and the properties the answer is to have.",
675            ),
676        );
677    }
678
679    run("extract", input, Some(prompt), Some(schema), &options)
680}
681
682/// Write prose about an input.
683///
684/// ```
685/// # use simpleplatform_sdk::prelude::*;
686/// # use simpleplatform_sdk::ai::Options;
687/// # use simpleplatform_sdk::testing;
688/// # let _session = testing::install(|_name, _params| {
689/// #     Ok(json!({ "data": "A refund was agreed.", "metadata": { "output_tokens": 5 } }))
690/// # });
691/// let written = simple::ai::summarize(
692///     json!({ "file_hash": "3a91…", "mime_type": "application/pdf" }),
693///     "Summarise what was agreed, in one sentence.",
694///     Options::default(),
695/// )?;
696///
697/// assert_eq!(written.data, "A refund was agreed.");
698/// # Ok::<(), Error>(())
699/// ```
700pub fn summarize(input: Value, prompt: &str, options: Options) -> Result<Execution<String>, Error> {
701    refuse_empty(
702        &input,
703        "summarize needs something to read.",
704        "Pass the text, the document handle, or the object the summary is to be written from.",
705    )?;
706    refuse_empty_prompt(prompt, "summarize")?;
707
708    run("summarize", input, Some(prompt), None, &options)
709}
710
711/// Turn audio or video into words.
712///
713/// `document` is the handle of an uploaded audio or video file. The prompt and
714/// the schema are written here, from what `wanted` asks for, which is why the
715/// answer is a [`Transcript`] rather than a type you have to name.
716///
717/// ```
718/// # use simpleplatform_sdk::prelude::*;
719/// # use simpleplatform_sdk::ai::{Options, Participants, TranscribeOptions};
720/// # use simpleplatform_sdk::testing;
721/// # let _session = testing::install(|_name, _params| {
722/// #     Ok(json!({
723/// #         "data": {
724/// #             "language": "en",
725/// #             "transcript": "Customer: My order is late.\nAgent: I have refunded it.",
726/// #             "participants": ["Customer", "Agent"]
727/// #         },
728/// #         "metadata": { "input_tokens": 4_200, "output_tokens": 96 }
729/// #     }))
730/// # });
731/// let heard = simple::ai::transcribe(
732///     json!({ "file_hash": "c07b…", "mime_type": "audio/mpeg" }),
733///     TranscribeOptions {
734///         include_transcript: true,
735///         participants: Participants::Named(vec!["Customer".into(), "Agent".into()]),
736///         ..Default::default()
737///     },
738///     Options::default(),
739/// )?;
740///
741/// assert_eq!(heard.data.language, "en");
742/// assert_eq!(heard.data.participants, ["Customer", "Agent"]);
743/// # Ok::<(), Error>(())
744/// ```
745pub fn transcribe(
746    document: Value,
747    wanted: TranscribeOptions,
748    options: Options,
749) -> Result<Execution<Transcript>, Error> {
750    refuse_unless_media(&document)?;
751
752    if !wanted.include_transcript && !wanted.summarize {
753        return Err(
754            Error::invalid("transcribe needs to be told what to produce.")
755                .hint("Set include_transcript, summarize, or both."),
756        );
757    }
758
759    let prompt = transcribe_prompt(&wanted);
760
761    run(
762        "extract",
763        document,
764        Some(&prompt),
765        Some(transcribe_schema(&wanted)),
766        &options,
767    )
768}
769
770/// Read the pages of a PDF that have no usable text of their own — scans,
771/// image-only exhibits, text that reads as noise — from their images.
772///
773/// Every such page is read once, and it is the same transcription an
774/// [`extract`] or [`summarize`] that asked for [`Delivery::Text`] is given in
775/// the page's place, so the text a caller holds for an image page is exactly the
776/// text the answer was built on. Pages the platform reads as text are neither
777/// read nor returned.
778///
779/// A page is not transcribed a second time to check the first: that would be
780/// the same model reading the same image again, doubling the cost of every
781/// scanned page without being independent of the first read. An independent
782/// check reads the pages a second way, as the document itself
783/// ([`Delivery::Document`]), and compares the answers.
784///
785/// Transcriptions are kept per version of the file and page, and shared with
786/// every other operation, so a page already read for an [`extract`] or
787/// [`summarize`] that asked for text is not read again.
788///
789/// `pages` narrows the reading to a range, counted from 1 with both ends
790/// included, and the answer still numbers pages in the original document. The
791/// file is taken as a [`DocumentHandle`] rather than a [`DocumentInput`]
792/// because the pages are answered as text, so a delivery has no meaning here
793/// and cannot be sent. Anything that is not a PDF is refused before the host
794/// is asked.
795///
796/// ```
797/// # use simpleplatform_sdk::prelude::*;
798/// # use simpleplatform_sdk::ai::{PageTranscription, Pages, TranscribePagesOptions};
799/// # use simpleplatform_sdk::testing;
800/// # let _session = testing::install(|_name, _params| {
801/// #     Ok(json!({
802/// #         "data": { "pages": [
803/// #             { "page": 41, "text": "## Scope of Work\n\nThe Contractor shall furnish…" },
804/// #             { "page": 44, "error": "the page image could not be read" }
805/// #         ] },
806/// #         "metadata": { "input_tokens": 0, "output_tokens": 0 }
807/// #     }))
808/// # });
809/// # let contract = DocumentHandle {
810/// #     file_hash: "9f2c…".into(),
811/// #     filename: "contract.pdf".into(),
812/// #     mime_type: "application/pdf".into(),
813/// #     size: 1_048_576,
814/// #     storage_path: "acme/documents/9f/2c/9f2c….pdf".into(),
815/// # };
816/// let read = simple::ai::transcribe_pages(
817///     &contract,
818///     Some(Pages::new(40, 52)),
819///     TranscribePagesOptions::default(),
820/// )?;
821///
822/// for page in &read.data {
823///     match page {
824///         PageTranscription::Read { page, text } => {
825///             println!("page {page}: {text}");
826///         }
827///         PageTranscription::Unread { page, error } => {
828///             println!("page {page} could not be read: {error}");
829///         }
830///     }
831/// }
832///
833/// assert_eq!(
834///     read.data[0].text(),
835///     Some("## Scope of Work\n\nThe Contractor shall furnish…")
836/// );
837/// assert_eq!(read.data[1].page(), 44);
838/// # Ok::<(), Error>(())
839/// ```
840pub fn transcribe_pages(
841    document: &DocumentHandle,
842    pages: Option<Pages>,
843    options: TranscribePagesOptions,
844) -> Result<Execution<Vec<PageTranscription>>, Error> {
845    refuse_unless_pdf(document)?;
846
847    let input = json!(DocumentInput {
848        file: document.clone(),
849        pages,
850        deliver_as: None,
851    });
852
853    // The same universal options every other caller of the operation sends,
854    // since the whole map is part of the key a kept result is found under.
855    let options = Options {
856        regenerate: options.regenerate,
857        timeout: options.timeout,
858        ..Options::default()
859    };
860
861    let read: Execution<Value> = run("transcribe", input, None, None, &options)?;
862
863    Ok(Execution {
864        data: page_transcriptions(&read.data),
865        metadata: read.metadata,
866    })
867}
868
869/// Add a face to the tenant's collection, under the subject it belongs to.
870///
871/// `image` is either a base64 image or a document handle. The answer is the id
872/// of the enrolled face, which is what [`delete_face`] takes.
873///
874/// ```
875/// # use simpleplatform_sdk::prelude::*;
876/// # use simpleplatform_sdk::testing;
877/// # let _session = testing::install(|_name, _params| Ok(json!({ "face_id": "FACE-1" })));
878/// let face_id = simple::ai::enroll_face("EMP-42", json!("iVBORw0KGgo…"))?;
879///
880/// assert_eq!(face_id, "FACE-1");
881/// # Ok::<(), Error>(())
882/// ```
883pub fn enroll_face(subject_id: &str, image: Value) -> Result<String, Error> {
884    if subject_id.trim().is_empty() {
885        return Err(
886            Error::invalid("enroll_face needs the subject the face belongs to.")
887                .hint("Pass the id the face is to be found under, such as an employee or user id."),
888        );
889    }
890
891    refuse_empty(
892        &image,
893        "enroll_face needs an image of the face.",
894        "Pass a base64 image, or the handle of an uploaded one.",
895    )?;
896
897    let image = upload_pending(image)?;
898
899    let enrolled = host::transport()?
900        .call(
901            FACE_ENROLL.to_string(),
902            json!({ "image": image, "subject_id": subject_id }),
903        )
904        .map_err(|cause| {
905            cause.hint(
906                "Search the subject before enrolling the same face again, and report the failure.",
907            )
908        })?;
909
910    enrolled
911        .get("face_id")
912        .and_then(Value::as_str)
913        .map(str::to_string)
914        .ok_or_else(|| {
915            Error::Json(Fault::new(
916                Code::unspecified(),
917                "The enrolment answered without the id of the face it enrolled.",
918            ))
919            .hint("Search the subject to establish what was enrolled before enrolling again.")
920        })
921}
922
923/// Find the faces in the tenant's collection that match an image.
924///
925/// `image` is either a base64 image or a document handle. The matches are
926/// decoded into the type the call asks for; name your own to read them, or ask
927/// for a [`Value`] to keep them whole.
928///
929/// ```
930/// # use simpleplatform_sdk::prelude::*;
931/// # use simpleplatform_sdk::ai::FaceSearch;
932/// # use simpleplatform_sdk::testing;
933/// # let _session = testing::install(|_name, _params| Ok(json!([{ "subject_id": "EMP-42" }])));
934/// let found: Value = simple::ai::search_face(
935///     json!("iVBORw0KGgo…"),
936///     FaceSearch {
937///         max_faces: Some(3),
938///         similarity_threshold: Some(92.5),
939///     },
940/// )?;
941///
942/// assert_eq!(found[0]["subject_id"], json!("EMP-42"));
943/// # Ok::<(), Error>(())
944/// ```
945pub fn search_face<T: DeserializeOwned>(image: Value, options: FaceSearch) -> Result<T, Error> {
946    refuse_empty(
947        &image,
948        "search_face needs an image to search with.",
949        "Pass a base64 image, or the handle of an uploaded one.",
950    )?;
951
952    let image = upload_pending(image)?;
953
954    let mut narrowing = Map::new();
955
956    if let Some(max_faces) = options.max_faces {
957        narrowing.insert("max_faces".to_string(), Value::from(max_faces));
958    }
959
960    if let Some(threshold) = options
961        .similarity_threshold
962        .filter(|value| value.is_finite())
963    {
964        narrowing.insert("similarity_threshold".to_string(), Value::from(threshold));
965    }
966
967    let found = host::transport()?
968        .call(
969            FACE_SEARCH.to_string(),
970            json!({ "image": image, "options": narrowing }),
971        )
972        .map_err(|cause| {
973            cause.hint("Nothing in the collection was changed. Report the search failure.")
974        })?;
975
976    serde_json::from_value(found).map_err(|cause| {
977        Error::Json(Fault::new(
978            Code::unspecified(),
979            format!("The matches could not be read into the type this action asked for: {cause}"),
980        ))
981        .hint("Read the matches as a Value, or name a type that matches what the search answers.")
982    })
983}
984
985/// Remove faces from the tenant's collection, by the ids they were enrolled
986/// under.
987///
988/// The answer is the ids the platform reports as removed.
989///
990/// ```
991/// # use simpleplatform_sdk::prelude::*;
992/// # use simpleplatform_sdk::testing;
993/// # let _session = testing::install(|_name, _params| Ok(json!({ "deleted": ["FACE-1"] })));
994/// let removed = simple::ai::delete_face(&["FACE-1".to_string()])?;
995///
996/// assert_eq!(removed, ["FACE-1"]);
997/// # Ok::<(), Error>(())
998/// ```
999pub fn delete_face(face_ids: &[String]) -> Result<Vec<String>, Error> {
1000    if face_ids.is_empty() {
1001        return Err(Error::invalid("delete_face needs at least one face id.")
1002            .hint("Pass the ids enroll_face answered with."));
1003    }
1004
1005    let removed = host::transport()?
1006        .call(FACE_DELETE.to_string(), json!({ "face_ids": face_ids }))
1007        .map_err(|cause| {
1008            cause.hint(
1009                "Treat the removal as unfinished: search the collection to establish what is \
1010                 still in it before deleting again.",
1011            )
1012        })?;
1013
1014    let listed = match &removed {
1015        Value::Array(ids) => Some(ids),
1016        other => other.get("deleted").and_then(Value::as_array),
1017    };
1018
1019    Ok(listed
1020        .map(|ids| {
1021            ids.iter()
1022                .filter_map(Value::as_str)
1023                .map(str::to_string)
1024                .collect()
1025        })
1026        .unwrap_or_default())
1027}
1028
1029/// One operation: upload what is pending, ask the primitive, read the answer.
1030fn run<T: DeserializeOwned>(
1031    operation: &str,
1032    input: Value,
1033    prompt: Option<&str>,
1034    schema: Option<Value>,
1035    options: &Options,
1036) -> Result<Execution<T>, Error> {
1037    let input = upload_pending(input)?;
1038
1039    let answered = host::transport()?
1040        .call(
1041            ORCHESTRATOR.to_string(),
1042            payload(operation, input, prompt, schema, options),
1043        )
1044        .map_err(|cause| {
1045            cause.hint(
1046                "No tenant record was changed. Review the input, the prompt and the schema, \
1047                 and report the failure.",
1048            )
1049        })?;
1050
1051    // The metadata is read on its own and never decides the outcome, so a run
1052    // that answered is reported as one whatever it said about its own cost.
1053    let metadata = metadata_of(answered.get("metadata"));
1054    let data = answered.get("data").cloned().unwrap_or(Value::Null);
1055
1056    let data = serde_json::from_value(data).map_err(|cause| {
1057        Error::Json(Fault::new(
1058            Code::unspecified(),
1059            format!("The {operation} answer could not be read into the type this action asked for: {cause}"),
1060        ))
1061        .hint(
1062            "The operation ran. Read the answer as a Value, or name a type that matches the \
1063             schema, rather than running it again.",
1064        )
1065    })?;
1066
1067    Ok(Execution { data, metadata })
1068}
1069
1070/// Everything the primitive is told about one operation.
1071///
1072/// A choice that was not made is left out, so what arrives is what the call
1073/// asked for and the platform's own defaults fill the rest.
1074fn payload(
1075    operation: &str,
1076    input: Value,
1077    prompt: Option<&str>,
1078    schema: Option<Value>,
1079    options: &Options,
1080) -> Value {
1081    // The engine reads `reasoning` and `reasoning_budget` from here, and the
1082    // whole map is part of the key a cached result is found under.
1083    let mut universal = Map::new();
1084
1085    universal.insert("reasoning".to_string(), Value::Bool(options.reasoning));
1086
1087    if let Some(budget) = options.reasoning_budget {
1088        universal.insert("reasoning_budget".to_string(), Value::from(budget));
1089    }
1090
1091    if let Some(temperature) = options.temperature.filter(|value| value.is_finite()) {
1092        universal.insert("temperature".to_string(), Value::from(temperature));
1093    }
1094
1095    let mut payload = Map::new();
1096
1097    payload.insert("operation".to_string(), Value::from(operation));
1098    payload.insert("input".to_string(), input);
1099    payload.insert("options".to_string(), Value::Object(universal));
1100
1101    // An operation whose instructions are the platform's is sent none, rather
1102    // than an empty one it would have to tell apart from a prompt.
1103    if let Some(prompt) = prompt {
1104        payload.insert("prompt".to_string(), Value::from(prompt));
1105    }
1106    payload.insert("regenerate".to_string(), Value::Bool(options.regenerate));
1107
1108    if let Some(schema) = schema {
1109        payload.insert("schema".to_string(), schema);
1110    }
1111
1112    if let Some(model) = options.model {
1113        payload.insert("model".to_string(), Value::from(model.as_str()));
1114    }
1115
1116    if let Some(system_prompt) = &options.system_prompt {
1117        payload.insert(
1118            "systemPrompt".to_string(),
1119            Value::from(system_prompt.as_str()),
1120        );
1121    }
1122
1123    if let Some(timeout) = options.timeout {
1124        payload.insert("timeout".to_string(), Value::from(milliseconds(timeout)));
1125    }
1126
1127    Value::Object(payload)
1128}
1129
1130/// A duration as the milliseconds the wire carries.
1131fn milliseconds(timeout: Duration) -> u64 {
1132    u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX)
1133}
1134
1135/// What a run reported about itself.
1136fn metadata_of(metadata: Option<&Value>) -> Metadata {
1137    let Some(metadata) = metadata else {
1138        return Metadata::default();
1139    };
1140
1141    Metadata {
1142        input_tokens: count(metadata, "input_tokens").unwrap_or(0),
1143        output_tokens: count(metadata, "output_tokens").unwrap_or(0),
1144        reasoning_tokens: count(metadata, "reasoning_tokens"),
1145        reasoning: metadata
1146            .get("reasoning")
1147            .and_then(Value::as_str)
1148            .map(str::to_string),
1149        delivery: delivery_of(metadata.get("delivery")),
1150    }
1151}
1152
1153/// One token count, when it is there and is a count.
1154fn count(metadata: &Value, key: &str) -> Option<u64> {
1155    metadata.get(key).and_then(Value::as_u64)
1156}
1157
1158/// How each file travelled, as far as the report can be read.
1159///
1160/// The report explains an answer; it never decides whether there is one. An
1161/// entry that is not an object, names no file, or names a way of travelling
1162/// this SDK does not know is left out rather than failing a run that answered.
1163fn delivery_of(delivery: Option<&Value>) -> Vec<FileDelivery> {
1164    delivery
1165        .and_then(Value::as_array)
1166        .map(|files| files.iter().filter_map(file_delivery).collect())
1167        .unwrap_or_default()
1168}
1169
1170/// One file's delivery, when the entry can be read as one.
1171///
1172/// A range is kept only when both ends are there, because a range with one end
1173/// is not one the engine could have cut. A key sent as `null` reads as absent.
1174fn file_delivery(file: &Value) -> Option<FileDelivery> {
1175    let delivered_as = match file.get("delivered_as").and_then(Value::as_str)? {
1176        "document" => DeliveredAs::Document,
1177        "text" => DeliveredAs::Text,
1178        "image" => DeliveredAs::Image,
1179        _ => return None,
1180    };
1181
1182    let first_page = file.get("first_page").and_then(page_number);
1183    let last_page = file.get("last_page").and_then(page_number);
1184
1185    Some(FileDelivery {
1186        filename: file.get("filename").and_then(Value::as_str)?.to_string(),
1187        delivered_as,
1188        pages: first_page
1189            .zip(last_page)
1190            .map(|(first, last)| Pages::new(first, last)),
1191        transcribed_pages: page_numbers(file.get("transcribed_pages")),
1192        fallback: file
1193            .get("fallback")
1194            .filter(|fallback| fallback.is_object())
1195            .map(delivery_fallback),
1196    })
1197}
1198
1199/// A fallback, read leniently: that the document went instead of its text is
1200/// the fact that matters, so a member missing from the report does not lose it.
1201fn delivery_fallback(fallback: &Value) -> DeliveryFallback {
1202    let text = |key: &str| {
1203        fallback
1204            .get(key)
1205            .and_then(Value::as_str)
1206            .unwrap_or_default()
1207            .to_string()
1208    };
1209
1210    DeliveryFallback {
1211        reason: text("reason"),
1212        pages: page_numbers(fallback.get("pages")),
1213        message: text("message"),
1214    }
1215}
1216
1217/// A page number, when this is one: a whole number from 1.
1218fn page_number(value: &Value) -> Option<u32> {
1219    value
1220        .as_u64()
1221        .and_then(|number| u32::try_from(number).ok())
1222        .filter(|page| *page > 0)
1223}
1224
1225/// The page numbers in a list, leaving out anything that is not one.
1226fn page_numbers(pages: Option<&Value>) -> Vec<u32> {
1227    pages
1228        .and_then(Value::as_array)
1229        .map(|pages| pages.iter().filter_map(page_number).collect())
1230        .unwrap_or_default()
1231}
1232
1233/// Why a page the platform answered for without its text is unread.
1234const NO_TRANSCRIPTION: &str = "no transcription was returned for this page";
1235
1236/// Each page a transcription answered for, in the shape a caller can rely on.
1237///
1238/// An answer without a list of pages has none. An entry that names no page is
1239/// left out, since there is no page it could be the transcription of.
1240fn page_transcriptions(data: &Value) -> Vec<PageTranscription> {
1241    data.get("pages")
1242        .and_then(Value::as_array)
1243        .map(|pages| pages.iter().filter_map(page_transcription).collect())
1244        .unwrap_or_default()
1245}
1246
1247/// One page's transcription: its text or its error, never both.
1248///
1249/// The platform's page is mapped rather than passed through. An error wins
1250/// over any text, so no partial text is handed on, and a page that carries no
1251/// text is answered as unread rather than as text that is not there.
1252fn page_transcription(entry: &Value) -> Option<PageTranscription> {
1253    let page = entry.get("page").and_then(page_number)?;
1254    let error = entry.get("error").and_then(Value::as_str);
1255    let text = entry.get("text").and_then(Value::as_str);
1256
1257    Some(match (error, text) {
1258        (None, Some(text)) => PageTranscription::Read {
1259            page,
1260            text: text.to_string(),
1261        },
1262        (error, _) => PageTranscription::Unread {
1263            page,
1264            error: error.unwrap_or(NO_TRANSCRIPTION).to_string(),
1265        },
1266    })
1267}
1268
1269/// Put whatever is still pending into ephemeral storage, and answer with what
1270/// the operation is to be given instead.
1271///
1272/// A handle that is already stored, and anything that is not a handle at all,
1273/// comes back as it went in.
1274fn upload_pending(value: Value) -> Result<Value, Error> {
1275    if is_pending(&value) {
1276        let stored = host::transport()?
1277            .call(UPLOAD_EPHEMERAL.to_string(), value.clone())
1278            .map_err(|cause| {
1279                cause.hint(
1280                    "The file was not uploaded and the operation did not run. Upload the file, \
1281                     then pass the handle that upload answered with.",
1282                )
1283            })?;
1284
1285        return Ok(stored_handle(value, stored));
1286    }
1287
1288    if let Value::Array(items) = value {
1289        let mut uploaded = Vec::with_capacity(items.len());
1290
1291        for item in items {
1292            uploaded.push(upload_pending(item)?);
1293        }
1294
1295        return Ok(Value::Array(uploaded));
1296    }
1297
1298    Ok(value)
1299}
1300
1301/// The handle the operation reads, once a pending file has been stored.
1302///
1303/// The upload answers with the stored file's own reference: where it is, what
1304/// it hashes to, its name, its type and its size. Every one of those keys
1305/// describes the file, so every one of them is taken from the upload and none
1306/// is carried over. A value from before the upload would name a file that is
1307/// no longer the one being read.
1308///
1309/// Everything else on the handle is the caller's: how the file is to be sent,
1310/// which of its pages, and any key added to a file reference later. Those
1311/// describe the request, not the file, and the upload knows nothing about
1312/// them, so they are carried over. A caller reads a pending file and a stored
1313/// one the same way.
1314///
1315/// `pending` is the one key that is neither. It said the bytes had not been
1316/// stored yet. They have been now, so it is dropped. A handle that still
1317/// called itself pending would be uploaded a second time on the next pass.
1318fn stored_handle(handle: Value, stored: Value) -> Value {
1319    match (handle, stored) {
1320        (Value::Object(mut handle), Value::Object(stored)) => {
1321            handle.remove("pending");
1322            handle.extend(stored);
1323
1324            Value::Object(handle)
1325        }
1326        (_, stored) => stored,
1327    }
1328}
1329
1330/// Whether this is a document handle whose file is still to be uploaded.
1331///
1332/// It is one when it says it is pending and carries the hash of the file it
1333/// stands for, since that hash is what the upload is asked for by.
1334fn is_pending(value: &Value) -> bool {
1335    value.get("pending").and_then(Value::as_bool) == Some(true) && has_file_hash(value)
1336}
1337
1338/// Whether this carries the hash of a file.
1339fn has_file_hash(value: &Value) -> bool {
1340    value
1341        .get("file_hash")
1342        .and_then(Value::as_str)
1343        .is_some_and(|hash| !hash.is_empty())
1344}
1345
1346/// Refuse a value that names nothing, in the words of whoever needed it.
1347fn refuse_empty(value: &Value, message: &str, hint: &str) -> Result<(), Error> {
1348    let empty = match value {
1349        Value::Null => true,
1350        Value::String(text) => text.is_empty(),
1351        _ => false,
1352    };
1353
1354    if empty {
1355        return Err(Error::invalid(message).hint(hint));
1356    }
1357
1358    Ok(())
1359}
1360
1361/// Refuse an operation that has been given nothing to do.
1362fn refuse_empty_prompt(prompt: &str, operation: &str) -> Result<(), Error> {
1363    if prompt.trim().is_empty() {
1364        return Err(Error::invalid(format!("{operation} needs a prompt."))
1365            .hint("Say what the operation is to produce, in a sentence the model can act on."));
1366    }
1367
1368    Ok(())
1369}
1370
1371/// Refuse anything that is not the handle of an audio or video file.
1372fn refuse_unless_media(document: &Value) -> Result<(), Error> {
1373    if !has_file_hash(document) {
1374        return Err(
1375            Error::invalid("transcribe needs a document handle to work on.").hint(
1376                "Pass the handle of the uploaded file, the one carrying file_hash and mime_type.",
1377            ),
1378        );
1379    }
1380
1381    let media = document
1382        .get("mime_type")
1383        .and_then(Value::as_str)
1384        .map(str::to_lowercase)
1385        .is_some_and(|mime| mime.starts_with("audio/") || mime.starts_with("video/"));
1386
1387    if !media {
1388        return Err(Error::invalid("transcribe works on audio and video.")
1389            .hint("Pass a handle whose mime_type begins audio/ or video/."));
1390    }
1391
1392    Ok(())
1393}
1394
1395/// Refuse anything that is not the handle of a stored PDF.
1396///
1397/// The type is compared without its parameters and without regard to case, so
1398/// `Application/PDF; version=1.7` is a PDF.
1399fn refuse_unless_pdf(document: &DocumentHandle) -> Result<(), Error> {
1400    if document.file_hash.is_empty() {
1401        return Err(
1402            Error::invalid("transcribe_pages needs the handle of a stored file.").hint(
1403                "Pass the handle the upload or the record answered with, the one carrying \
1404                 file_hash.",
1405            ),
1406        );
1407    }
1408
1409    let essence = document.mime_type.split(';').next().unwrap_or_default();
1410
1411    if !essence.trim().eq_ignore_ascii_case("application/pdf") {
1412        return Err(Error::invalid("transcribe_pages reads the pages of a PDF.")
1413            .hint("Pass the handle of a PDF. Read any other file with extract or summarize."));
1414    }
1415
1416    Ok(())
1417}
1418
1419/// The schema a transcription is held to, written from what was asked for.
1420fn transcribe_schema(wanted: &TranscribeOptions) -> Value {
1421    let identify = wanted.participants.wanted();
1422
1423    let mut properties = Map::new();
1424    let mut required = vec![Value::from("language")];
1425
1426    properties.insert(
1427        "language".to_string(),
1428        json!({
1429            "description": "The detected language of the audio (ISO 639-1 code, e.g., \"en\", \"es\")",
1430            "type": "string"
1431        }),
1432    );
1433
1434    if wanted.include_transcript {
1435        let description = match (identify, wanted.include_timestamps) {
1436            (true, true) => {
1437                "The full transcript with participant labels and timestamps. \
1438                 Format: [MM:SS] Participant Name: text"
1439            }
1440            (true, false) => {
1441                "The full transcript with participant labels. Format: Participant Name: text"
1442            }
1443            (false, true) => "The full transcript with timestamps. Format: [MM:SS] text",
1444            (false, false) => "The full transcript of the audio",
1445        };
1446
1447        properties.insert(
1448            "transcript".to_string(),
1449            json!({ "description": description, "type": "string" }),
1450        );
1451        required.push(Value::from("transcript"));
1452    }
1453
1454    if wanted.summarize {
1455        let description = if identify {
1456            "A concise summary of the audio content, including key points from each participant"
1457        } else {
1458            "A concise summary of the audio content"
1459        };
1460
1461        properties.insert(
1462            "summary".to_string(),
1463            json!({ "description": description, "type": "string" }),
1464        );
1465        required.push(Value::from("summary"));
1466    }
1467
1468    if identify {
1469        properties.insert(
1470            "participants".to_string(),
1471            json!({
1472                "description": "List of identified participants in the audio",
1473                "items": { "type": "string" },
1474                "type": "array"
1475            }),
1476        );
1477        required.push(Value::from("participants"));
1478    }
1479
1480    json!({ "properties": properties, "required": required, "type": "object" })
1481}
1482
1483/// The prompt a transcription runs on, written from what was asked for.
1484fn transcribe_prompt(wanted: &TranscribeOptions) -> String {
1485    let identify = wanted.participants.wanted();
1486    let names = wanted.participants.names();
1487
1488    let mut prompt = String::from("Analyze this audio/video file and provide:\n");
1489
1490    if wanted.include_transcript {
1491        if identify {
1492            match &names {
1493                Some(names) => {
1494                    prompt.push_str("- A complete transcript identifying these participants: ");
1495                    prompt.push_str(names);
1496                    prompt.push_str(". ");
1497                }
1498                None => prompt.push_str(
1499                    "- A complete transcript with participant identification \
1500                     (label participants as Participant 1, Participant 2, etc.). ",
1501                ),
1502            }
1503
1504            if wanted.include_timestamps {
1505                prompt.push_str(
1506                    "Include timestamps in [MM:SS] format before each participant segment.\n",
1507                );
1508            } else {
1509                prompt.push_str("Format each line as \"Participant Name: text\".\n");
1510            }
1511        } else if wanted.include_timestamps {
1512            prompt.push_str(
1513                "- A complete transcript with timestamps in [MM:SS] format before each segment\n",
1514            );
1515        } else {
1516            prompt.push_str("- A complete transcript of all spoken content\n");
1517        }
1518    }
1519
1520    if wanted.summarize {
1521        if identify {
1522            prompt.push_str("- A concise summary highlighting key points from each participant\n");
1523        } else {
1524            prompt.push_str("- A concise summary of the main points and key information\n");
1525        }
1526    }
1527
1528    if identify {
1529        match &names {
1530            Some(names) => {
1531                prompt.push_str("- Identify and distinguish between these participants: ");
1532                prompt.push_str(names);
1533                prompt.push('\n');
1534            }
1535            None => prompt.push_str("- Identify and list all distinct participants in the audio\n"),
1536        }
1537    }
1538
1539    prompt.push_str("- The detected language code (ISO 639-1 format)\n");
1540
1541    prompt
1542}
1543
1544#[cfg(test)]
1545mod tests {
1546    use serde_json::json;
1547
1548    use super::*;
1549    use crate::testing;
1550
1551    /// What the primitive answers when a test does not care what came back.
1552    fn answered() -> Value {
1553        json!({ "data": { "ok": true }, "metadata": { "input_tokens": 1, "output_tokens": 2 } })
1554    }
1555
1556    /// A stored file, as an upload answers with one.
1557    fn stored() -> DocumentHandle {
1558        DocumentHandle {
1559            file_hash: "ffce20f1".to_string(),
1560            filename: "contract.pdf".to_string(),
1561            mime_type: "application/pdf".to_string(),
1562            size: 1_234,
1563            storage_path: "acme/documents/ffce20f1".to_string(),
1564        }
1565    }
1566
1567    #[test]
1568    fn a_document_input_travels_as_the_handle_with_the_keys_the_engine_reads() {
1569        let sent = json!(DocumentInput {
1570            file: stored(),
1571            pages: Some(Pages::new(12, 30)),
1572            deliver_as: Some(Delivery::Text),
1573        });
1574
1575        assert_eq!(sent["storage_path"], json!("acme/documents/ffce20f1"));
1576        assert_eq!(sent["file_hash"], json!("ffce20f1"));
1577        assert_eq!(sent["mime_type"], json!("application/pdf"));
1578        assert_eq!(
1579            sent["first_page"],
1580            json!(12),
1581            "a range travels as the two keys the engine reads, not as a nested one"
1582        );
1583        assert_eq!(sent["last_page"], json!(30));
1584        assert_eq!(sent["deliver_as"], json!("text"));
1585    }
1586
1587    #[test]
1588    fn a_document_input_that_asks_for_nothing_carries_neither_key() {
1589        let sent = json!(DocumentInput {
1590            file: stored(),
1591            ..Default::default()
1592        });
1593
1594        assert_eq!(sent["filename"], json!("contract.pdf"));
1595        assert_eq!(
1596            sent.get("deliver_as"),
1597            None,
1598            "asking for nothing says nothing, so the default stays the engine's"
1599        );
1600        assert_eq!(sent.get("first_page"), None);
1601        assert_eq!(sent.get("last_page"), None);
1602    }
1603
1604    #[test]
1605    fn an_extraction_asks_the_primitive_and_answers_with_the_type_it_was_asked_for() {
1606        #[derive(Debug, Deserialize, PartialEq)]
1607        struct Invoice {
1608            number: String,
1609            total: f64,
1610        }
1611
1612        let session = testing::install(|name, params| {
1613            assert_eq!(name, ORCHESTRATOR);
1614            assert_eq!(params["operation"], json!("extract"));
1615            assert_eq!(params["prompt"], json!("Read it."));
1616            assert_eq!(params["input"], json!("INV-8, 240.00"));
1617            assert_eq!(params["schema"]["type"], json!("object"));
1618
1619            Ok(json!({
1620                "data": { "number": "INV-8", "total": 240.0 },
1621                "metadata": { "input_tokens": 11, "output_tokens": 3 }
1622            }))
1623        });
1624
1625        let read: Execution<Invoice> = extract(
1626            json!("INV-8, 240.00"),
1627            "Read it.",
1628            json!({ "type": "object" }),
1629            Options::default(),
1630        )
1631        .unwrap();
1632
1633        assert_eq!(
1634            read.data,
1635            Invoice {
1636                number: "INV-8".to_string(),
1637                total: 240.0
1638            }
1639        );
1640        assert_eq!(read.metadata.input_tokens, 11);
1641        assert_eq!(read.metadata.output_tokens, 3);
1642        assert_eq!(session.calls().len(), 1);
1643    }
1644
1645    #[test]
1646    fn a_default_call_reasons_does_not_regenerate_and_leaves_the_rest_to_the_platform() {
1647        let session = testing::install(|_name, _params| Ok(answered()));
1648
1649        let _: Execution<Value> = extract(
1650            json!("text"),
1651            "Read it.",
1652            json!({ "type": "object" }),
1653            Options::default(),
1654        )
1655        .unwrap();
1656
1657        let sent = session.calls()[0].params.clone();
1658
1659        assert_eq!(sent["options"], json!({ "reasoning": true }));
1660        assert_eq!(sent["regenerate"], json!(false));
1661        assert!(sent.get("model").is_none());
1662        assert!(sent.get("systemPrompt").is_none());
1663        assert!(sent.get("timeout").is_none());
1664    }
1665
1666    #[test]
1667    fn every_option_that_was_set_travels_with_the_operation() {
1668        let session = testing::install(|_name, _params| Ok(answered()));
1669
1670        let _: Execution<Value> = extract(
1671            json!("text"),
1672            "Read it.",
1673            json!({ "type": "object" }),
1674            Options {
1675                model: Some(Model::Xl),
1676                system_prompt: Some("You are an auditor.".to_string()),
1677                temperature: Some(0.25),
1678                reasoning: false,
1679                reasoning_budget: Some(2_048),
1680                regenerate: true,
1681                timeout: Some(Duration::from_secs(90)),
1682            },
1683        )
1684        .unwrap();
1685
1686        let sent = session.calls()[0].params.clone();
1687
1688        assert_eq!(sent["model"], json!("xl"));
1689        assert_eq!(sent["systemPrompt"], json!("You are an auditor."));
1690        assert_eq!(sent["regenerate"], json!(true));
1691        assert_eq!(sent["timeout"], json!(90_000));
1692        assert_eq!(
1693            sent["options"],
1694            json!({ "reasoning": false, "reasoning_budget": 2_048, "temperature": 0.25 })
1695        );
1696    }
1697
1698    #[test]
1699    fn a_summary_answers_with_the_prose_and_carries_no_schema() {
1700        let session = testing::install(|_name, params| {
1701            assert_eq!(params["operation"], json!("summarize"));
1702            assert!(params.get("schema").is_none());
1703
1704            Ok(json!({ "data": "A refund was agreed.", "metadata": { "output_tokens": 5 } }))
1705        });
1706
1707        let written = summarize(json!("A long thread."), "One sentence.", Options::default())
1708            .expect("a summary answers with prose");
1709
1710        assert_eq!(written.data, "A refund was agreed.");
1711        assert_eq!(written.metadata.output_tokens, 5);
1712        assert_eq!(session.calls().len(), 1);
1713    }
1714
1715    #[test]
1716    fn a_pending_document_is_uploaded_before_the_operation_reads_it() {
1717        let session = testing::install(|name, _params| {
1718            if name == UPLOAD_EPHEMERAL {
1719                return Ok(json!({ "file_hash": "abc", "storage_path": "ephemeral/abc" }));
1720            }
1721
1722            Ok(answered())
1723        });
1724
1725        let _: Execution<Value> = extract(
1726            json!({ "file_hash": "abc", "pending": true }),
1727            "Read it.",
1728            json!({ "type": "object" }),
1729            Options::default(),
1730        )
1731        .unwrap();
1732
1733        let calls = session.calls();
1734
1735        assert_eq!(calls.len(), 2);
1736        assert_eq!(calls[0].name, UPLOAD_EPHEMERAL);
1737        assert_eq!(calls[1].name, ORCHESTRATOR);
1738        assert_eq!(
1739            calls[1].params["input"],
1740            json!({ "file_hash": "abc", "storage_path": "ephemeral/abc" }),
1741            "the operation reads the handle the upload answered with"
1742        );
1743    }
1744
1745    #[test]
1746    fn a_pending_document_is_read_the_way_the_caller_asked_for_it() {
1747        let session = testing::install(|name, _params| {
1748            if name == UPLOAD_EPHEMERAL {
1749                return Ok(json!({
1750                    "file_hash": "ffce20f1",
1751                    "filename": "contract.pdf",
1752                    "mime_type": "application/pdf",
1753                    "size": 1_234,
1754                    "storage_path": "acme/ephemeral/ffce20f1"
1755                }));
1756            }
1757
1758            Ok(answered())
1759        });
1760
1761        let _: Execution<Value> = extract(
1762            json!({
1763                "file_hash": "staged",
1764                "first_page": 12,
1765                "last_page": 30,
1766                "pending": true,
1767                "deliver_as": "text",
1768                "storage_path": "acme/staged/staged"
1769            }),
1770            "Read it.",
1771            json!({ "type": "object" }),
1772            Options::default(),
1773        )
1774        .unwrap();
1775
1776        let calls = session.calls();
1777        let sent = calls[1].params["input"].clone();
1778
1779        assert_eq!(
1780            sent["deliver_as"],
1781            json!("text"),
1782            "the caller said how the file was to be sent"
1783        );
1784        assert_eq!(
1785            sent["first_page"],
1786            json!(12),
1787            "the caller asked for a range"
1788        );
1789        assert_eq!(sent["last_page"], json!(30), "the caller asked for a range");
1790        assert_eq!(
1791            sent["file_hash"],
1792            json!("ffce20f1"),
1793            "the file is the one the upload stored"
1794        );
1795        assert_eq!(
1796            sent["storage_path"],
1797            json!("acme/ephemeral/ffce20f1"),
1798            "the file is read from where the upload put it"
1799        );
1800        assert_eq!(sent["filename"], json!("contract.pdf"));
1801        assert_eq!(sent["mime_type"], json!("application/pdf"));
1802        assert_eq!(sent["size"], json!(1_234));
1803        assert!(
1804            sent.get("pending").is_none(),
1805            "the file is stored, so the handle may not still call itself pending"
1806        );
1807    }
1808
1809    #[test]
1810    fn a_document_that_is_already_stored_is_read_where_it_is() {
1811        let session = testing::install(|_name, _params| Ok(answered()));
1812
1813        let handle = json!({ "file_hash": "abc", "mime_type": "application/pdf" });
1814
1815        let _: Execution<Value> = extract(
1816            handle.clone(),
1817            "Read it.",
1818            json!({ "type": "object" }),
1819            Options::default(),
1820        )
1821        .unwrap();
1822
1823        let calls = session.calls();
1824
1825        assert_eq!(calls.len(), 1, "nothing was uploaded");
1826        assert_eq!(calls[0].params["input"], handle);
1827    }
1828
1829    #[test]
1830    fn a_list_of_documents_is_uploaded_item_by_item() {
1831        let session = testing::install(|name, params| {
1832            if name == UPLOAD_EPHEMERAL {
1833                let hash = params["file_hash"].as_str().unwrap_or_default();
1834
1835                return Ok(json!({
1836                    "file_hash": hash,
1837                    "storage_path": format!("ephemeral/{hash}")
1838                }));
1839            }
1840
1841            Ok(answered())
1842        });
1843
1844        let _: Execution<Value> = extract(
1845            json!([
1846                { "file_hash": "one", "pending": true },
1847                { "file_hash": "two", "mime_type": "image/png" },
1848                { "file_hash": "three", "pending": true }
1849            ]),
1850            "Read them.",
1851            json!({ "type": "object" }),
1852            Options::default(),
1853        )
1854        .unwrap();
1855
1856        let calls = session.calls();
1857
1858        assert_eq!(calls.len(), 3, "two uploads and the operation");
1859        assert_eq!(
1860            calls[2].params["input"],
1861            json!([
1862                { "file_hash": "one", "storage_path": "ephemeral/one" },
1863                { "file_hash": "two", "mime_type": "image/png" },
1864                { "file_hash": "three", "storage_path": "ephemeral/three" }
1865            ])
1866        );
1867    }
1868
1869    #[test]
1870    fn a_handle_is_pending_only_while_it_says_so_and_carries_a_hash_to_upload_by() {
1871        assert!(is_pending(&json!({ "file_hash": "abc", "pending": true })));
1872        assert!(!is_pending(&json!({ "file_hash": "abc" })));
1873        assert!(!is_pending(&json!({ "pending": true })));
1874        assert!(!is_pending(&json!({ "file_hash": "", "pending": true })));
1875        assert!(!is_pending(
1876            &json!({ "file_hash": "abc", "pending": "yes" })
1877        ));
1878        assert!(!is_pending(&json!("some words")));
1879        assert!(!is_pending(&json!([
1880            { "file_hash": "abc", "pending": true }
1881        ])));
1882    }
1883
1884    #[test]
1885    fn an_answer_that_reports_nothing_about_its_cost_is_still_an_answer() {
1886        let _session = testing::install(|_name, _params| Ok(json!({ "data": "done" })));
1887
1888        let written = summarize(json!("text"), "One sentence.", Options::default()).unwrap();
1889
1890        assert_eq!(written.data, "done");
1891        assert_eq!(written.metadata, Metadata::default());
1892    }
1893
1894    #[test]
1895    fn reasoning_reaches_the_caller_when_the_run_reported_it() {
1896        let _session = testing::install(|_name, _params| {
1897            Ok(json!({
1898                "data": "done",
1899                "metadata": {
1900                    "input_tokens": 9,
1901                    "output_tokens": 4,
1902                    "reasoning": "Read the total from the last line.",
1903                    "reasoning_tokens": 120
1904                }
1905            }))
1906        });
1907
1908        let written = summarize(json!("text"), "One sentence.", Options::default()).unwrap();
1909
1910        assert_eq!(written.metadata.reasoning_tokens, Some(120));
1911        assert_eq!(
1912            written.metadata.reasoning.as_deref(),
1913            Some("Read the total from the last line.")
1914        );
1915    }
1916
1917    #[test]
1918    fn a_run_reports_how_each_file_travelled_in_the_order_the_files_were_given() {
1919        let _session = testing::install(|_name, _params| {
1920            Ok(json!({
1921                "data": "done",
1922                "metadata": {
1923                    "input_tokens": 9,
1924                    "output_tokens": 4,
1925                    "delivery": [
1926                        {
1927                            "filename": "contract.pdf",
1928                            "delivered_as": "text",
1929                            "first_page": 40,
1930                            "last_page": 52,
1931                            "transcribed_pages": [41, 44],
1932                            "fallback": null
1933                        },
1934                        {
1935                            "filename": "drawings.pdf",
1936                            "delivered_as": "document",
1937                            "first_page": null,
1938                            "last_page": null,
1939                            "transcribed_pages": [],
1940                            "fallback": {
1941                                "reason": "transcription_failed",
1942                                "pages": [3, 7],
1943                                "message": "page 3: timed out; page 7: blank answer"
1944                            }
1945                        },
1946                        {
1947                            "filename": "site.jpg",
1948                            "delivered_as": "image",
1949                            "first_page": null,
1950                            "last_page": null,
1951                            "transcribed_pages": [],
1952                            "fallback": null
1953                        }
1954                    ]
1955                }
1956            }))
1957        });
1958
1959        let written = summarize(json!("text"), "One sentence.", Options::default()).unwrap();
1960
1961        assert_eq!(
1962            written.metadata.delivery,
1963            vec![
1964                FileDelivery {
1965                    filename: "contract.pdf".to_string(),
1966                    delivered_as: DeliveredAs::Text,
1967                    pages: Some(Pages::new(40, 52)),
1968                    transcribed_pages: vec![41, 44],
1969                    fallback: None,
1970                },
1971                FileDelivery {
1972                    filename: "drawings.pdf".to_string(),
1973                    delivered_as: DeliveredAs::Document,
1974                    pages: None,
1975                    transcribed_pages: Vec::new(),
1976                    fallback: Some(DeliveryFallback {
1977                        reason: "transcription_failed".to_string(),
1978                        pages: vec![3, 7],
1979                        message: "page 3: timed out; page 7: blank answer".to_string(),
1980                    }),
1981                },
1982                FileDelivery {
1983                    filename: "site.jpg".to_string(),
1984                    delivered_as: DeliveredAs::Image,
1985                    pages: None,
1986                    transcribed_pages: Vec::new(),
1987                    fallback: None,
1988                },
1989            ]
1990        );
1991        assert_eq!(
1992            written.metadata.input_tokens, 9,
1993            "the members a result already had read as they did"
1994        );
1995    }
1996
1997    #[test]
1998    fn an_answer_without_a_delivery_report_carries_none() {
1999        for metadata in [
2000            json!({ "input_tokens": 1 }),
2001            json!({ "delivery": null }),
2002            json!({ "delivery": "document" }),
2003            json!({ "delivery": { "filename": "contract.pdf", "delivered_as": "text" } }),
2004        ] {
2005            let _session = testing::install(move |_name, _params| {
2006                Ok(json!({ "data": "done", "metadata": metadata.clone() }))
2007            });
2008
2009            let written = summarize(json!("text"), "One sentence.", Options::default())
2010                .expect("a report that cannot be read never fails the run");
2011
2012            assert!(written.metadata.delivery.is_empty());
2013        }
2014    }
2015
2016    #[test]
2017    fn a_delivery_entry_that_cannot_be_read_is_left_out_and_the_rest_are_kept() {
2018        let delivery = delivery_of(Some(&json!([
2019            "contract.pdf",
2020            { "filename": "no-way.pdf" },
2021            { "filename": "audio.mp3", "delivered_as": "audio" },
2022            { "delivered_as": "text" },
2023            { "filename": 7, "delivered_as": "text" },
2024            {
2025                "filename": "kept.pdf",
2026                "delivered_as": "text",
2027                "first_page": 4,
2028                "transcribed_pages": ["5", -1, 0, 6, 4_294_967_296_u64, null]
2029            }
2030        ])));
2031
2032        assert_eq!(
2033            delivery,
2034            vec![FileDelivery {
2035                filename: "kept.pdf".to_string(),
2036                delivered_as: DeliveredAs::Text,
2037                pages: None,
2038                transcribed_pages: vec![6],
2039                fallback: None,
2040            }],
2041            "a range with one end is not a range, and only whole pages from 1 are pages"
2042        );
2043    }
2044
2045    #[test]
2046    fn a_fallback_that_is_reported_is_never_lost_to_a_missing_member() {
2047        let delivery = delivery_of(Some(&json!([
2048            {
2049                "filename": "contract.pdf",
2050                "delivered_as": "document",
2051                "fallback": { "reason": "transcription_failed" }
2052            },
2053            {
2054                "filename": "drawings.pdf",
2055                "delivered_as": "document",
2056                "fallback": "transcription_failed"
2057            }
2058        ])));
2059
2060        assert_eq!(
2061            delivery[0].fallback,
2062            Some(DeliveryFallback {
2063                reason: "transcription_failed".to_string(),
2064                pages: Vec::new(),
2065                message: String::new(),
2066            })
2067        );
2068        assert_eq!(
2069            delivery[1].fallback, None,
2070            "a fallback that is not an object says nothing that can be read"
2071        );
2072        assert!(delivery[0].transcribed_pages.is_empty());
2073    }
2074
2075    #[test]
2076    fn an_operation_with_nothing_to_do_is_refused_before_anything_is_sent() {
2077        let session = testing::install(|_name, _params| Ok(answered()));
2078
2079        let missing_input = extract::<Value>(
2080            json!(null),
2081            "Read it.",
2082            json!({ "type": "object" }),
2083            Options::default(),
2084        )
2085        .unwrap_err();
2086
2087        let missing_prompt = extract::<Value>(
2088            json!("text"),
2089            "   ",
2090            json!({ "type": "object" }),
2091            Options::default(),
2092        )
2093        .unwrap_err();
2094
2095        let missing_schema =
2096            extract::<Value>(json!("text"), "Read it.", json!([]), Options::default()).unwrap_err();
2097
2098        let missing_summary_input =
2099            summarize(json!(""), "One sentence.", Options::default()).unwrap_err();
2100
2101        for refusal in [
2102            &missing_input,
2103            &missing_prompt,
2104            &missing_schema,
2105            &missing_summary_input,
2106        ] {
2107            assert_eq!(refusal.code().as_str(), "INVALID_TOOL_INPUT");
2108        }
2109
2110        assert!(session.calls().is_empty());
2111    }
2112
2113    #[test]
2114    fn a_transcription_writes_its_own_schema_and_prompt() {
2115        let session = testing::install(|_name, _params| {
2116            Ok(json!({
2117                "data": { "language": "en", "transcript": "Hello." },
2118                "metadata": { "input_tokens": 400, "output_tokens": 12 }
2119            }))
2120        });
2121
2122        let heard = transcribe(
2123            json!({ "file_hash": "abc", "mime_type": "AUDIO/MPEG" }),
2124            TranscribeOptions {
2125                include_transcript: true,
2126                ..Default::default()
2127            },
2128            Options::default(),
2129        )
2130        .expect("an uppercase mime type is the same mime type");
2131
2132        assert_eq!(heard.data.language, "en");
2133        assert_eq!(heard.data.transcript.as_deref(), Some("Hello."));
2134        assert_eq!(heard.data.summary, None);
2135        assert!(heard.data.participants.is_empty());
2136
2137        let sent = session.calls()[0].params.clone();
2138
2139        assert_eq!(sent["operation"], json!("extract"));
2140        assert_eq!(
2141            sent["schema"]["required"],
2142            json!(["language", "transcript"])
2143        );
2144        assert_eq!(sent["schema"]["properties"]["transcript"]["type"], "string");
2145        assert_eq!(
2146            sent["prompt"],
2147            json!(
2148                "Analyze this audio/video file and provide:\n\
2149                 - A complete transcript of all spoken content\n\
2150                 - The detected language code (ISO 639-1 format)\n"
2151            )
2152        );
2153    }
2154
2155    #[test]
2156    fn named_participants_reach_both_the_prompt_and_the_schema() {
2157        let session = testing::install(|_name, _params| {
2158            Ok(json!({ "data": { "language": "en", "participants": ["Customer", "Agent"] } }))
2159        });
2160
2161        let heard = transcribe(
2162            json!({ "file_hash": "abc", "mime_type": "video/mp4" }),
2163            TranscribeOptions {
2164                include_transcript: true,
2165                include_timestamps: true,
2166                summarize: true,
2167                participants: Participants::Named(vec![
2168                    "Customer".to_string(),
2169                    "Agent".to_string(),
2170                ]),
2171            },
2172            Options::default(),
2173        )
2174        .unwrap();
2175
2176        assert_eq!(heard.data.participants, ["Customer", "Agent"]);
2177
2178        let sent = session.calls()[0].params.clone();
2179        let prompt = sent["prompt"].as_str().unwrap_or_default().to_string();
2180
2181        assert_eq!(
2182            sent["schema"]["required"],
2183            json!(["language", "transcript", "summary", "participants"])
2184        );
2185        assert_eq!(
2186            sent["schema"]["properties"]["participants"]["items"]["type"],
2187            json!("string")
2188        );
2189        assert!(prompt.contains("identifying these participants: Customer, Agent. "));
2190        assert!(prompt.contains("Include timestamps in [MM:SS] format"));
2191        assert!(
2192            prompt.contains("- A concise summary highlighting key points from each participant\n")
2193        );
2194        assert!(prompt
2195            .contains("- Identify and distinguish between these participants: Customer, Agent\n"));
2196    }
2197
2198    #[test]
2199    fn detected_participants_are_labelled_rather_than_named() {
2200        let session =
2201            testing::install(|_name, _params| Ok(json!({ "data": { "language": "en" } })));
2202
2203        let _ = transcribe(
2204            json!({ "file_hash": "abc", "mime_type": "audio/wav" }),
2205            TranscribeOptions {
2206                include_transcript: true,
2207                participants: Participants::Detect,
2208                ..Default::default()
2209            },
2210            Options::default(),
2211        )
2212        .unwrap();
2213
2214        let prompt = session.calls()[0].params["prompt"]
2215            .as_str()
2216            .unwrap_or_default()
2217            .to_string();
2218
2219        assert!(prompt.contains("label participants as Participant 1, Participant 2, etc."));
2220        assert!(prompt.contains("Format each line as \"Participant Name: text\".\n"));
2221        assert!(prompt.contains("- Identify and list all distinct participants in the audio\n"));
2222    }
2223
2224    #[test]
2225    fn naming_nobody_asks_for_the_same_thing_as_detecting() {
2226        let wanted = |participants| TranscribeOptions {
2227            include_transcript: true,
2228            participants,
2229            ..Default::default()
2230        };
2231
2232        assert_eq!(
2233            transcribe_prompt(&wanted(Participants::Named(Vec::new()))),
2234            transcribe_prompt(&wanted(Participants::Detect))
2235        );
2236        assert_eq!(
2237            transcribe_schema(&wanted(Participants::Named(Vec::new()))),
2238            transcribe_schema(&wanted(Participants::Detect))
2239        );
2240    }
2241
2242    #[test]
2243    fn a_transcription_that_cannot_run_is_refused_before_anything_is_sent() {
2244        let session = testing::install(|_name, _params| Ok(answered()));
2245
2246        let audio = json!({ "file_hash": "abc", "mime_type": "audio/mpeg" });
2247
2248        let nothing_wanted = transcribe(
2249            audio.clone(),
2250            TranscribeOptions::default(),
2251            Options::default(),
2252        )
2253        .unwrap_err();
2254
2255        let not_media = transcribe(
2256            json!({ "file_hash": "abc", "mime_type": "application/pdf" }),
2257            TranscribeOptions {
2258                include_transcript: true,
2259                ..Default::default()
2260            },
2261            Options::default(),
2262        )
2263        .unwrap_err();
2264
2265        let not_a_handle = transcribe(
2266            json!("some words"),
2267            TranscribeOptions {
2268                summarize: true,
2269                ..Default::default()
2270            },
2271            Options::default(),
2272        )
2273        .unwrap_err();
2274
2275        for refusal in [&nothing_wanted, &not_media, &not_a_handle] {
2276            assert_eq!(refusal.code().as_str(), "INVALID_TOOL_INPUT");
2277        }
2278
2279        assert!(session.calls().is_empty());
2280    }
2281
2282    #[test]
2283    fn a_page_transcription_asks_for_the_transcribe_operation_with_the_range_and_no_prompt() {
2284        let session = testing::install(|_name, _params| {
2285            Ok(json!({
2286                "data": {
2287                    "pages": [
2288                        { "page": 41, "text": "Scope of Work" },
2289                        { "page": 42, "error": "the answer carries no pages" }
2290                    ]
2291                },
2292                "metadata": {
2293                    "delivery": null,
2294                    "input_tokens": 0,
2295                    "output_tokens": 0,
2296                    "reasoning_tokens": 0
2297                }
2298            }))
2299        });
2300
2301        let read = transcribe_pages(
2302            &stored(),
2303            Some(Pages::new(40, 52)),
2304            TranscribePagesOptions::default(),
2305        )
2306        .unwrap();
2307
2308        let calls = session.calls();
2309
2310        assert_eq!(calls.len(), 1);
2311        assert_eq!(calls[0].name, ORCHESTRATOR);
2312        assert_eq!(
2313            calls[0].params,
2314            json!({
2315                "operation": "transcribe",
2316                "input": {
2317                    "file_hash": "ffce20f1",
2318                    "filename": "contract.pdf",
2319                    "mime_type": "application/pdf",
2320                    "size": 1_234,
2321                    "storage_path": "acme/documents/ffce20f1",
2322                    "first_page": 40,
2323                    "last_page": 52
2324                },
2325                "options": { "reasoning": true },
2326                "regenerate": false
2327            }),
2328            "no prompt, schema, model or delivery: the instructions are the platform's"
2329        );
2330
2331        assert_eq!(
2332            read.data,
2333            vec![
2334                PageTranscription::Read {
2335                    page: 41,
2336                    text: "Scope of Work".to_string()
2337                },
2338                PageTranscription::Unread {
2339                    page: 42,
2340                    error: "the answer carries no pages".to_string()
2341                },
2342            ]
2343        );
2344        assert_eq!(read.metadata.input_tokens, 0);
2345        assert!(read.metadata.delivery.is_empty());
2346    }
2347
2348    #[test]
2349    fn a_page_transcription_carries_only_the_whole_file_and_the_options_it_was_given() {
2350        let session = testing::install(|_name, _params| Ok(json!({ "data": { "pages": [] } })));
2351
2352        let read = transcribe_pages(
2353            &stored(),
2354            None,
2355            TranscribePagesOptions {
2356                regenerate: true,
2357                timeout: Some(Duration::from_secs(90)),
2358            },
2359        )
2360        .unwrap();
2361
2362        let sent = session.calls()[0].params.clone();
2363
2364        assert!(read.data.is_empty());
2365        assert!(sent["input"].get("first_page").is_none());
2366        assert!(sent["input"].get("last_page").is_none());
2367        assert_eq!(sent["regenerate"], json!(true));
2368        assert_eq!(sent["timeout"], json!(90_000));
2369        assert_eq!(sent["options"], json!({ "reasoning": true }));
2370    }
2371
2372    #[test]
2373    fn a_page_with_no_text_is_answered_as_unread_never_as_text_that_is_not_there() {
2374        let _session = testing::install(|_name, _params| {
2375            Ok(json!({
2376                "data": {
2377                    "pages": [
2378                        { "page": 7, "text": "" },
2379                        { "page": 8, "error": "the model timed out", "text": "Scope of" },
2380                        { "page": 9 },
2381                        { "page": 10, "transcriptions": ["Scope of Work", "Scope of Work"] },
2382                        { "page": 11, "error": null, "text": "Clause 4" },
2383                        { "page": 12, "error": 500, "text": "Clause 5" },
2384                        { "page": 13, "text": null },
2385                        { "text": "a page with no number" },
2386                        { "page": "14", "text": "a number that is a string" },
2387                        "page 15"
2388                    ]
2389                }
2390            }))
2391        });
2392
2393        let read = transcribe_pages(&stored(), None, TranscribePagesOptions::default()).unwrap();
2394
2395        let unread = |page: u32, error: &str| PageTranscription::Unread {
2396            page,
2397            error: error.to_string(),
2398        };
2399        let read_as = |page: u32, text: &str| PageTranscription::Read {
2400            page,
2401            text: text.to_string(),
2402        };
2403
2404        assert_eq!(
2405            read.data,
2406            vec![
2407                read_as(7, ""),
2408                unread(8, "the model timed out"),
2409                unread(9, NO_TRANSCRIPTION),
2410                unread(10, NO_TRANSCRIPTION),
2411                read_as(11, "Clause 4"),
2412                read_as(12, "Clause 5"),
2413                unread(13, NO_TRANSCRIPTION),
2414            ]
2415        );
2416        assert_eq!(read.data[1].text(), None, "an error wins over partial text");
2417        assert_eq!(read.data[0].text(), Some(""), "a blank page is a read");
2418        assert_eq!(read.data[2].page(), 9);
2419    }
2420
2421    #[test]
2422    fn an_answer_without_a_list_of_pages_has_no_pages() {
2423        for data in [
2424            json!(null),
2425            json!({}),
2426            json!({ "pages": "none" }),
2427            json!([]),
2428        ] {
2429            let _session =
2430                testing::install(move |_name, _params| Ok(json!({ "data": data.clone() })));
2431
2432            let read =
2433                transcribe_pages(&stored(), None, TranscribePagesOptions::default()).unwrap();
2434
2435            assert!(read.data.is_empty());
2436        }
2437    }
2438
2439    #[test]
2440    fn a_page_transcription_of_anything_but_a_pdf_is_refused_before_the_host_is_asked() {
2441        let session = testing::install(|_name, _params| Ok(json!({ "data": { "pages": [] } })));
2442
2443        let spreadsheet = transcribe_pages(
2444            &DocumentHandle {
2445                filename: "rates.csv".to_string(),
2446                mime_type: "text/csv".to_string(),
2447                ..stored()
2448            },
2449            None,
2450            TranscribePagesOptions::default(),
2451        )
2452        .unwrap_err();
2453
2454        let unstored = transcribe_pages(
2455            &DocumentHandle {
2456                file_hash: String::new(),
2457                ..stored()
2458            },
2459            None,
2460            TranscribePagesOptions::default(),
2461        )
2462        .unwrap_err();
2463
2464        for refusal in [&spreadsheet, &unstored] {
2465            assert_eq!(refusal.code().as_str(), "INVALID_TOOL_INPUT");
2466        }
2467
2468        assert!(spreadsheet.message().contains("reads the pages of a PDF"));
2469        assert!(session.calls().is_empty());
2470
2471        transcribe_pages(
2472            &DocumentHandle {
2473                mime_type: "Application/PDF; version=1.7".to_string(),
2474                ..stored()
2475            },
2476            None,
2477            TranscribePagesOptions::default(),
2478        )
2479        .expect("a PDF type with parameters or in capitals is still a PDF");
2480
2481        assert_eq!(session.calls().len(), 1);
2482    }
2483
2484    #[test]
2485    fn an_enrolment_answers_with_the_id_the_face_was_filed_under() {
2486        let session = testing::install(|name, params| {
2487            assert_eq!(name, FACE_ENROLL);
2488            assert_eq!(params["subject_id"], json!("EMP-42"));
2489            assert_eq!(params["image"], json!("iVBORw0KGgo"));
2490
2491            Ok(json!({ "face_id": "FACE-1" }))
2492        });
2493
2494        assert_eq!(
2495            enroll_face("EMP-42", json!("iVBORw0KGgo")).unwrap(),
2496            "FACE-1"
2497        );
2498        assert_eq!(session.calls().len(), 1);
2499    }
2500
2501    #[test]
2502    fn an_enrolment_uploads_a_pending_image_first() {
2503        let session = testing::install(|name, _params| {
2504            if name == UPLOAD_EPHEMERAL {
2505                return Ok(json!({ "file_hash": "abc", "storage_path": "ephemeral/abc" }));
2506            }
2507
2508            Ok(json!({ "face_id": "FACE-1" }))
2509        });
2510
2511        enroll_face("EMP-42", json!({ "file_hash": "abc", "pending": true })).unwrap();
2512
2513        let calls = session.calls();
2514
2515        assert_eq!(calls[0].name, UPLOAD_EPHEMERAL);
2516        assert_eq!(
2517            calls[1].params["image"],
2518            json!({ "file_hash": "abc", "storage_path": "ephemeral/abc" })
2519        );
2520    }
2521
2522    #[test]
2523    fn an_enrolment_without_a_subject_or_an_image_is_refused() {
2524        let session = testing::install(|_name, _params| Ok(json!({ "face_id": "FACE-1" })));
2525
2526        assert_eq!(
2527            enroll_face("  ", json!("image"))
2528                .unwrap_err()
2529                .code()
2530                .as_str(),
2531            "INVALID_TOOL_INPUT"
2532        );
2533        assert_eq!(
2534            enroll_face("EMP-42", json!(null))
2535                .unwrap_err()
2536                .code()
2537                .as_str(),
2538            "INVALID_TOOL_INPUT"
2539        );
2540        assert!(session.calls().is_empty());
2541    }
2542
2543    #[test]
2544    fn an_enrolment_that_answers_without_an_id_is_reported_rather_than_guessed_at() {
2545        let _session = testing::install(|_name, _params| Ok(json!({ "enrolled": true })));
2546
2547        let error = enroll_face("EMP-42", json!("image")).unwrap_err();
2548
2549        assert!(matches!(error, Error::Json(_)));
2550        assert_eq!(error.code().as_str(), "ACTION_FAILED");
2551    }
2552
2553    #[test]
2554    fn a_search_carries_only_the_narrowing_it_was_given() {
2555        let session = testing::install(|name, params| {
2556            assert_eq!(name, FACE_SEARCH);
2557
2558            Ok(json!({ "matches": params["options"].clone() }))
2559        });
2560
2561        let wide: Value = search_face(json!("image"), FaceSearch::default()).unwrap();
2562
2563        let narrow: Value = search_face(
2564            json!("image"),
2565            FaceSearch {
2566                max_faces: Some(3),
2567                similarity_threshold: Some(92.5),
2568            },
2569        )
2570        .unwrap();
2571
2572        assert_eq!(wide["matches"], json!({}), "the platform's own defaults");
2573        assert_eq!(
2574            narrow["matches"],
2575            json!({ "max_faces": 3, "similarity_threshold": 92.5 })
2576        );
2577        assert_eq!(session.calls().len(), 2);
2578    }
2579
2580    #[test]
2581    fn a_search_answers_with_the_type_the_call_asked_for() {
2582        #[derive(Debug, Deserialize, PartialEq)]
2583        struct Match {
2584            similarity: f64,
2585            subject_id: String,
2586        }
2587
2588        let _session = testing::install(|_name, _params| {
2589            Ok(json!([{ "similarity": 99.1, "subject_id": "EMP-42" }]))
2590        });
2591
2592        let found: Vec<Match> = search_face(json!("image"), FaceSearch::default()).unwrap();
2593
2594        assert_eq!(
2595            found,
2596            vec![Match {
2597                similarity: 99.1,
2598                subject_id: "EMP-42".to_string()
2599            }]
2600        );
2601    }
2602
2603    #[test]
2604    fn a_search_with_no_image_is_refused_before_anything_is_sent() {
2605        let session = testing::install(|_name, _params| Ok(json!([])));
2606
2607        let error = search_face::<Value>(json!(null), FaceSearch::default()).unwrap_err();
2608
2609        assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
2610        assert!(session.calls().is_empty());
2611    }
2612
2613    #[test]
2614    fn a_removal_reports_the_ids_the_platform_says_it_removed() {
2615        let session = testing::install(|name, params| {
2616            assert_eq!(name, FACE_DELETE);
2617            assert_eq!(params["face_ids"], json!(["FACE-1", "FACE-2"]));
2618
2619            Ok(json!({ "deleted": ["FACE-1"] }))
2620        });
2621
2622        let removed = delete_face(&["FACE-1".to_string(), "FACE-2".to_string()]).unwrap();
2623
2624        assert_eq!(removed, ["FACE-1"]);
2625        assert_eq!(session.calls().len(), 1);
2626    }
2627
2628    #[test]
2629    fn a_removal_reads_a_bare_list_of_ids_as_well() {
2630        let _session = testing::install(|_name, _params| Ok(json!(["FACE-1", "FACE-2"])));
2631
2632        assert_eq!(
2633            delete_face(&["FACE-1".to_string()]).unwrap(),
2634            ["FACE-1", "FACE-2"]
2635        );
2636    }
2637
2638    #[test]
2639    fn a_removal_that_names_nothing_removed_answers_with_nothing() {
2640        let _session = testing::install(|_name, _params| Ok(json!({ "ok": true })));
2641
2642        assert!(delete_face(&["FACE-1".to_string()]).unwrap().is_empty());
2643    }
2644
2645    #[test]
2646    fn a_removal_with_no_ids_is_refused_before_anything_is_sent() {
2647        let session = testing::install(|_name, _params| Ok(json!([])));
2648
2649        let error = delete_face(&[]).unwrap_err();
2650
2651        assert_eq!(error.code().as_str(), "INVALID_TOOL_INPUT");
2652        assert!(session.calls().is_empty());
2653    }
2654
2655    #[test]
2656    fn what_the_host_said_survives_into_every_failure() {
2657        let _session = testing::install(|_name, _params| Err(Error::failed("the engine is busy")));
2658
2659        let operation = summarize(json!("text"), "One sentence.", Options::default()).unwrap_err();
2660        let enrolment = enroll_face("EMP-42", json!("image")).unwrap_err();
2661        let search = search_face::<Value>(json!("image"), FaceSearch::default()).unwrap_err();
2662        let removal = delete_face(&["FACE-1".to_string()]).unwrap_err();
2663
2664        for failure in [&operation, &enrolment, &search, &removal] {
2665            assert!(failure.message().contains("the engine is busy"));
2666            assert!(!failure.fault().hint().is_empty(), "and says what to do");
2667        }
2668    }
2669
2670    #[test]
2671    fn an_answer_that_does_not_fit_the_type_is_reported_as_having_run() {
2672        let _session = testing::install(|_name, _params| Ok(json!({ "data": { "total": 240.0 } })));
2673
2674        let error = summarize(json!("text"), "One sentence.", Options::default()).unwrap_err();
2675
2676        assert!(matches!(error, Error::Json(_)));
2677        assert_eq!(error.code().as_str(), "ACTION_FAILED");
2678        assert!(error
2679            .fault()
2680            .hint()
2681            .contains("rather than running it again"));
2682    }
2683
2684    #[test]
2685    fn a_timeout_travels_as_the_milliseconds_the_wire_carries() {
2686        assert_eq!(milliseconds(Duration::from_secs(30)), 30_000);
2687        assert_eq!(milliseconds(Duration::from_millis(1)), 1);
2688        assert_eq!(milliseconds(Duration::MAX), u64::MAX);
2689    }
2690
2691    #[test]
2692    fn every_model_names_itself_as_the_wire_spells_it() {
2693        assert_eq!(Model::Lite.to_string(), "lite");
2694        assert_eq!(Model::Medium.to_string(), "medium");
2695        assert_eq!(Model::Large.to_string(), "large");
2696        assert_eq!(Model::Xl.to_string(), "xl");
2697    }
2698}