openai_api_dispatch/queue/
mod.rs

1use serde::{Deserialize, Serialize};
2
3use crate::task::{Response, Task};
4
5#[cfg(feature = "memory-queue")]
6/// Shared in-process queues with different `std` and `no_std` behavior.
7pub mod memory;
8
9#[cfg(feature = "nats-queue")]
10/// JSON request/reply over Core NATS; requires `std` and a Tokio runtime.
11pub mod nats;
12
13/// Submits tasks and retrieves their responses.
14///
15/// Waiting and cancellation semantics depend on the backend. Returned futures
16/// do not carry a `Send` guarantee; this trait is not dyn-compatible.
17pub trait QueueProducer {
18    /// Backend-specific handle used to correlate a submitted task's response.
19    type Message;
20
21    /// Submits a task and returns its response handle, not its execution result.
22    fn send_task(&self, task: Task) -> impl Future<Output = anyhow::Result<Self::Message>>;
23
24    /// Consumes a handle to retrieve a response.
25    ///
26    /// `None` may mean no response is ready (`no_std` memory) or the response
27    /// stream ended (NATS); consult the backend before treating it as terminal.
28    fn receive_response(
29        &self,
30        message: Self::Message,
31    ) -> impl Future<Output = anyhow::Result<Option<Response>>>;
32}
33
34#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
35/// A task paired with the backend metadata needed to route its reply.
36pub struct WrappedTask<T> {
37    /// Reply metadata; pass it unchanged to [`QueueWorker::send_response`].
38    pub message: T,
39    /// Work to execute.
40    pub task: Task,
41}
42
43/// Receives queued tasks and routes their responses back to producers.
44///
45/// Returned futures do not carry a `Send` guarantee; this trait is not
46/// dyn-compatible.
47pub trait QueueWorker {
48    /// Backend-specific reply metadata attached to each received task.
49    type Message;
50
51    /// Receives a task, or `None` when none is available or the stream ends.
52    ///
53    /// NATS and `std` memory wait for work; `no_std` memory polls once.
54    fn receive_task(
55        &self,
56    ) -> impl Future<Output = anyhow::Result<Option<WrappedTask<Self::Message>>>>;
57
58    /// Sends a response using the metadata from the corresponding task.
59    fn send_response(
60        &self,
61        message: Self::Message,
62        response: Response,
63    ) -> impl Future<Output = anyhow::Result<()>>;
64}
openai_api_dispatch/queue/mod.rs

openai_api_dispatch/queue/
mod.rs
