Skip to main content

burn_router/client/
base.rs

1use crate::{RouterChannel, RouterTensor};
2use alloc::boxed::Box;
3use alloc::vec::Vec;
4use burn_backend::{
5    DType, ProfileDuration, ProfileOptions, ProfileToken, TensorData,
6    backend::{DeviceId, DeviceOps, ExecutionError},
7};
8use burn_ir::{GraphBindings, GraphId, OperationIr, TensorId, TensorIr};
9use burn_std::future::DynFut;
10use core::{marker::PhantomData, ops::DerefMut};
11use hashbrown::HashMap;
12use spin::Mutex;
13
14/// Type alias for `<R as RouterChannel>::Client`.
15pub type Client<R> = <R as RouterChannel>::Client;
16pub(crate) static CLIENTS: RouterClientLocator = RouterClientLocator::new();
17
18type Key = (core::any::TypeId, DeviceId);
19
20/// Define how to interact with the interpreter.
21pub trait RouterClient: Clone + Send + Sync + Sized {
22    /// Device type.
23    type Device: DeviceOps;
24
25    /// Register a new tensor operation to be executed by the interpreter (server).
26    fn register_op(&self, op: OperationIr);
27    /// Register a new tensor operation to be executed by the interpreter (server).
28    ///
29    /// Returns the new (uninitialized) output tensor(s) generated by the registered operation.
30    fn register(&self, op: OperationIr) -> Vec<RouterTensor<Self>> {
31        let out = op
32            .outputs()
33            .map(|output| {
34                RouterTensor::new(output.id, output.shape.clone(), output.dtype, self.clone())
35            })
36            .collect();
37        self.register_op(op);
38
39        out
40    }
41    /// Read the values contained by a tensor.
42    fn read_tensor_async(&self, tensor: TensorIr) -> DynFut<Result<TensorData, ExecutionError>>;
43    /// Sync the interpreter, ensure that all computations are finished.
44    fn sync(&self) -> Result<(), ExecutionError>;
45    /// Eagerly submit the operations registered so far without waiting for them to complete.
46    ///
47    /// Unlike [`sync`](Self::sync), this does not block on results — it only ensures the
48    /// buffered operations are handed off for execution (and, for the remote backend, sent to
49    /// the server) instead of sitting in a local buffer.
50    fn flush(&self);
51    /// Create a new (uninitialized) empty tensor and returns its corresponding [tensor id](TensorId).
52    fn create_empty_handle(&self) -> TensorId;
53    /// Create a new [RouterTensor] from the tensor data.
54    fn register_tensor_data(&self, data: TensorData) -> RouterTensor<Self>;
55    /// Get the current device used by all operations handled by this client.
56    fn device(&self) -> Self::Device;
57    /// Seed the interpreter.
58    fn seed(&self, seed: u64);
59    /// Returns the supported data type usage set
60    fn dtype_usage(&self, dtype: DType) -> burn_backend::DTypeUsageSet;
61    /// Open a profiling window on the interpreter, where the calling stream
62    /// stands — see [`Backend::profile_start`](burn_backend::Backend::profile_start).
63    ///
64    /// `None`, the default, from an interpreter that opens no windows.
65    fn profile_start(&self) -> Result<Option<ProfileToken>, ExecutionError> {
66        Ok(None)
67    }
68    /// Close the window `token` where the calling stream stands, flushing the
69    /// interpreter's backend first when `options` ask for it.
70    fn profile_end(
71        &self,
72        token: ProfileToken,
73        options: ProfileOptions,
74    ) -> Result<ProfileDuration, ExecutionError> {
75        let _ = (token, options);
76        Err(ExecutionError::with_context(
77            "profiling windows are not supported by this interpreter",
78        ))
79    }
80
81    /// Drop the window `token` without measuring it, for a caller that will
82    /// never reach [`profile_end`](Self::profile_end) — see
83    /// [`Backend::profile_abandon`](burn_backend::Backend::profile_abandon).
84    ///
85    /// The default closes the window and discards the measurement, which any
86    /// interpreter that opens one can already do. An interpreter that can say
87    /// so more cheaply — a remote one, where the close is a round trip and
88    /// the caller is unwinding — does that instead.
89    fn profile_abandon(&self, token: ProfileToken) {
90        let _ = self.profile_end(token, ProfileOptions::default());
91    }
92
93    /// Register a reusable group of operations (in relative form) under `graph_id` *and* run its
94    /// first invocation with `bindings`, so it can later be replayed by id with
95    /// [`execute_graph`](Self::execute_graph).
96    ///
97    /// Registration always coincides with the first execution, so they're combined to save a
98    /// round-trip on a cache miss. Used by the fusion layer to avoid re-sending a recurring
99    /// op-graph.
100    fn register_and_execute_graph(
101        &self,
102        graph_id: GraphId,
103        relative_graph: Vec<OperationIr>,
104        bindings: GraphBindings,
105    );
106
107    /// Replay a previously [registered](Self::register_and_execute_graph) graph with the given
108    /// concrete bindings.
109    fn execute_graph(&self, graph_id: GraphId, bindings: GraphBindings);
110
111    /// Register `new_id` as an alias of `src_id` — a second handle over the same backing buffer.
112    ///
113    /// Used by the fusion layer's cross-stream sharing (see
114    /// `burn_fusion::FusionRuntime::alias_handle`): when a tensor is
115    /// shared to another stream, that stream's view needs its own id so that consuming it (a
116    /// `ReadWrite` last-use) frees only this alias, leaving the original handle valid. The server
117    /// clones the source handle (an `Arc`-style refcount on the device buffer) under `new_id`.
118    fn register_alias(&self, new_id: TensorId, src_id: TensorId);
119}
120
121pub(crate) struct RouterClientLocator {
122    clients: Mutex<Option<HashMap<Key, Box<dyn core::any::Any + Send>>>>,
123}
124
125/// Get the client currently associated with `device`.
126///
127/// An existing scoped or unscoped registration is returned as-is. On a cache miss, the channel's
128/// [`RouterChannel::init_client`] implementation creates an unscoped client that remains cached in
129/// the global locator. Use [`register_scoped_client`] when the caller, rather than the locator,
130/// must control the client's lifetime.
131pub fn get_client<R: RouterChannel>(device: &R::Device) -> Client<R> {
132    CLIENTS.client::<R>(device)
133}
134
135/// Guard owning a router client's registration in the global locator.
136///
137/// The locator stores a clone of the registered client; this guard stores the corresponding lookup
138/// key. Dropping the guard removes that clone. Tensor handles remain safe because they own client
139/// clones independently of the locator. Most router clients are process-long lived and use
140/// [`get_client`]; lifecycle-owned channels use [`register_scoped_client`] and retain this guard.
141#[must_use = "dropping the registration immediately unregisters its router client"]
142pub struct RouterClientRegistration<R: RouterChannel> {
143    key: Key,
144    _channel: PhantomData<R>,
145}
146
147impl<R: RouterChannel> Drop for RouterClientRegistration<R> {
148    fn drop(&mut self) {
149        CLIENTS.remove(self.key);
150    }
151}
152
153/// Register `client` with a locator entry owned by the returned guard.
154///
155/// This inserts the caller-created client directly and does not call
156/// [`RouterChannel::init_client`]. While the guard is alive, [`get_client`] for the same channel and
157/// device returns a clone of this client. Dropping the guard removes the locator entry, allowing a
158/// later scope to associate the same device with a different client.
159///
160/// Unlike [`get_client`], this requires the device not to have a registered client already. The
161/// guard gives one lifecycle owner responsibility for cleanup and prevents exposing unrestricted
162/// client removal to downstream crates. Returns `None` when the device already has a client.
163pub fn register_scoped_client<R: RouterChannel>(
164    device: &R::Device,
165    client: Client<R>,
166) -> Option<RouterClientRegistration<R>> {
167    CLIENTS
168        .register_scoped::<R>(device, client)
169        .map(|key| RouterClientRegistration {
170            key,
171            _channel: PhantomData,
172        })
173}
174
175/// Initialize a new client for the given device.
176///
177/// If a (global) seed was previously set, the client seed is set.
178fn new_client<R: RouterChannel>(device: &R::Device) -> Client<R> {
179    R::init_client(device)
180}
181
182impl RouterClientLocator {
183    /// Create a new client locator.
184    pub const fn new() -> Self {
185        Self {
186            clients: Mutex::new(None),
187        }
188    }
189
190    /// Get the router client for the given device.
191    ///
192    /// If a client isn't already initialized, it is created.
193    pub fn client<R: RouterChannel + 'static>(&self, device: &R::Device) -> Client<R> {
194        let device_id = device.id();
195        let client_id = (core::any::TypeId::of::<R>(), device_id);
196        let mut clients = self.clients.lock();
197
198        if clients.is_none() {
199            let client = new_client::<R>(device);
200            Self::register_inner::<R>(client_id, client, &mut clients);
201        }
202
203        match clients.deref_mut() {
204            Some(clients) => match clients.get(&client_id) {
205                Some(client) => {
206                    let client: &Client<R> = client.downcast_ref().unwrap();
207                    client.clone()
208                }
209                None => {
210                    let client = new_client::<R>(device);
211                    let any = Box::new(client.clone());
212                    clients.insert(client_id, any);
213                    client
214                }
215            },
216            _ => unreachable!(),
217        }
218    }
219
220    /// Register a client with a unique lifecycle owner.
221    fn register_scoped<R: RouterChannel + 'static>(
222        &self,
223        device: &R::Device,
224        client: Client<R>,
225    ) -> Option<Key> {
226        let key = (core::any::TypeId::of::<R>(), device.id());
227        let mut clients = self.clients.lock();
228        let clients = clients.get_or_insert_with(HashMap::new);
229        if clients.contains_key(&key) {
230            return None;
231        }
232        clients.insert(key, Box::new(client));
233        Some(key)
234    }
235
236    /// Remove the client identified by a scoped registration guard.
237    fn remove(&self, key: Key) {
238        let mut clients = self.clients.lock();
239        if let Some(clients) = clients.as_mut() {
240            clients.remove(&key);
241        }
242    }
243
244    fn register_inner<R: RouterChannel + 'static>(
245        key: Key,
246        client: Client<R>,
247        clients: &mut Option<HashMap<Key, Box<dyn core::any::Any + Send>>>,
248    ) {
249        if clients.is_none() {
250            *clients = Some(HashMap::new());
251        }
252
253        if let Some(clients) = clients {
254            if clients.contains_key(&key) {
255                panic!("Client already created for device {key:?}");
256            }
257
258            clients.insert(key, Box::new(client));
259        }
260    }
261}