burn_router/client/base.rs
1use crate::{RouterChannel, RouterTensor};
2use alloc::boxed::Box;
3use alloc::vec::Vec;
4use burn_backend::{
5 DType, ProfileDuration, ProfileOptions, ProfileToken, TensorData,
6 backend::{DeviceId, DeviceOps, ExecutionError},
7};
8use burn_ir::{GraphBindings, GraphId, OperationIr, TensorId, TensorIr};
9use burn_std::future::DynFut;
10use core::{marker::PhantomData, ops::DerefMut};
11use hashbrown::HashMap;
12use spin::Mutex;
13
14/// Type alias for `<R as RouterChannel>::Client`.
15pub type Client<R> = <R as RouterChannel>::Client;
16pub(crate) static CLIENTS: RouterClientLocator = RouterClientLocator::new();
17
18type Key = (core::any::TypeId, DeviceId);
19
20/// Define how to interact with the interpreter.
21pub trait RouterClient: Clone + Send + Sync + Sized {
22 /// Device type.
23 type Device: DeviceOps;
24
25 /// Register a new tensor operation to be executed by the interpreter (server).
26 fn register_op(&self, op: OperationIr);
27 /// Register a new tensor operation to be executed by the interpreter (server).
28 ///
29 /// Returns the new (uninitialized) output tensor(s) generated by the registered operation.
30 fn register(&self, op: OperationIr) -> Vec<RouterTensor<Self>> {
31 let out = op
32 .outputs()
33 .map(|output| {
34 RouterTensor::new(output.id, output.shape.clone(), output.dtype, self.clone())
35 })
36 .collect();
37 self.register_op(op);
38
39 out
40 }
41 /// Read the values contained by a tensor.
42 fn read_tensor_async(&self, tensor: TensorIr) -> DynFut<Result<TensorData, ExecutionError>>;
43 /// Sync the interpreter, ensure that all computations are finished.
44 fn sync(&self) -> Result<(), ExecutionError>;
45 /// Eagerly submit the operations registered so far without waiting for them to complete.
46 ///
47 /// Unlike [`sync`](Self::sync), this does not block on results — it only ensures the
48 /// buffered operations are handed off for execution (and, for the remote backend, sent to
49 /// the server) instead of sitting in a local buffer.
50 fn flush(&self);
51 /// Create a new (uninitialized) empty tensor and returns its corresponding [tensor id](TensorId).
52 fn create_empty_handle(&self) -> TensorId;
53 /// Create a new [RouterTensor] from the tensor data.
54 fn register_tensor_data(&self, data: TensorData) -> RouterTensor<Self>;
55 /// Get the current device used by all operations handled by this client.
56 fn device(&self) -> Self::Device;
57 /// Seed the interpreter.
58 fn seed(&self, seed: u64);
59 /// Returns the supported data type usage set
60 fn dtype_usage(&self, dtype: DType) -> burn_backend::DTypeUsageSet;
61 /// Open a profiling window on the interpreter, where the calling stream
62 /// stands — see [`Backend::profile_start`](burn_backend::Backend::profile_start).
63 ///
64 /// `None`, the default, from an interpreter that opens no windows.
65 fn profile_start(&self) -> Result<Option<ProfileToken>, ExecutionError> {
66 Ok(None)
67 }
68 /// Close the window `token` where the calling stream stands, flushing the
69 /// interpreter's backend first when `options` ask for it.
70 fn profile_end(
71 &self,
72 token: ProfileToken,
73 options: ProfileOptions,
74 ) -> Result<ProfileDuration, ExecutionError> {
75 let _ = (token, options);
76 Err(ExecutionError::with_context(
77 "profiling windows are not supported by this interpreter",
78 ))
79 }
80
81 /// Drop the window `token` without measuring it, for a caller that will
82 /// never reach [`profile_end`](Self::profile_end) — see
83 /// [`Backend::profile_abandon`](burn_backend::Backend::profile_abandon).
84 ///
85 /// The default closes the window and discards the measurement, which any
86 /// interpreter that opens one can already do. An interpreter that can say
87 /// so more cheaply — a remote one, where the close is a round trip and
88 /// the caller is unwinding — does that instead.
89 fn profile_abandon(&self, token: ProfileToken) {
90 let _ = self.profile_end(token, ProfileOptions::default());
91 }
92
93 /// Register a reusable group of operations (in relative form) under `graph_id` *and* run its
94 /// first invocation with `bindings`, so it can later be replayed by id with
95 /// [`execute_graph`](Self::execute_graph).
96 ///
97 /// Registration always coincides with the first execution, so they're combined to save a
98 /// round-trip on a cache miss. Used by the fusion layer to avoid re-sending a recurring
99 /// op-graph.
100 fn register_and_execute_graph(
101 &self,
102 graph_id: GraphId,
103 relative_graph: Vec<OperationIr>,
104 bindings: GraphBindings,
105 );
106
107 /// Replay a previously [registered](Self::register_and_execute_graph) graph with the given
108 /// concrete bindings.
109 fn execute_graph(&self, graph_id: GraphId, bindings: GraphBindings);
110
111 /// Register `new_id` as an alias of `src_id` — a second handle over the same backing buffer.
112 ///
113 /// Used by the fusion layer's cross-stream sharing (see
114 /// `burn_fusion::FusionRuntime::alias_handle`): when a tensor is
115 /// shared to another stream, that stream's view needs its own id so that consuming it (a
116 /// `ReadWrite` last-use) frees only this alias, leaving the original handle valid. The server
117 /// clones the source handle (an `Arc`-style refcount on the device buffer) under `new_id`.
118 fn register_alias(&self, new_id: TensorId, src_id: TensorId);
119}
120
121pub(crate) struct RouterClientLocator {
122 clients: Mutex<Option<HashMap<Key, Box<dyn core::any::Any + Send>>>>,
123}
124
125/// Get the client currently associated with `device`.
126///
127/// An existing scoped or unscoped registration is returned as-is. On a cache miss, the channel's
128/// [`RouterChannel::init_client`] implementation creates an unscoped client that remains cached in
129/// the global locator. Use [`register_scoped_client`] when the caller, rather than the locator,
130/// must control the client's lifetime.
131pub fn get_client<R: RouterChannel>(device: &R::Device) -> Client<R> {
132 CLIENTS.client::<R>(device)
133}
134
135/// Guard owning a router client's registration in the global locator.
136///
137/// The locator stores a clone of the registered client; this guard stores the corresponding lookup
138/// key. Dropping the guard removes that clone. Tensor handles remain safe because they own client
139/// clones independently of the locator. Most router clients are process-long lived and use
140/// [`get_client`]; lifecycle-owned channels use [`register_scoped_client`] and retain this guard.
141#[must_use = "dropping the registration immediately unregisters its router client"]
142pub struct RouterClientRegistration<R: RouterChannel> {
143 key: Key,
144 _channel: PhantomData<R>,
145}
146
147impl<R: RouterChannel> Drop for RouterClientRegistration<R> {
148 fn drop(&mut self) {
149 CLIENTS.remove(self.key);
150 }
151}
152
153/// Register `client` with a locator entry owned by the returned guard.
154///
155/// This inserts the caller-created client directly and does not call
156/// [`RouterChannel::init_client`]. While the guard is alive, [`get_client`] for the same channel and
157/// device returns a clone of this client. Dropping the guard removes the locator entry, allowing a
158/// later scope to associate the same device with a different client.
159///
160/// Unlike [`get_client`], this requires the device not to have a registered client already. The
161/// guard gives one lifecycle owner responsibility for cleanup and prevents exposing unrestricted
162/// client removal to downstream crates. Returns `None` when the device already has a client.
163pub fn register_scoped_client<R: RouterChannel>(
164 device: &R::Device,
165 client: Client<R>,
166) -> Option<RouterClientRegistration<R>> {
167 CLIENTS
168 .register_scoped::<R>(device, client)
169 .map(|key| RouterClientRegistration {
170 key,
171 _channel: PhantomData,
172 })
173}
174
175/// Initialize a new client for the given device.
176///
177/// If a (global) seed was previously set, the client seed is set.
178fn new_client<R: RouterChannel>(device: &R::Device) -> Client<R> {
179 R::init_client(device)
180}
181
182impl RouterClientLocator {
183 /// Create a new client locator.
184 pub const fn new() -> Self {
185 Self {
186 clients: Mutex::new(None),
187 }
188 }
189
190 /// Get the router client for the given device.
191 ///
192 /// If a client isn't already initialized, it is created.
193 pub fn client<R: RouterChannel + 'static>(&self, device: &R::Device) -> Client<R> {
194 let device_id = device.id();
195 let client_id = (core::any::TypeId::of::<R>(), device_id);
196 let mut clients = self.clients.lock();
197
198 if clients.is_none() {
199 let client = new_client::<R>(device);
200 Self::register_inner::<R>(client_id, client, &mut clients);
201 }
202
203 match clients.deref_mut() {
204 Some(clients) => match clients.get(&client_id) {
205 Some(client) => {
206 let client: &Client<R> = client.downcast_ref().unwrap();
207 client.clone()
208 }
209 None => {
210 let client = new_client::<R>(device);
211 let any = Box::new(client.clone());
212 clients.insert(client_id, any);
213 client
214 }
215 },
216 _ => unreachable!(),
217 }
218 }
219
220 /// Register a client with a unique lifecycle owner.
221 fn register_scoped<R: RouterChannel + 'static>(
222 &self,
223 device: &R::Device,
224 client: Client<R>,
225 ) -> Option<Key> {
226 let key = (core::any::TypeId::of::<R>(), device.id());
227 let mut clients = self.clients.lock();
228 let clients = clients.get_or_insert_with(HashMap::new);
229 if clients.contains_key(&key) {
230 return None;
231 }
232 clients.insert(key, Box::new(client));
233 Some(key)
234 }
235
236 /// Remove the client identified by a scoped registration guard.
237 fn remove(&self, key: Key) {
238 let mut clients = self.clients.lock();
239 if let Some(clients) = clients.as_mut() {
240 clients.remove(&key);
241 }
242 }
243
244 fn register_inner<R: RouterChannel + 'static>(
245 key: Key,
246 client: Client<R>,
247 clients: &mut Option<HashMap<Key, Box<dyn core::any::Any + Send>>>,
248 ) {
249 if clients.is_none() {
250 *clients = Some(HashMap::new());
251 }
252
253 if let Some(clients) = clients {
254 if clients.contains_key(&key) {
255 panic!("Client already created for device {key:?}");
256 }
257
258 clients.insert(key, Box::new(client));
259 }
260 }
261}