pub trait ComputeServer:
Send
+ Debug
+ ServerCommunication
+ DeviceService
+ 'staticwhere
Self: Sized,{
type Kernel: KernelMetadata;
type Info: Debug + Send + Sync;
type MemoryLayoutPolicy: MemoryLayoutPolicy;
type Storage: ComputeStorage;
Show 38 methods
// Required methods
fn initialize_memory(
&mut self,
memory: ManagedMemoryHandle,
size: u64,
stream_id: StreamId,
);
fn logger(&self) -> Arc<ServerLogger> ⓘ;
fn utilities(&self) -> Arc<ServerUtilities<Self>> ⓘ;
fn read(
&mut self,
descriptors: Vec<CopyDescriptor>,
stream_id: StreamId,
) -> DynFut<Result<Vec<Bytes>, ServerError>>;
fn write(
&mut self,
descriptors: Vec<(CopyDescriptor, Bytes)>,
stream_id: StreamId,
);
fn sync(&mut self, stream_id: StreamId) -> DynFut<Result<(), ServerError>>;
fn get_resource(
&mut self,
binding: Binding,
stream_id: StreamId,
) -> Result<ManagedResource<<Self::Storage as ComputeStorage>::Resource>, ServerError>;
unsafe fn launch(
&mut self,
kernel: Self::Kernel,
count: CubeCount,
bindings: KernelArguments,
kind: ExecutionMode,
stream_id: StreamId,
);
fn flush(&mut self, stream_id: StreamId) -> Result<(), ServerError>;
fn memory_usage(
&mut self,
stream_id: StreamId,
) -> Result<MemoryUsage, ServerError>;
fn memory_cleanup(&mut self, stream_id: StreamId);
fn start_profile(
&mut self,
stream_id: StreamId,
) -> Result<ProfilingToken, ServerError>;
fn end_profile(
&mut self,
stream_id: StreamId,
token: ProfilingToken,
) -> Result<ProfileDuration, ProfileError>;
fn allocation_mode(
&mut self,
mode: MemoryAllocationMode,
stream_id: StreamId,
);
// Provided methods
fn graph_capture_supported(&self) -> bool { ... }
fn graph_defer_frees(&mut self, _defer: bool, _stream_id: StreamId) { ... }
fn graph_arena_begin(&mut self, _stream_id: StreamId) -> u64 { ... }
fn graph_arena_end(&mut self, _stream_id: StreamId) { ... }
fn graph_arena_stats(&mut self, _stream_id: StreamId) -> ArenaStats { ... }
fn graph_arena_reset_counters(&mut self, _stream_id: StreamId) { ... }
fn reserve_timing(&mut self, _stream_id: StreamId) -> (u64, u64) { ... }
fn reserve_timing_reset(&mut self, _stream_id: StreamId) { ... }
fn graph_capture_begin(&mut self, _stream_id: StreamId) { ... }
fn graph_capture_end(&mut self, _stream_id: StreamId) -> u64 { ... }
fn graph_replay(&mut self, _id: u64, _stream_id: StreamId) { ... }
fn graph_capture_status(&mut self, _stream_id: StreamId) -> u32 { ... }
fn graph_node_count(&mut self, _id: u64) -> usize { ... }
fn graph_capture_launch_count(&mut self) -> usize { ... }
fn graph_launch_count(&mut self, _id: u64) -> usize { ... }
fn graph_launch_params(
&mut self,
_id: u64,
_idx: usize,
) -> GraphLaunchParams { ... }
fn graph_patch_launch(
&mut self,
_id: u64,
_idx: usize,
_patch: GraphLaunchPatch,
) { ... }
fn graph_node_kinds(&mut self, _id: u64) -> Vec<(u32, usize)> { ... }
fn graph_patch_launches(
&mut self,
_id: u64,
_patches: Vec<(usize, GraphLaunchPatch)>,
) { ... }
fn graph_alloc_regions(&mut self, _id: u64) -> Vec<(u64, u64)> { ... }
fn graph_memcpy_specs(&mut self, _id: u64) -> Vec<(u64, u64, u64, u32)> { ... }
fn graph_destroy(&mut self, _id: u64) { ... }
fn register_external_aliased(
&mut self,
_ptr: *mut c_void,
_page_len: u64,
_offset: u64,
_size: u64,
_keepalive: Arc<dyn Any + Send + Sync>,
_stream_id: StreamId,
) -> Handle { ... }
fn staging(
&mut self,
_sizes: &[usize],
_stream_id: StreamId,
) -> Result<Vec<Bytes>, ServerError> { ... }
}Expand description
The compute server is responsible for handling resources and computations over resources.
Everything in the server is mutable, therefore it should be solely accessed through the
ComputeClient for thread safety.
Required Associated Types§
Sourcetype Kernel: KernelMetadata
type Kernel: KernelMetadata
The kernel type defines the computation algorithms.
Sourcetype MemoryLayoutPolicy: MemoryLayoutPolicy
type MemoryLayoutPolicy: MemoryLayoutPolicy
Manages how allocations are performed for a server.
Sourcetype Storage: ComputeStorage
type Storage: ComputeStorage
The storage type defines how data is stored and accessed.
Required Methods§
Sourcefn initialize_memory(
&mut self,
memory: ManagedMemoryHandle,
size: u64,
stream_id: StreamId,
)
fn initialize_memory( &mut self, memory: ManagedMemoryHandle, size: u64, stream_id: StreamId, )
Sourcefn logger(&self) -> Arc<ServerLogger> ⓘ
fn logger(&self) -> Arc<ServerLogger> ⓘ
Retrieve the server logger.
Sourcefn utilities(&self) -> Arc<ServerUtilities<Self>> ⓘ
fn utilities(&self) -> Arc<ServerUtilities<Self>> ⓘ
Retrieve the server utilities.
Sourcefn read(
&mut self,
descriptors: Vec<CopyDescriptor>,
stream_id: StreamId,
) -> DynFut<Result<Vec<Bytes>, ServerError>>
fn read( &mut self, descriptors: Vec<CopyDescriptor>, stream_id: StreamId, ) -> DynFut<Result<Vec<Bytes>, ServerError>>
Given bindings, returns the owned resources as bytes.
Sourcefn write(
&mut self,
descriptors: Vec<(CopyDescriptor, Bytes)>,
stream_id: StreamId,
)
fn write( &mut self, descriptors: Vec<(CopyDescriptor, Bytes)>, stream_id: StreamId, )
Writes the specified bytes into the buffers given
Sourcefn sync(&mut self, stream_id: StreamId) -> DynFut<Result<(), ServerError>>
fn sync(&mut self, stream_id: StreamId) -> DynFut<Result<(), ServerError>>
Wait for the completion of every task in the server.
Sourcefn get_resource(
&mut self,
binding: Binding,
stream_id: StreamId,
) -> Result<ManagedResource<<Self::Storage as ComputeStorage>::Resource>, ServerError>
fn get_resource( &mut self, binding: Binding, stream_id: StreamId, ) -> Result<ManagedResource<<Self::Storage as ComputeStorage>::Resource>, ServerError>
Given a resource handle, returns the storage resource.
Sourceunsafe fn launch(
&mut self,
kernel: Self::Kernel,
count: CubeCount,
bindings: KernelArguments,
kind: ExecutionMode,
stream_id: StreamId,
)
unsafe fn launch( &mut self, kernel: Self::Kernel, count: CubeCount, bindings: KernelArguments, kind: ExecutionMode, stream_id: StreamId, )
Executes the kernel over the given memory handles.
Kernels have mutable access to every resource they are given and are responsible of determining which should be read or written.
§Safety
When executing with mode ExecutionMode::Unchecked, out-of-bound reads and writes can happen.
Sourcefn flush(&mut self, stream_id: StreamId) -> Result<(), ServerError>
fn flush(&mut self, stream_id: StreamId) -> Result<(), ServerError>
Flush all outstanding tasks in the server.
Sourcefn memory_usage(
&mut self,
stream_id: StreamId,
) -> Result<MemoryUsage, ServerError>
fn memory_usage( &mut self, stream_id: StreamId, ) -> Result<MemoryUsage, ServerError>
The current memory usage of the server.
Sourcefn memory_cleanup(&mut self, stream_id: StreamId)
fn memory_cleanup(&mut self, stream_id: StreamId)
Ask the server to release memory that it can release.
Sourcefn start_profile(
&mut self,
stream_id: StreamId,
) -> Result<ProfilingToken, ServerError>
fn start_profile( &mut self, stream_id: StreamId, ) -> Result<ProfilingToken, ServerError>
Enable collecting timestamps.
Sourcefn end_profile(
&mut self,
stream_id: StreamId,
token: ProfilingToken,
) -> Result<ProfileDuration, ProfileError>
fn end_profile( &mut self, stream_id: StreamId, token: ProfilingToken, ) -> Result<ProfileDuration, ProfileError>
Disable collecting timestamps.
Sourcefn allocation_mode(&mut self, mode: MemoryAllocationMode, stream_id: StreamId)
fn allocation_mode(&mut self, mode: MemoryAllocationMode, stream_id: StreamId)
Update the memory mode of allocation in the server.
Provided Methods§
Sourcefn graph_capture_supported(&self) -> bool
fn graph_capture_supported(&self) -> bool
Whether this backend can capture a region of enqueued work into a replayable object at all. Callers must check this before capturing: the capture methods below panic on a backend that cannot.
Sourcefn graph_defer_frees(&mut self, _defer: bool, _stream_id: StreamId)
fn graph_defer_frees(&mut self, _defer: bool, _stream_id: StreamId)
Defer the stream’s pending-drop flush without opening a capture.
This is the PRE-WARM for a capture, and it is needed because a capture changes what the allocator has to hold at once. Normally a staging buffer is handed back a few launches after it is used, so a region of N launches only ever needs a handful of pages live. Inside a capture nothing is handed back – the flush waits on a fence, which a capture cannot contain, and the buffers must stay alive anyway because the graph re-reads them on every replay. So the region’s requirement jumps from its per-launch working set to its SIMULTANEOUS HIGH-WATER MARK, and the pool discovers that by allocating, which is the one thing that breaks a capture.
Running the region once with frees deferred and no capture open lets the pool reach that high-water mark the legal way. Flushing afterwards returns the slices but keeps the pages, so the capture that follows finds every page it needs already there.
Sourcefn graph_arena_begin(&mut self, _stream_id: StreamId) -> u64
fn graph_arena_begin(&mut self, _stream_id: StreamId) -> u64
Route every allocation on stream_id into the CAPTURE ARENA, and
return the generation this opening is.
Call this once to WARM (arena open, no capture: the region runs eagerly
and the arena learns exactly which slices it needs), then again around
the capture itself. See
MemoryManagement::arena_begin
for why an arena and not a hold.
Reopening the arena and capturing again DELIBERATELY hands the second graph the same slices: that is what makes two captures of one region comparable, and what lets one graph stand in for another step. The two then share scratch, which is safe exactly when they are captures of the same region – checked by the request sequence each capture recorded – and replayed serially, which the single stream gives for free.
Sourcefn graph_arena_end(&mut self, _stream_id: StreamId)
fn graph_arena_end(&mut self, _stream_id: StreamId)
Stop routing allocations into the arena. The arena keeps every page it owns – closing it ends allocation from it, never its reservation.
Sourcefn graph_arena_stats(&mut self, _stream_id: StreamId) -> ArenaStats
fn graph_arena_stats(&mut self, _stream_id: StreamId) -> ArenaStats
What the arena holds, and how many requests it had to take from the driver. A miss taken while a capture is open is a graph memory node.
Sourcefn graph_arena_reset_counters(&mut self, _stream_id: StreamId)
fn graph_arena_reset_counters(&mut self, _stream_id: StreamId)
Zero the arena’s served/miss counters so the capture pass can be counted apart from the warm pass.
Sourcefn reserve_timing(&mut self, _stream_id: StreamId) -> (u64, u64)
fn reserve_timing(&mut self, _stream_id: StreamId) -> (u64, u64)
What the allocator’s RESERVE half costs, as (calls, nanoseconds).
A decode step makes roughly one reservation per kernel launch, and all
of it is host work inside the term a graph replay removes. The count has
been reported for a while; this is the time, because a count is not a
time and the two lead to opposite conclusions at 0.5 us and at 5 us a
request. Zero unless CUBECL_TIME_RESERVE=1. It does NOT include the
free half, which happens on handle drop.
Sourcefn reserve_timing_reset(&mut self, _stream_id: StreamId)
fn reserve_timing_reset(&mut self, _stream_id: StreamId)
Zero the reservation timer.
Sourcefn graph_capture_begin(&mut self, _stream_id: StreamId)
fn graph_capture_begin(&mut self, _stream_id: StreamId)
Begin capturing work enqueued on stream_id instead of running it.
Between this and ComputeServer::graph_capture_end nothing executes;
the enqueued work is recorded as a graph. Anything that BLOCKS THE HOST
on device progress inside the captured region (a sync, a fence wait, a
blocking read) invalidates the capture, because there is no device
progress to wait for.
Sourcefn graph_capture_end(&mut self, _stream_id: StreamId) -> u64
fn graph_capture_end(&mut self, _stream_id: StreamId) -> u64
Close the capture opened by ComputeServer::graph_capture_begin,
instantiate the result, and return an id naming it for replay. The
captured graph holds the EXACT device pointers seen during capture, so
every buffer it touches must live at a stable address across replays.
Sourcefn graph_replay(&mut self, _id: u64, _stream_id: StreamId)
fn graph_replay(&mut self, _id: u64, _stream_id: StreamId)
Enqueue one replay of the graph named by id on stream_id.
Sourcefn graph_capture_status(&mut self, _stream_id: StreamId) -> u32
fn graph_capture_status(&mut self, _stream_id: StreamId) -> u32
0 = not capturing, 1 = capturing, 2 = INVALIDATED, 3 = query failed.
A capture that has been invalidated keeps accepting work silently and
only reports it at end, so this is the only way to find WHICH call
broke one.
Sourcefn graph_node_count(&mut self, _id: u64) -> usize
fn graph_node_count(&mut self, _id: u64) -> usize
How many nodes the captured graph holds. This is the unit the replay cost is quoted per, so it belongs next to the graph, not in a log line.
Sourcefn graph_capture_launch_count(&mut self) -> usize
fn graph_capture_launch_count(&mut self) -> usize
How many kernel launches the OPEN capture has recorded so far.
This is the index a caller brackets its own calls with, so that after the capture closes it can name the nodes a particular call produced without knowing anything about how many kernels that call fans out to. It counts launches, not nodes: a launch is one node, but a graph also holds memcpy and memset nodes that no launch made.
Sourcefn graph_launch_count(&mut self, _id: u64) -> usize
fn graph_launch_count(&mut self, _id: u64) -> usize
How many launches the closed graph id recorded.
Sourcefn graph_launch_params(&mut self, _id: u64, _idx: usize) -> GraphLaunchParams
fn graph_launch_params(&mut self, _id: u64, _idx: usize) -> GraphLaunchParams
What launch idx of graph id was recorded WITH: its grid, its packed
scalar+metadata blob, and the device address of every buffer it bound.
This is the half of patching that makes it checkable. A caller that wants to move a scalar has to know which slot of the blob holds it, and the honest way to learn that is to look at what the blob actually contained when the value was known – not to re-derive cubecl’s packing order by hand and hope.
Sourcefn graph_patch_launch(
&mut self,
_id: u64,
_idx: usize,
_patch: GraphLaunchPatch,
)
fn graph_patch_launch( &mut self, _id: u64, _idx: usize, _patch: GraphLaunchPatch, )
Rewrite launch idx of graph id’s EXECUTABLE in place.
The graph is untouched; only the instantiated exec changes, which is
what makes this cheap enough to do per step. Everything not named in
patch keeps the value it was captured with.
Sourcefn graph_node_kinds(&mut self, _id: u64) -> Vec<(u32, usize)>
fn graph_node_kinds(&mut self, _id: u64) -> Vec<(u32, usize)>
A census of the graph’s nodes by KIND, as (CUgraphNodeType, count).
A graph is not only its kernels. Everything else a capture recorded – host-to-device copies, memsets, child graphs – is a node too, and a node no kernel-parameter rewrite can reach. Whether a region is replayable on a later step is therefore first of all a question about how much of it is NOT a kernel launch.
Sourcefn graph_patch_launches(
&mut self,
_id: u64,
_patches: Vec<(usize, GraphLaunchPatch)>,
)
fn graph_patch_launches( &mut self, _id: u64, _patches: Vec<(usize, GraphLaunchPatch)>, )
Rewrite several launches of one graph in a single round trip.
Per-launch ComputeServer::graph_patch_launch is one submit each, and
on a region with a hundred moving launches the submits cost far more
than the rewrites do. The batch exists because that overhead is the
caller’s to avoid, not something to measure and report as the price of
patching.
Sourcefn graph_alloc_regions(&mut self, _id: u64) -> Vec<(u64, u64)>
fn graph_alloc_regions(&mut self, _id: u64) -> Vec<(u64, u64)>
Every (address, bytes) the graph ALLOCATES for itself.
A capture that could not avoid allocating turns each allocation into a memory node, and the memory a graph owns has a virtual address the graph fixes – the same one on every launch of that exec, and NOT the same one as any other graph’s. So these addresses are already pinned for as long as one exec is being replayed, and rewriting them to another graph’s is wrong by construction rather than merely unnecessary. A caller re-pointing a region’s buffers has to be able to tell them apart from the ones that came from the pool.
Sourcefn graph_memcpy_specs(&mut self, _id: u64) -> Vec<(u64, u64, u64, u32)>
fn graph_memcpy_specs(&mut self, _id: u64) -> Vec<(u64, u64, u64, u32)>
Every memcpy node’s (src, dst, bytes, kind), sorted.
A capture records copies as well as launches, and a copy is not a kernel: no parameter rewrite reaches it. When the data a copy carries is per-step, the region is not replayable on a later step no matter how well its kernels are patched, and this is how that is checked rather than assumed.
Sourcefn graph_destroy(&mut self, _id: u64)
fn graph_destroy(&mut self, _id: u64)
Release a captured graph. An unknown id is ignored.
Sourcefn register_external_aliased(
&mut self,
_ptr: *mut c_void,
_page_len: u64,
_offset: u64,
_size: u64,
_keepalive: Arc<dyn Any + Send + Sync>,
_stream_id: StreamId,
) -> Handle
fn register_external_aliased( &mut self, _ptr: *mut c_void, _page_len: u64, _offset: u64, _size: u64, _keepalive: Arc<dyn Any + Send + Sync>, _stream_id: StreamId, ) -> Handle
Register an externally-owned GPU buffer that ALIASES host memory (e.g.
mmap’d pile pages) as a tensor Handle, with ZERO copy. ptr/page_len
describe a page-aligned superset region (required by Metal’s
newBufferWithBytesNoCopy:); offset/size locate the actual tensor
bytes inside it. The caller guarantees the host memory outlives every
handle derived from it. Default impl panics — only backends that can
import host buffers (wgpu/Metal) override this.
Dyn Compatibility§
This trait is not dyn compatible.
In older versions of Rust, dyn compatibility was called "object safety".