Skip to main content

GpuBatch

handwired_rs_gpu

Struct GpuBatch 

Source 
pub struct GpuBatch<Args, R> {
    pub program: GpuProgram,
    /* private fields */
}
Expand description

A #[handwired] function’s circuit on the GPU, run on typed arguments.

Fields§

§program: GpuProgram

Implementations§

Source§

impl<Args: Params, R: HwType> GpuBatch<Args, R>

Source

pub fn new( gpu: &Gpu, compiled: &Compiled<Args, R>, ) -> Result<GpuBatch<Args, R>, GpuError>

Examples found in repository?
examples/kv_store.rs (line 136)
85fn main() {
86    let start = Instant::now();
87    let compiled = kv_step_circuit();
88    println!(
89        "kv_step: {} live operations in {} layers, built in {:.1?}",
90        compiled.program.live_operation_count(),
91        compiled.program.layers.len(),
92        start.elapsed()
93    );
94    let report = compiled.verify(20_000).unwrap();
95    println!("verified against the function on {} sampled inputs", report.checked);
96
97    // A session: the circuit holds the store's state from one operation to the next, like a database.
98    let mut executor = TensorExecutor::new(&compiled.program);
99    let mut circuit_state: State = ([0; SLOTS], [0; SLOTS]);
100    let mut native_state: State = ([0; SLOTS], [0; SLOTS]);
101    let script = [(PUT, 3, 300), (PUT, 7, 700), (GET, 3, 0), (PUT, 3, 333), (GET, 3, 0), (DELETE, 7, 0), (GET, 7, 0)];
102    let mut source = Vec::new();
103    for (op, key, value) in script {
104        source.clear();
105        (circuit_state.0, circuit_state.1, op, key, value).encode(&mut source);
106        let outputs = executor.run(&source);
107        let keys: [u16; SLOTS] = std::array::from_fn(|slot| outputs[slot] as u16);
108        let values: [u16; SLOTS] = std::array::from_fn(|slot| outputs[SLOTS + slot] as u16);
109        let (found, previous) = (outputs[2 * SLOTS] == 1.0, outputs[2 * SLOTS + 1] as u16);
110        circuit_state = (keys, values);
111        let native = kv_step(native_state.0, native_state.1, op, key, value);
112        native_state = (native.0, native.1);
113        assert_eq!((circuit_state, found, previous), (native_state, native.2, native.3));
114        let name = ["get", "put", "delete"][op as usize];
115        println!("  {name:6} {key:>2} {value:>4}  ->  found {found:5}, previous {previous}");
116    }
117
118    // 100,000 independent operations on 100,000 different stores, in one GPU dispatch.
119    let mut rng = Rng(0x9E37_79B9_7F4A_7C15);
120    let records: Vec<Operation> = (0..100_000)
121        .map(|_| {
122            let keys: [u16; SLOTS] = std::array::from_fn(|_| rng.below(16) as u16);
123            let values: [u16; SLOTS] = std::array::from_fn(|_| rng.below(65_536) as u16);
124            (keys, values, rng.below(3) as u8, 1 + rng.below(15) as u16, rng.below(65_536) as u16)
125        })
126        .collect();
127    let start = Instant::now();
128    let native: Vec<_> =
129        records.iter().map(|&(keys, values, op, key, value)| kv_step(keys, values, op, key, value)).collect();
130    let native_time = start.elapsed();
131    let start = Instant::now();
132    let on_cpu = compiled.run_batch(&records).unwrap();
133    let cpu_time = start.elapsed();
134    assert_eq!(on_cpu, native, "the CPU batch executor disagrees with the function");
135    let gpu = Gpu::new().expect("a GPU adapter");
136    let batch = GpuBatch::new(&gpu, &compiled).unwrap();
137    let _ = batch.run(&gpu, &records[..1024]).unwrap();
138    let start = Instant::now();
139    let on_gpu = batch.run(&gpu, &records).unwrap();
140    let gpu_time = start.elapsed();
141    assert_eq!(on_gpu, native, "the GPU disagrees with the function");
142    println!("100,000 operations on separate stores; every result matches the function");
143    let threads = std::thread::available_parallelism().map_or(1, |count| count.get());
144    println!("plain Rust: {native_time:>9.1?}");
145    println!("CPU batch executor ({threads} threads): {cpu_time:>9.1?}  (range checks included)");
146    println!("GPU ({}, {:?}): {gpu_time:>9.1?}  (upload, run, read back)", gpu.info.name, gpu.info.device_type);
147}
More examples
Hide additional examples
examples/search.rs (line 91)
25fn main() {
26    let records = 100_000;
27    let words: [&[u8]; 8] = [b"cat ", b"dog ", b"the ", b"cart ", b"scatter ", b"a ", b"category ", b"doge "];
28    let mut seed = 0x2545_F491_4F6C_DD1Du64;
29    let mut next = move || {
30        seed ^= seed << 13;
31        seed ^= seed >> 7;
32        seed ^= seed << 17;
33        seed
34    };
35    let inputs: Vec<([u8; TEXT], [u8; 3])> = (0..records)
36        .map(|_| {
37            let mut text = Vec::new();
38            while text.len() < TEXT {
39                text.extend_from_slice(words[(next() % 8) as usize]);
40            }
41            let text: [u8; TEXT] = text[..TEXT].try_into().unwrap();
42            let pattern = if next() % 2 == 0 { *b"cat" } else { *b"dog" };
43            (text, pattern)
44        })
45        .collect();
46
47    let start = Instant::now();
48    let compiled = find_pattern_circuit();
49    println!(
50        "circuit: {} live operations in {} layers, built in {:.1?}",
51        compiled.program.live_operation_count(),
52        compiled.program.layers.len(),
53        start.elapsed()
54    );
55
56    let start = Instant::now();
57    let native: Vec<(bool, u8)> = inputs.iter().map(|&(text, pattern)| find_pattern(text, pattern)).collect();
58    let native_time = start.elapsed();
59
60    let start = Instant::now();
61    let mut executor = TensorExecutor::new(&compiled.program);
62    let mut source = Vec::with_capacity(TEXT + 3);
63    let cpu: Vec<(bool, u8)> = inputs
64        .iter()
65        .map(|record| {
66            source.clear();
67            handwired_rs::dsl::Params::encode(&(record.0, record.1), &mut source);
68            let outputs = executor.run(&source);
69            (outputs[0] == 1.0, outputs[1] as u8)
70        })
71        .collect();
72    let cpu_time = start.elapsed();
73
74    let mut sources = Vec::with_capacity(records * (TEXT + 3));
75    for record in &inputs {
76        handwired_rs::dsl::Params::encode(record, &mut sources);
77    }
78    let decode = |outputs: Vec<f32>| -> Vec<(bool, u8)> {
79        outputs.chunks(2).map(|record| (record[0] == 1.0, record[1] as u8)).collect()
80    };
81    let start = Instant::now();
82    let batched = decode(BatchExecutor::new(&compiled.program).run(&sources));
83    let batch_time = start.elapsed();
84    let start = Instant::now();
85    let parallel = decode(BatchExecutor::run_parallel(&compiled.program, &sources, None));
86    let parallel_time = start.elapsed();
87    let threads = std::thread::available_parallelism().map_or(1, |count| count.get());
88
89    let gpu = Gpu::new().expect("a GPU adapter");
90    let start = Instant::now();
91    let batch = GpuBatch::new(&gpu, &compiled).unwrap();
92    let shader_time = start.elapsed();
93    let _ = batch.run(&gpu, &inputs[..1024]).unwrap();
94    let start = Instant::now();
95    let on_gpu = batch.run(&gpu, &inputs).unwrap();
96    let gpu_time = start.elapsed();
97
98    assert_eq!(cpu, native, "the CPU tensor executor disagrees with the function");
99    assert_eq!(batched, native, "the batch executor disagrees with the function");
100    assert_eq!(parallel, native, "the parallel batch executor disagrees with the function");
101    assert_eq!(on_gpu, native, "the GPU disagrees with the function");
102    let found = native.iter().filter(|result| result.0).count();
103    println!("{records} records, {found} contain their pattern; every result matches the function");
104    println!("plain Rust:           {native_time:>10.1?}");
105    println!("CPU tensor executor:  {cpu_time:>10.1?}  (one record at a time)");
106    println!("CPU batch executor:   {batch_time:>10.1?}  ({LANES} records at a time, one thread)");
107    println!("CPU batch executor:   {parallel_time:>10.1?}  ({threads} threads)");
108    println!(
109        "GPU ({}, {:?}): {gpu_time:>10.1?}  (upload, run, read back; shader built once in {shader_time:.1?})",
110        gpu.info.name, gpu.info.device_type
111    );
112}
Source

pub fn staged( gpu: &Gpu, compiled: &Compiled<Args, R>, statements: usize, ) -> Result<GpuBatch<Args, R>, GpuError>

Like GpuBatch::new, starting a new shader stage after statements statements (usize::MAX for one shader).

Source

pub fn encode(&self, records: &[Args]) -> Result<Vec<f32>, GpuError>

The source tables for records, checked against the declared ranges.

Source

pub fn run(&self, gpu: &Gpu, records: &[Args]) -> Result<Vec<R>, GpuError>

Examples found in repository?
examples/kv_store.rs (line 137)
85fn main() {
86    let start = Instant::now();
87    let compiled = kv_step_circuit();
88    println!(
89        "kv_step: {} live operations in {} layers, built in {:.1?}",
90        compiled.program.live_operation_count(),
91        compiled.program.layers.len(),
92        start.elapsed()
93    );
94    let report = compiled.verify(20_000).unwrap();
95    println!("verified against the function on {} sampled inputs", report.checked);
96
97    // A session: the circuit holds the store's state from one operation to the next, like a database.
98    let mut executor = TensorExecutor::new(&compiled.program);
99    let mut circuit_state: State = ([0; SLOTS], [0; SLOTS]);
100    let mut native_state: State = ([0; SLOTS], [0; SLOTS]);
101    let script = [(PUT, 3, 300), (PUT, 7, 700), (GET, 3, 0), (PUT, 3, 333), (GET, 3, 0), (DELETE, 7, 0), (GET, 7, 0)];
102    let mut source = Vec::new();
103    for (op, key, value) in script {
104        source.clear();
105        (circuit_state.0, circuit_state.1, op, key, value).encode(&mut source);
106        let outputs = executor.run(&source);
107        let keys: [u16; SLOTS] = std::array::from_fn(|slot| outputs[slot] as u16);
108        let values: [u16; SLOTS] = std::array::from_fn(|slot| outputs[SLOTS + slot] as u16);
109        let (found, previous) = (outputs[2 * SLOTS] == 1.0, outputs[2 * SLOTS + 1] as u16);
110        circuit_state = (keys, values);
111        let native = kv_step(native_state.0, native_state.1, op, key, value);
112        native_state = (native.0, native.1);
113        assert_eq!((circuit_state, found, previous), (native_state, native.2, native.3));
114        let name = ["get", "put", "delete"][op as usize];
115        println!("  {name:6} {key:>2} {value:>4}  ->  found {found:5}, previous {previous}");
116    }
117
118    // 100,000 independent operations on 100,000 different stores, in one GPU dispatch.
119    let mut rng = Rng(0x9E37_79B9_7F4A_7C15);
120    let records: Vec<Operation> = (0..100_000)
121        .map(|_| {
122            let keys: [u16; SLOTS] = std::array::from_fn(|_| rng.below(16) as u16);
123            let values: [u16; SLOTS] = std::array::from_fn(|_| rng.below(65_536) as u16);
124            (keys, values, rng.below(3) as u8, 1 + rng.below(15) as u16, rng.below(65_536) as u16)
125        })
126        .collect();
127    let start = Instant::now();
128    let native: Vec<_> =
129        records.iter().map(|&(keys, values, op, key, value)| kv_step(keys, values, op, key, value)).collect();
130    let native_time = start.elapsed();
131    let start = Instant::now();
132    let on_cpu = compiled.run_batch(&records).unwrap();
133    let cpu_time = start.elapsed();
134    assert_eq!(on_cpu, native, "the CPU batch executor disagrees with the function");
135    let gpu = Gpu::new().expect("a GPU adapter");
136    let batch = GpuBatch::new(&gpu, &compiled).unwrap();
137    let _ = batch.run(&gpu, &records[..1024]).unwrap();
138    let start = Instant::now();
139    let on_gpu = batch.run(&gpu, &records).unwrap();
140    let gpu_time = start.elapsed();
141    assert_eq!(on_gpu, native, "the GPU disagrees with the function");
142    println!("100,000 operations on separate stores; every result matches the function");
143    let threads = std::thread::available_parallelism().map_or(1, |count| count.get());
144    println!("plain Rust: {native_time:>9.1?}");
145    println!("CPU batch executor ({threads} threads): {cpu_time:>9.1?}  (range checks included)");
146    println!("GPU ({}, {:?}): {gpu_time:>9.1?}  (upload, run, read back)", gpu.info.name, gpu.info.device_type);
147}
More examples
Hide additional examples
examples/search.rs (line 93)
25fn main() {
26    let records = 100_000;
27    let words: [&[u8]; 8] = [b"cat ", b"dog ", b"the ", b"cart ", b"scatter ", b"a ", b"category ", b"doge "];
28    let mut seed = 0x2545_F491_4F6C_DD1Du64;
29    let mut next = move || {
30        seed ^= seed << 13;
31        seed ^= seed >> 7;
32        seed ^= seed << 17;
33        seed
34    };
35    let inputs: Vec<([u8; TEXT], [u8; 3])> = (0..records)
36        .map(|_| {
37            let mut text = Vec::new();
38            while text.len() < TEXT {
39                text.extend_from_slice(words[(next() % 8) as usize]);
40            }
41            let text: [u8; TEXT] = text[..TEXT].try_into().unwrap();
42            let pattern = if next() % 2 == 0 { *b"cat" } else { *b"dog" };
43            (text, pattern)
44        })
45        .collect();
46
47    let start = Instant::now();
48    let compiled = find_pattern_circuit();
49    println!(
50        "circuit: {} live operations in {} layers, built in {:.1?}",
51        compiled.program.live_operation_count(),
52        compiled.program.layers.len(),
53        start.elapsed()
54    );
55
56    let start = Instant::now();
57    let native: Vec<(bool, u8)> = inputs.iter().map(|&(text, pattern)| find_pattern(text, pattern)).collect();
58    let native_time = start.elapsed();
59
60    let start = Instant::now();
61    let mut executor = TensorExecutor::new(&compiled.program);
62    let mut source = Vec::with_capacity(TEXT + 3);
63    let cpu: Vec<(bool, u8)> = inputs
64        .iter()
65        .map(|record| {
66            source.clear();
67            handwired_rs::dsl::Params::encode(&(record.0, record.1), &mut source);
68            let outputs = executor.run(&source);
69            (outputs[0] == 1.0, outputs[1] as u8)
70        })
71        .collect();
72    let cpu_time = start.elapsed();
73
74    let mut sources = Vec::with_capacity(records * (TEXT + 3));
75    for record in &inputs {
76        handwired_rs::dsl::Params::encode(record, &mut sources);
77    }
78    let decode = |outputs: Vec<f32>| -> Vec<(bool, u8)> {
79        outputs.chunks(2).map(|record| (record[0] == 1.0, record[1] as u8)).collect()
80    };
81    let start = Instant::now();
82    let batched = decode(BatchExecutor::new(&compiled.program).run(&sources));
83    let batch_time = start.elapsed();
84    let start = Instant::now();
85    let parallel = decode(BatchExecutor::run_parallel(&compiled.program, &sources, None));
86    let parallel_time = start.elapsed();
87    let threads = std::thread::available_parallelism().map_or(1, |count| count.get());
88
89    let gpu = Gpu::new().expect("a GPU adapter");
90    let start = Instant::now();
91    let batch = GpuBatch::new(&gpu, &compiled).unwrap();
92    let shader_time = start.elapsed();
93    let _ = batch.run(&gpu, &inputs[..1024]).unwrap();
94    let start = Instant::now();
95    let on_gpu = batch.run(&gpu, &inputs).unwrap();
96    let gpu_time = start.elapsed();
97
98    assert_eq!(cpu, native, "the CPU tensor executor disagrees with the function");
99    assert_eq!(batched, native, "the batch executor disagrees with the function");
100    assert_eq!(parallel, native, "the parallel batch executor disagrees with the function");
101    assert_eq!(on_gpu, native, "the GPU disagrees with the function");
102    let found = native.iter().filter(|result| result.0).count();
103    println!("{records} records, {found} contain their pattern; every result matches the function");
104    println!("plain Rust:           {native_time:>10.1?}");
105    println!("CPU tensor executor:  {cpu_time:>10.1?}  (one record at a time)");
106    println!("CPU batch executor:   {batch_time:>10.1?}  ({LANES} records at a time, one thread)");
107    println!("CPU batch executor:   {parallel_time:>10.1?}  ({threads} threads)");
108    println!(
109        "GPU ({}, {:?}): {gpu_time:>10.1?}  (upload, run, read back; shader built once in {shader_time:.1?})",
110        gpu.info.name, gpu.info.device_type
111    );
112}

Auto Trait Implementations§

§

impl<Args, R> !RefUnwindSafe for GpuBatch<Args, R>

§

impl<Args, R> !UnwindSafe for GpuBatch<Args, R>

§

impl<Args, R> Freeze for GpuBatch<Args, R>
where PhantomData<fn(Args) -> R>: Freeze,

§

impl<Args, R> Send for GpuBatch<Args, R>
where PhantomData<fn(Args) -> R>: Send,

§

impl<Args, R> Sync for GpuBatch<Args, R>
where PhantomData<fn(Args) -> R>: Sync,

§

impl<Args, R> Unpin for GpuBatch<Args, R>
where PhantomData<fn(Args) -> R>: Unpin,

§

impl<Args, R> UnsafeUnpin for GpuBatch<Args, R>
where PhantomData<fn(Args) -> R>: UnsafeUnpin,

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
where ST: ?Sized, DT: ?Sized,

Source§

impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
where ST: ?Sized, DT: ?Sized,

Source§

impl<T> Downcast<T> for T

Source§

fn downcast(&self) -> &T

Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> Read<Exclusive, BecauseExclusive> for T
where T: ?Sized,

Source§

impl<E> StoreElem<usize> for E

Source§

fn store(items: &mut [E], index: usize, value: E)

Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
Source§

impl<T> Upcast<T> for T

Source§

fn upcast(&self) -> Option<&T>

Source§

impl<T> WasmNotSend for T
where T: Send,

Source§

impl<T> WasmNotSendSync for T
where T: WasmNotSend + WasmNotSync,

Source§

impl<T> WasmNotSync for T
where T: Sync,