Struct Graph

Source

pub struct Graph { /* private fields */ }

Expand description

A pipeline builder. Each operation declares placement and actual buffer use.

Implementations§

Source §

impl Graph

Source

pub unsafe fn gpu_scoped( &mut self, bindings: &[GpuAccess], callback: impl FnMut(&[View<'_>]) -> Result<()> + Send + 'static, ) -> Result<Node>

Integrate owned native GPU execution into this graph’s hazards and producer dependencies. The callback runs on a worker, never in poll.

§Safety

The callback must retain all private native resources it uses, access these views only as declared, and synchronize every stream touching them before returning. It must not let raw pointers/views escape this scope. Private storage must not be concurrently accessible outside the callback. On an uncertain failure return an error; captured resources and bindings are then quarantined together, preserving native lifetimes.

Source

pub fn fill(&mut self, destination: BufferView, value: u8) -> Result<Node>

Fill a GPU-visible range with a byte pattern.

Examples found in repository ?

examples/graph_prepare.rs (line 13)

4fn main() -> hrx::Result<()> {
5    let runtime = Runtime::new()?;
6    let buffer = runtime.allocate(4096, MemoryPlacement::GpuLocal)?;
7    let stream = Stream::open()?;
8    let raw = stream.allocate(4096)?;
9    for count in [128, 512, 1024, 2048] {
10        let begin = Instant::now();
11        let mut graph = runtime.graph();
12        for _ in 0..count {
13            graph.fill(buffer.view(), 0)?;
14        }
15        let graph = graph.prepare()?;
16        let inferred = begin.elapsed();
17        let begin = Instant::now();
18        let mut native = stream.graph()?;
19        let mut last = None;
20        for _ in 0..count {
21            last = Some(native.fill(last.as_slice(), raw.binding(), 0)?);
22        }
23        let _native = native.finish()?;
24        println!(
25            "nodes={count} inferred_us={} explicit_us={}",
26            inferred.as_micros(),
27            begin.elapsed().as_micros()
28        );
29        graph.submit()?.wait()?;
30    }
31    Ok(())
32}

More examples

Hide additional examples

examples/shared_roundtrip.rs (line 41)

8fn main() -> Result<()> {
9    let args: Vec<_> = std::env::args().collect();
10    if args.len() != 4 {
11        return Err(hrx::Error::Message(
12            "usage: shared_roundtrip <passthrough.xclbin> <instructions.bin> <bytes>".into(),
13        ));
14    }
15    let bytes = args[3]
16        .parse::<usize>()
17        .map_err(|e| hrx::Error::Message(e.to_string()))?;
18    let runtime = Runtime::new()?;
19    // This example accepts trusted local artifacts with the documented ABI.
20    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
21    let binding = |bytes, access| BindingContract {
22        bytes,
23        access,
24        alignment: 4,
25        layout: "u32 passthrough".into(),
26    };
27    let contract = KernelContract {
28        bindings: vec![
29            binding(bytes, Access::Read),
30            binding(4096, Access::Read),
31            binding(bytes, Access::Write),
32        ],
33        constants: vec![],
34    };
35    let kernel = unsafe { program.kernel(&std::fs::read(&args[2])?, contract) }?;
36    let a = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
37    let b = runtime.allocate(4096, MemoryPlacement::Shared(program.clone()))?;
38    let c = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
39    let result = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
40    let mut graph = runtime.graph();
41    graph.fill(a.view(), 0x5a)?;
42    graph.npu(&kernel, &[a.view(), b.view(), c.view()])?;
43    graph.copy(result.view(), c.view())?;
44    let graph = graph.prepare()?;
45    for _ in 0..10 {
46        graph.submit()?.wait()?;
47        let mapped = result.map_read()?;
48        if mapped.iter().any(|&b| b != 0x5a) {
49            return Err(hrx::Error::Message("GPU/NPU roundtrip mismatch".into()));
50        }
51    }
52    println!(
53        "GPU -> NPU -> GPU: all {bytes} bytes correct, 10 executions, {} prepared regions",
54        graph.regions()
55    );
56    Ok(())
57}

examples/shared_bench.rs (line 44)

9fn main() -> Result<()> {
10    let args: Vec<_> = std::env::args().collect();
11    if args.len() != 4 {
12        return Err(hrx::Error::Message(
13            "usage: shared_bench <passthrough.xclbin> <instructions.bin> <bytes>".into(),
14        ));
15    }
16    let bytes = args[3]
17        .parse::<usize>()
18        .map_err(|e| hrx::Error::Message(e.to_string()))?;
19    let runtime = Runtime::new()?;
20    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
21    let binding = |bytes, access| BindingContract {
22        bytes,
23        access,
24        alignment: 4,
25        layout: "passthrough".into(),
26    };
27    let kernel = unsafe {
28        program.kernel(
29            &std::fs::read(&args[2])?,
30            KernelContract {
31                bindings: vec![
32                    binding(bytes, Access::Read),
33                    binding(4096, Access::Read),
34                    binding(bytes, Access::Write),
35                ],
36                constants: vec![],
37            },
38        )
39    }?;
40    let a = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
41    let b = runtime.allocate(4096, MemoryPlacement::Shared(program.clone()))?;
42    let c = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
43    let mut graph = runtime.graph();
44    graph.fill(a.view(), 0x35)?;
45    graph.npu(&kernel, &[a.view(), b.view(), c.view()])?;
46    let graph = graph.prepare()?;
47    for _ in 0..5 {
48        graph.submit()?.wait()?;
49    }
50    let mut samples = Vec::new();
51    for _ in 0..100 {
52        let start = Instant::now();
53        graph.submit()?.wait()?;
54        samples.push(start.elapsed().as_secs_f64());
55    }
56    if c.map_read()?.iter().any(|&b| b != 0x35) {
57        return Err(hrx::Error::Message("incorrect benchmark output".into()));
58    }
59    samples.sort_by(f64::total_cmp);
60    println!(
61        "bytes={bytes} p50_us={:.2} p95_us={:.2}",
62        percentile(&samples, 50) * 1e6,
63        percentile(&samples, 95) * 1e6
64    );
65    Ok(())
66}

examples/gemm_pipeline.rs (line 127)

58fn main() -> Result<()> {
59    let args: Vec<_> = std::env::args().collect();
60    if args.len() != 6 {
61        return Err(hrx::Error::Message(
62            "usage: gemm_pipeline <row-major-bf16.xclbin> <instructions.bin> M K N".into(),
63        ));
64    }
65    let parse = |i: usize| {
66        args[i]
67            .parse::<usize>()
68            .map_err(|e| hrx::Error::Message(e.to_string()))
69    };
70    let (m, k, n) = (parse(3)?, parse(4)?, parse(5)?);
71    let extent = |a: usize, b: usize, width: usize| {
72        a.checked_mul(b)
73            .filter(|&count| count > 0 && count <= (1 << 30))
74            .and_then(|count| count.checked_mul(width))
75            .ok_or_else(|| hrx::Error::Message("invalid matrix extent".into()))
76    };
77    let (a_bytes, b_bytes, c_bytes) = (extent(m, k, 2)?, extent(k, n, 2)?, extent(m, n, 4)?);
78    let runtime = Runtime::new()?;
79    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
80    let npu = unsafe {
81        program.kernel(
82            &std::fs::read(&args[2])?,
83            KernelContract {
84                bindings: vec![
85                    binding(a_bytes, Access::Read),
86                    binding(b_bytes, Access::Read),
87                    binding(c_bytes, Access::Write),
88                ],
89                constants: vec![],
90            },
91        )
92    }?;
93    let prep = gpu_kernel(
94        &runtime,
95        include_str!("../tests/kernels/euler.loom"),
96        "krea2_euler",
97        &[
98            ("krea2.euler.grid_x", (m * k).div_ceil(256).to_string()),
99            ("krea2.euler.grid_y", "1".into()),
100        ],
101        m * k,
102        vec![
103            binding(a_bytes, Access::ReadWrite),
104            binding(a_bytes, Access::Read),
105        ],
106    )?;
107    let epilogue = gpu_kernel(
108        &runtime,
109        include_str!("../tests/kernels/add_f32.loom"),
110        "add_f32",
111        &[("add.grid", (m * n).div_ceil(256).to_string())],
112        m * n,
113        vec![binding(c_bytes, Access::ReadWrite)],
114    )?;
115    let allocate = |bytes| runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()));
116    let weights = allocate(b_bytes)?;
117    let ones = allocate(a_bytes)?;
118    for buffer in [&weights, &ones] {
119        for word in buffer.map_write()?.as_chunks_mut::<2>().0 {
120            word.copy_from_slice(&0x3f80u16.to_le_bytes());
121        }
122    }
123    let prepare = || -> Result<(ExecutableGraph, Buffer)> {
124        let a = allocate(a_bytes)?;
125        let c = allocate(c_bytes)?;
126        let mut graph = runtime.graph();
127        graph.fill(a.view(), 0)?;
128        graph.gpu(&prep, &[a.view(), ones.view()])?;
129        graph.npu(&npu, &[a.view(), weights.view(), c.view()])?;
130        graph.gpu(&epilogue, &[c.view()])?;
131        Ok((graph.prepare()?, c))
132    };
133    let (first, c0) = prepare()?;
134    let (second, c1) = prepare()?;
135    for _ in 0..5 {
136        first.submit()?.wait()?;
137        second.submit()?.wait()?;
138    }
139    let before = runtime.statistics();
140    let mut latency = Vec::with_capacity(50);
141    for _ in 0..50 {
142        let start = Instant::now();
143        first.submit()?.wait()?;
144        latency.push(start.elapsed().as_secs_f64());
145    }
146    let start = Instant::now();
147    for _ in 0..50 {
148        let a = first.submit()?;
149        let b = second.submit()?;
150        a.wait()?;
151        b.wait()?;
152    }
153    let requests_per_second = 100.0 / start.elapsed().as_secs_f64();
154    for c in [&c0, &c1] {
155        if c.map_read()?
156            .as_chunks::<4>()
157            .0
158            .iter()
159            .any(|v| f32::from_le_bytes(*v) != k as f32 + 1.0)
160        {
161            return Err(hrx::Error::Message("GEMM pipeline output mismatch".into()));
162        }
163    }
164    let after = runtime.statistics();
165    assert_eq!(before.allocations, after.allocations);
166    assert_eq!(before.imports, after.imports);
167    assert_eq!(after.copied_bytes, 0);
168    latency.sort_by(f64::total_cmp);
169    println!(
170        "M={m} K={k} N={n} p50_ms={:.3} p95_ms={:.3} pipelined_requests_s={requests_per_second:.2}",
171        percentile(&latency, 50) * 1e3,
172        percentile(&latency, 95) * 1e3
173    );
174    println!("{}", serde_json::to_string(&after)?);
175    Ok(())
176}

Source

pub fn copy( &mut self, destination: BufferView, source: BufferView, ) -> Result<Node>

Explicit GPU copy. Lengths must match and ranges must not overlap.

Examples found in repository ?

examples/shared_roundtrip.rs (line 43)

8fn main() -> Result<()> {
9    let args: Vec<_> = std::env::args().collect();
10    if args.len() != 4 {
11        return Err(hrx::Error::Message(
12            "usage: shared_roundtrip <passthrough.xclbin> <instructions.bin> <bytes>".into(),
13        ));
14    }
15    let bytes = args[3]
16        .parse::<usize>()
17        .map_err(|e| hrx::Error::Message(e.to_string()))?;
18    let runtime = Runtime::new()?;
19    // This example accepts trusted local artifacts with the documented ABI.
20    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
21    let binding = |bytes, access| BindingContract {
22        bytes,
23        access,
24        alignment: 4,
25        layout: "u32 passthrough".into(),
26    };
27    let contract = KernelContract {
28        bindings: vec![
29            binding(bytes, Access::Read),
30            binding(4096, Access::Read),
31            binding(bytes, Access::Write),
32        ],
33        constants: vec![],
34    };
35    let kernel = unsafe { program.kernel(&std::fs::read(&args[2])?, contract) }?;
36    let a = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
37    let b = runtime.allocate(4096, MemoryPlacement::Shared(program.clone()))?;
38    let c = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
39    let result = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
40    let mut graph = runtime.graph();
41    graph.fill(a.view(), 0x5a)?;
42    graph.npu(&kernel, &[a.view(), b.view(), c.view()])?;
43    graph.copy(result.view(), c.view())?;
44    let graph = graph.prepare()?;
45    for _ in 0..10 {
46        graph.submit()?.wait()?;
47        let mapped = result.map_read()?;
48        if mapped.iter().any(|&b| b != 0x5a) {
49            return Err(hrx::Error::Message("GPU/NPU roundtrip mismatch".into()));
50        }
51    }
52    println!(
53        "GPU -> NPU -> GPU: all {bytes} bytes correct, 10 executions, {} prepared regions",
54        graph.regions()
55    );
56    Ok(())
57}

Source

pub fn copy_on( &mut self, destination: BufferView, source: BufferView, lane: GpuLane, ) -> Result<Node>

Copy on an explicit lane, retaining the same cross-lane hazard checks.

Source

pub fn gpu( &mut self, kernel: &GpuKernel, bindings: &[BufferView], ) -> Result<Node>

Invoke a trusted GPU kernel; binding extents and alignment are checked. The selected runtime must expose allocation-address queries (interop ABI 1), including for GPU-local bindings. No base-pointer alignment is assumed.

Examples found in repository ?

examples/gemm_pipeline.rs (line 128)

58fn main() -> Result<()> {
59    let args: Vec<_> = std::env::args().collect();
60    if args.len() != 6 {
61        return Err(hrx::Error::Message(
62            "usage: gemm_pipeline <row-major-bf16.xclbin> <instructions.bin> M K N".into(),
63        ));
64    }
65    let parse = |i: usize| {
66        args[i]
67            .parse::<usize>()
68            .map_err(|e| hrx::Error::Message(e.to_string()))
69    };
70    let (m, k, n) = (parse(3)?, parse(4)?, parse(5)?);
71    let extent = |a: usize, b: usize, width: usize| {
72        a.checked_mul(b)
73            .filter(|&count| count > 0 && count <= (1 << 30))
74            .and_then(|count| count.checked_mul(width))
75            .ok_or_else(|| hrx::Error::Message("invalid matrix extent".into()))
76    };
77    let (a_bytes, b_bytes, c_bytes) = (extent(m, k, 2)?, extent(k, n, 2)?, extent(m, n, 4)?);
78    let runtime = Runtime::new()?;
79    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
80    let npu = unsafe {
81        program.kernel(
82            &std::fs::read(&args[2])?,
83            KernelContract {
84                bindings: vec![
85                    binding(a_bytes, Access::Read),
86                    binding(b_bytes, Access::Read),
87                    binding(c_bytes, Access::Write),
88                ],
89                constants: vec![],
90            },
91        )
92    }?;
93    let prep = gpu_kernel(
94        &runtime,
95        include_str!("../tests/kernels/euler.loom"),
96        "krea2_euler",
97        &[
98            ("krea2.euler.grid_x", (m * k).div_ceil(256).to_string()),
99            ("krea2.euler.grid_y", "1".into()),
100        ],
101        m * k,
102        vec![
103            binding(a_bytes, Access::ReadWrite),
104            binding(a_bytes, Access::Read),
105        ],
106    )?;
107    let epilogue = gpu_kernel(
108        &runtime,
109        include_str!("../tests/kernels/add_f32.loom"),
110        "add_f32",
111        &[("add.grid", (m * n).div_ceil(256).to_string())],
112        m * n,
113        vec![binding(c_bytes, Access::ReadWrite)],
114    )?;
115    let allocate = |bytes| runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()));
116    let weights = allocate(b_bytes)?;
117    let ones = allocate(a_bytes)?;
118    for buffer in [&weights, &ones] {
119        for word in buffer.map_write()?.as_chunks_mut::<2>().0 {
120            word.copy_from_slice(&0x3f80u16.to_le_bytes());
121        }
122    }
123    let prepare = || -> Result<(ExecutableGraph, Buffer)> {
124        let a = allocate(a_bytes)?;
125        let c = allocate(c_bytes)?;
126        let mut graph = runtime.graph();
127        graph.fill(a.view(), 0)?;
128        graph.gpu(&prep, &[a.view(), ones.view()])?;
129        graph.npu(&npu, &[a.view(), weights.view(), c.view()])?;
130        graph.gpu(&epilogue, &[c.view()])?;
131        Ok((graph.prepare()?, c))
132    };
133    let (first, c0) = prepare()?;
134    let (second, c1) = prepare()?;
135    for _ in 0..5 {
136        first.submit()?.wait()?;
137        second.submit()?.wait()?;
138    }
139    let before = runtime.statistics();
140    let mut latency = Vec::with_capacity(50);
141    for _ in 0..50 {
142        let start = Instant::now();
143        first.submit()?.wait()?;
144        latency.push(start.elapsed().as_secs_f64());
145    }
146    let start = Instant::now();
147    for _ in 0..50 {
148        let a = first.submit()?;
149        let b = second.submit()?;
150        a.wait()?;
151        b.wait()?;
152    }
153    let requests_per_second = 100.0 / start.elapsed().as_secs_f64();
154    for c in [&c0, &c1] {
155        if c.map_read()?
156            .as_chunks::<4>()
157            .0
158            .iter()
159            .any(|v| f32::from_le_bytes(*v) != k as f32 + 1.0)
160        {
161            return Err(hrx::Error::Message("GEMM pipeline output mismatch".into()));
162        }
163    }
164    let after = runtime.statistics();
165    assert_eq!(before.allocations, after.allocations);
166    assert_eq!(before.imports, after.imports);
167    assert_eq!(after.copied_bytes, 0);
168    latency.sort_by(f64::total_cmp);
169    println!(
170        "M={m} K={k} N={n} p50_ms={:.3} p95_ms={:.3} pipelined_requests_s={requests_per_second:.2}",
171        percentile(&latency, 50) * 1e3,
172        percentile(&latency, 95) * 1e3
173    );
174    println!("{}", serde_json::to_string(&after)?);
175    Ok(())
176}

Source

pub unsafe fn gpu_aliasing( &mut self, kernel: &GpuKernel, bindings: &[BufferView], ) -> Result<Node>

Invoke a trusted GPU kernel permitting writable argument aliases. Extents, alignment, runtime identity and cross-operation hazards remain checked.

§Safety

The kernel must be valid for the overlapping argument regions supplied; no inter-thread data races or unsupported in-place accesses may occur.

Source

pub fn npu( &mut self, kernel: &NpuKernel, bindings: &[BufferView], ) -> Result<Node>

Invoke a trusted NPU specialization with checked device and memory-bank compatibility. Host-only and imported BOs may originate in a different program context on the same device and memory bank. Their owning contexts remain retained.

Examples found in repository ?

examples/shared_roundtrip.rs (line 42)

8fn main() -> Result<()> {
9    let args: Vec<_> = std::env::args().collect();
10    if args.len() != 4 {
11        return Err(hrx::Error::Message(
12            "usage: shared_roundtrip <passthrough.xclbin> <instructions.bin> <bytes>".into(),
13        ));
14    }
15    let bytes = args[3]
16        .parse::<usize>()
17        .map_err(|e| hrx::Error::Message(e.to_string()))?;
18    let runtime = Runtime::new()?;
19    // This example accepts trusted local artifacts with the documented ABI.
20    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
21    let binding = |bytes, access| BindingContract {
22        bytes,
23        access,
24        alignment: 4,
25        layout: "u32 passthrough".into(),
26    };
27    let contract = KernelContract {
28        bindings: vec![
29            binding(bytes, Access::Read),
30            binding(4096, Access::Read),
31            binding(bytes, Access::Write),
32        ],
33        constants: vec![],
34    };
35    let kernel = unsafe { program.kernel(&std::fs::read(&args[2])?, contract) }?;
36    let a = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
37    let b = runtime.allocate(4096, MemoryPlacement::Shared(program.clone()))?;
38    let c = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
39    let result = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
40    let mut graph = runtime.graph();
41    graph.fill(a.view(), 0x5a)?;
42    graph.npu(&kernel, &[a.view(), b.view(), c.view()])?;
43    graph.copy(result.view(), c.view())?;
44    let graph = graph.prepare()?;
45    for _ in 0..10 {
46        graph.submit()?.wait()?;
47        let mapped = result.map_read()?;
48        if mapped.iter().any(|&b| b != 0x5a) {
49            return Err(hrx::Error::Message("GPU/NPU roundtrip mismatch".into()));
50        }
51    }
52    println!(
53        "GPU -> NPU -> GPU: all {bytes} bytes correct, 10 executions, {} prepared regions",
54        graph.regions()
55    );
56    Ok(())
57}

More examples

Hide additional examples

examples/shared_bench.rs (line 45)

9fn main() -> Result<()> {
10    let args: Vec<_> = std::env::args().collect();
11    if args.len() != 4 {
12        return Err(hrx::Error::Message(
13            "usage: shared_bench <passthrough.xclbin> <instructions.bin> <bytes>".into(),
14        ));
15    }
16    let bytes = args[3]
17        .parse::<usize>()
18        .map_err(|e| hrx::Error::Message(e.to_string()))?;
19    let runtime = Runtime::new()?;
20    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
21    let binding = |bytes, access| BindingContract {
22        bytes,
23        access,
24        alignment: 4,
25        layout: "passthrough".into(),
26    };
27    let kernel = unsafe {
28        program.kernel(
29            &std::fs::read(&args[2])?,
30            KernelContract {
31                bindings: vec![
32                    binding(bytes, Access::Read),
33                    binding(4096, Access::Read),
34                    binding(bytes, Access::Write),
35                ],
36                constants: vec![],
37            },
38        )
39    }?;
40    let a = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
41    let b = runtime.allocate(4096, MemoryPlacement::Shared(program.clone()))?;
42    let c = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
43    let mut graph = runtime.graph();
44    graph.fill(a.view(), 0x35)?;
45    graph.npu(&kernel, &[a.view(), b.view(), c.view()])?;
46    let graph = graph.prepare()?;
47    for _ in 0..5 {
48        graph.submit()?.wait()?;
49    }
50    let mut samples = Vec::new();
51    for _ in 0..100 {
52        let start = Instant::now();
53        graph.submit()?.wait()?;
54        samples.push(start.elapsed().as_secs_f64());
55    }
56    if c.map_read()?.iter().any(|&b| b != 0x35) {
57        return Err(hrx::Error::Message("incorrect benchmark output".into()));
58    }
59    samples.sort_by(f64::total_cmp);
60    println!(
61        "bytes={bytes} p50_us={:.2} p95_us={:.2}",
62        percentile(&samples, 50) * 1e6,
63        percentile(&samples, 95) * 1e6
64    );
65    Ok(())
66}

examples/gemm_pipeline.rs (line 129)

58fn main() -> Result<()> {
59    let args: Vec<_> = std::env::args().collect();
60    if args.len() != 6 {
61        return Err(hrx::Error::Message(
62            "usage: gemm_pipeline <row-major-bf16.xclbin> <instructions.bin> M K N".into(),
63        ));
64    }
65    let parse = |i: usize| {
66        args[i]
67            .parse::<usize>()
68            .map_err(|e| hrx::Error::Message(e.to_string()))
69    };
70    let (m, k, n) = (parse(3)?, parse(4)?, parse(5)?);
71    let extent = |a: usize, b: usize, width: usize| {
72        a.checked_mul(b)
73            .filter(|&count| count > 0 && count <= (1 << 30))
74            .and_then(|count| count.checked_mul(width))
75            .ok_or_else(|| hrx::Error::Message("invalid matrix extent".into()))
76    };
77    let (a_bytes, b_bytes, c_bytes) = (extent(m, k, 2)?, extent(k, n, 2)?, extent(m, n, 4)?);
78    let runtime = Runtime::new()?;
79    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
80    let npu = unsafe {
81        program.kernel(
82            &std::fs::read(&args[2])?,
83            KernelContract {
84                bindings: vec![
85                    binding(a_bytes, Access::Read),
86                    binding(b_bytes, Access::Read),
87                    binding(c_bytes, Access::Write),
88                ],
89                constants: vec![],
90            },
91        )
92    }?;
93    let prep = gpu_kernel(
94        &runtime,
95        include_str!("../tests/kernels/euler.loom"),
96        "krea2_euler",
97        &[
98            ("krea2.euler.grid_x", (m * k).div_ceil(256).to_string()),
99            ("krea2.euler.grid_y", "1".into()),
100        ],
101        m * k,
102        vec![
103            binding(a_bytes, Access::ReadWrite),
104            binding(a_bytes, Access::Read),
105        ],
106    )?;
107    let epilogue = gpu_kernel(
108        &runtime,
109        include_str!("../tests/kernels/add_f32.loom"),
110        "add_f32",
111        &[("add.grid", (m * n).div_ceil(256).to_string())],
112        m * n,
113        vec![binding(c_bytes, Access::ReadWrite)],
114    )?;
115    let allocate = |bytes| runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()));
116    let weights = allocate(b_bytes)?;
117    let ones = allocate(a_bytes)?;
118    for buffer in [&weights, &ones] {
119        for word in buffer.map_write()?.as_chunks_mut::<2>().0 {
120            word.copy_from_slice(&0x3f80u16.to_le_bytes());
121        }
122    }
123    let prepare = || -> Result<(ExecutableGraph, Buffer)> {
124        let a = allocate(a_bytes)?;
125        let c = allocate(c_bytes)?;
126        let mut graph = runtime.graph();
127        graph.fill(a.view(), 0)?;
128        graph.gpu(&prep, &[a.view(), ones.view()])?;
129        graph.npu(&npu, &[a.view(), weights.view(), c.view()])?;
130        graph.gpu(&epilogue, &[c.view()])?;
131        Ok((graph.prepare()?, c))
132    };
133    let (first, c0) = prepare()?;
134    let (second, c1) = prepare()?;
135    for _ in 0..5 {
136        first.submit()?.wait()?;
137        second.submit()?.wait()?;
138    }
139    let before = runtime.statistics();
140    let mut latency = Vec::with_capacity(50);
141    for _ in 0..50 {
142        let start = Instant::now();
143        first.submit()?.wait()?;
144        latency.push(start.elapsed().as_secs_f64());
145    }
146    let start = Instant::now();
147    for _ in 0..50 {
148        let a = first.submit()?;
149        let b = second.submit()?;
150        a.wait()?;
151        b.wait()?;
152    }
153    let requests_per_second = 100.0 / start.elapsed().as_secs_f64();
154    for c in [&c0, &c1] {
155        if c.map_read()?
156            .as_chunks::<4>()
157            .0
158            .iter()
159            .any(|v| f32::from_le_bytes(*v) != k as f32 + 1.0)
160        {
161            return Err(hrx::Error::Message("GEMM pipeline output mismatch".into()));
162        }
163    }
164    let after = runtime.statistics();
165    assert_eq!(before.allocations, after.allocations);
166    assert_eq!(before.imports, after.imports);
167    assert_eq!(after.copied_bytes, 0);
168    latency.sort_by(f64::total_cmp);
169    println!(
170        "M={m} K={k} N={n} p50_ms={:.3} p95_ms={:.3} pipelined_requests_s={requests_per_second:.2}",
171        percentile(&latency, 50) * 1e3,
172        percentile(&latency, 95) * 1e3
173    );
174    println!("{}", serde_json::to_string(&after)?);
175    Ok(())
176}

Source

pub fn prepare(self) -> Result<ExecutableGraph>

Lower GPU regions and import NPU views once; reserve reusable run slots.

Examples found in repository ?

examples/graph_prepare.rs (line 15)

4fn main() -> hrx::Result<()> {
5    let runtime = Runtime::new()?;
6    let buffer = runtime.allocate(4096, MemoryPlacement::GpuLocal)?;
7    let stream = Stream::open()?;
8    let raw = stream.allocate(4096)?;
9    for count in [128, 512, 1024, 2048] {
10        let begin = Instant::now();
11        let mut graph = runtime.graph();
12        for _ in 0..count {
13            graph.fill(buffer.view(), 0)?;
14        }
15        let graph = graph.prepare()?;
16        let inferred = begin.elapsed();
17        let begin = Instant::now();
18        let mut native = stream.graph()?;
19        let mut last = None;
20        for _ in 0..count {
21            last = Some(native.fill(last.as_slice(), raw.binding(), 0)?);
22        }
23        let _native = native.finish()?;
24        println!(
25            "nodes={count} inferred_us={} explicit_us={}",
26            inferred.as_micros(),
27            begin.elapsed().as_micros()
28        );
29        graph.submit()?.wait()?;
30    }
31    Ok(())
32}

More examples

Hide additional examples

examples/shared_roundtrip.rs (line 44)

8fn main() -> Result<()> {
9    let args: Vec<_> = std::env::args().collect();
10    if args.len() != 4 {
11        return Err(hrx::Error::Message(
12            "usage: shared_roundtrip <passthrough.xclbin> <instructions.bin> <bytes>".into(),
13        ));
14    }
15    let bytes = args[3]
16        .parse::<usize>()
17        .map_err(|e| hrx::Error::Message(e.to_string()))?;
18    let runtime = Runtime::new()?;
19    // This example accepts trusted local artifacts with the documented ABI.
20    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
21    let binding = |bytes, access| BindingContract {
22        bytes,
23        access,
24        alignment: 4,
25        layout: "u32 passthrough".into(),
26    };
27    let contract = KernelContract {
28        bindings: vec![
29            binding(bytes, Access::Read),
30            binding(4096, Access::Read),
31            binding(bytes, Access::Write),
32        ],
33        constants: vec![],
34    };
35    let kernel = unsafe { program.kernel(&std::fs::read(&args[2])?, contract) }?;
36    let a = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
37    let b = runtime.allocate(4096, MemoryPlacement::Shared(program.clone()))?;
38    let c = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
39    let result = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
40    let mut graph = runtime.graph();
41    graph.fill(a.view(), 0x5a)?;
42    graph.npu(&kernel, &[a.view(), b.view(), c.view()])?;
43    graph.copy(result.view(), c.view())?;
44    let graph = graph.prepare()?;
45    for _ in 0..10 {
46        graph.submit()?.wait()?;
47        let mapped = result.map_read()?;
48        if mapped.iter().any(|&b| b != 0x5a) {
49            return Err(hrx::Error::Message("GPU/NPU roundtrip mismatch".into()));
50        }
51    }
52    println!(
53        "GPU -> NPU -> GPU: all {bytes} bytes correct, 10 executions, {} prepared regions",
54        graph.regions()
55    );
56    Ok(())
57}

examples/shared_bench.rs (line 46)

9fn main() -> Result<()> {
10    let args: Vec<_> = std::env::args().collect();
11    if args.len() != 4 {
12        return Err(hrx::Error::Message(
13            "usage: shared_bench <passthrough.xclbin> <instructions.bin> <bytes>".into(),
14        ));
15    }
16    let bytes = args[3]
17        .parse::<usize>()
18        .map_err(|e| hrx::Error::Message(e.to_string()))?;
19    let runtime = Runtime::new()?;
20    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
21    let binding = |bytes, access| BindingContract {
22        bytes,
23        access,
24        alignment: 4,
25        layout: "passthrough".into(),
26    };
27    let kernel = unsafe {
28        program.kernel(
29            &std::fs::read(&args[2])?,
30            KernelContract {
31                bindings: vec![
32                    binding(bytes, Access::Read),
33                    binding(4096, Access::Read),
34                    binding(bytes, Access::Write),
35                ],
36                constants: vec![],
37            },
38        )
39    }?;
40    let a = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
41    let b = runtime.allocate(4096, MemoryPlacement::Shared(program.clone()))?;
42    let c = runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()))?;
43    let mut graph = runtime.graph();
44    graph.fill(a.view(), 0x35)?;
45    graph.npu(&kernel, &[a.view(), b.view(), c.view()])?;
46    let graph = graph.prepare()?;
47    for _ in 0..5 {
48        graph.submit()?.wait()?;
49    }
50    let mut samples = Vec::new();
51    for _ in 0..100 {
52        let start = Instant::now();
53        graph.submit()?.wait()?;
54        samples.push(start.elapsed().as_secs_f64());
55    }
56    if c.map_read()?.iter().any(|&b| b != 0x35) {
57        return Err(hrx::Error::Message("incorrect benchmark output".into()));
58    }
59    samples.sort_by(f64::total_cmp);
60    println!(
61        "bytes={bytes} p50_us={:.2} p95_us={:.2}",
62        percentile(&samples, 50) * 1e6,
63        percentile(&samples, 95) * 1e6
64    );
65    Ok(())
66}

examples/gemm_pipeline.rs (line 131)

58fn main() -> Result<()> {
59    let args: Vec<_> = std::env::args().collect();
60    if args.len() != 6 {
61        return Err(hrx::Error::Message(
62            "usage: gemm_pipeline <row-major-bf16.xclbin> <instructions.bin> M K N".into(),
63        ));
64    }
65    let parse = |i: usize| {
66        args[i]
67            .parse::<usize>()
68            .map_err(|e| hrx::Error::Message(e.to_string()))
69    };
70    let (m, k, n) = (parse(3)?, parse(4)?, parse(5)?);
71    let extent = |a: usize, b: usize, width: usize| {
72        a.checked_mul(b)
73            .filter(|&count| count > 0 && count <= (1 << 30))
74            .and_then(|count| count.checked_mul(width))
75            .ok_or_else(|| hrx::Error::Message("invalid matrix extent".into()))
76    };
77    let (a_bytes, b_bytes, c_bytes) = (extent(m, k, 2)?, extent(k, n, 2)?, extent(m, n, 4)?);
78    let runtime = Runtime::new()?;
79    let program = unsafe { runtime.npu(0)?.load_program(&args[1]) }?;
80    let npu = unsafe {
81        program.kernel(
82            &std::fs::read(&args[2])?,
83            KernelContract {
84                bindings: vec![
85                    binding(a_bytes, Access::Read),
86                    binding(b_bytes, Access::Read),
87                    binding(c_bytes, Access::Write),
88                ],
89                constants: vec![],
90            },
91        )
92    }?;
93    let prep = gpu_kernel(
94        &runtime,
95        include_str!("../tests/kernels/euler.loom"),
96        "krea2_euler",
97        &[
98            ("krea2.euler.grid_x", (m * k).div_ceil(256).to_string()),
99            ("krea2.euler.grid_y", "1".into()),
100        ],
101        m * k,
102        vec![
103            binding(a_bytes, Access::ReadWrite),
104            binding(a_bytes, Access::Read),
105        ],
106    )?;
107    let epilogue = gpu_kernel(
108        &runtime,
109        include_str!("../tests/kernels/add_f32.loom"),
110        "add_f32",
111        &[("add.grid", (m * n).div_ceil(256).to_string())],
112        m * n,
113        vec![binding(c_bytes, Access::ReadWrite)],
114    )?;
115    let allocate = |bytes| runtime.allocate(bytes, MemoryPlacement::Shared(program.clone()));
116    let weights = allocate(b_bytes)?;
117    let ones = allocate(a_bytes)?;
118    for buffer in [&weights, &ones] {
119        for word in buffer.map_write()?.as_chunks_mut::<2>().0 {
120            word.copy_from_slice(&0x3f80u16.to_le_bytes());
121        }
122    }
123    let prepare = || -> Result<(ExecutableGraph, Buffer)> {
124        let a = allocate(a_bytes)?;
125        let c = allocate(c_bytes)?;
126        let mut graph = runtime.graph();
127        graph.fill(a.view(), 0)?;
128        graph.gpu(&prep, &[a.view(), ones.view()])?;
129        graph.npu(&npu, &[a.view(), weights.view(), c.view()])?;
130        graph.gpu(&epilogue, &[c.view()])?;
131        Ok((graph.prepare()?, c))
132    };
133    let (first, c0) = prepare()?;
134    let (second, c1) = prepare()?;
135    for _ in 0..5 {
136        first.submit()?.wait()?;
137        second.submit()?.wait()?;
138    }
139    let before = runtime.statistics();
140    let mut latency = Vec::with_capacity(50);
141    for _ in 0..50 {
142        let start = Instant::now();
143        first.submit()?.wait()?;
144        latency.push(start.elapsed().as_secs_f64());
145    }
146    let start = Instant::now();
147    for _ in 0..50 {
148        let a = first.submit()?;
149        let b = second.submit()?;
150        a.wait()?;
151        b.wait()?;
152    }
153    let requests_per_second = 100.0 / start.elapsed().as_secs_f64();
154    for c in [&c0, &c1] {
155        if c.map_read()?
156            .as_chunks::<4>()
157            .0
158            .iter()
159            .any(|v| f32::from_le_bytes(*v) != k as f32 + 1.0)
160        {
161            return Err(hrx::Error::Message("GEMM pipeline output mismatch".into()));
162        }
163    }
164    let after = runtime.statistics();
165    assert_eq!(before.allocations, after.allocations);
166    assert_eq!(before.imports, after.imports);
167    assert_eq!(after.copied_bytes, 0);
168    latency.sort_by(f64::total_cmp);
169    println!(
170        "M={m} K={k} N={n} p50_ms={:.3} p95_ms={:.3} pipelined_requests_s={requests_per_second:.2}",
171        percentile(&latency, 50) * 1e3,
172        percentile(&latency, 95) * 1e3
173    );
174    println!("{}", serde_json::to_string(&after)?);
175    Ok(())
176}

Auto Trait Implementations§

§

impl UnsafeUnpin for Graph

Blanket Implementations§

Source §

impl<T> Any for T
where T: 'static + ?Sized,

Source §

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more

Source §

impl<T> Borrow<T> for T
where T: ?Sized,

Source §

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more

Source §

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source §

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more

Source §

impl<T> DropFlavorWrapper<T> for T

Source §

type Flavor = MayDrop

The DropFlavor that wraps T into Self

Source §

impl<T> From<T> for T

Source §

fn from(t: T) -> T

Returns the argument unchanged.

Source §

impl<T, W> HasTypeWitness<W> for T
where W: MakeTypeWitness<Arg = T>, T: ?Sized,

Source §

const WITNESS: W = W::MAKE

A constant of the type witness

Source §

impl<T> Identity for T
where T: ?Sized,

Source §

const TYPE_EQ: TypeEq<T, <T as Identity>::Type> = TypeEq::NEW

Proof that Self is the same type as Self::Type, provides methods for casting between Self and Self::Type.

Source §

type Type = T

The same type as Self, used to emulate type equality bounds (T == U) with associated type equality constraints (T: Identity<Type = U>).

Source §

impl<T> Instrument for T

Source §

fn instrument(self, span: Span) -> Instrumented<Self> ⓘ

Instruments this type with the provided Span, returning an Instrumented wrapper. Read more

Source §

fn in_current_span(self) -> Instrumented<Self> ⓘ

Instruments this type with the current Span, returning an Instrumented wrapper. Read more

Source §

impl<T, U> Into for T
where U: From<T>,

Source §

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source §

impl<T> IntoEither for T

Source §

fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more

Source §

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more

Source §

impl<T> PolicyExt for T
where T: ?Sized,

Source §

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Sized + Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns Action::Follow only if self and other return Action::Follow. Read more

Source §

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Sized + Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns Action::Follow if either self or other returns Action::Follow. Read more

Source §

impl<T> Same for T

Source §

type Output = T

Should always be Self

Source §

impl<T, U> TryFrom for T
where U: Into<T>,

Source §

type Error = !

The type returned in the event of a conversion error.

Source §

fn try_from(value: U) -> Result<T, !>

Performs the conversion.

Source §

impl<T, U> TryInto for T
where U: TryFrom<T>,

Source §

type Error = >::Error

The type returned in the event of a conversion error.

Source §

fn try_into(self) -> Result<U, >::Error>

Performs the conversion.

Source §

impl<T> WithSubscriber for T

Source §

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self> ⓘ
where S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a WithDispatch wrapper. Read more

Source §

fn with_current_subscriber(self) -> WithDispatch<Self> ⓘ

Attaches the current default Subscriber to this type, returning a WithDispatch wrapper. Read more

Struct Graph Copy item path

Implementations§

impl Graph

pub unsafe fn gpu_scoped( &mut self, bindings: &[GpuAccess], callback: impl FnMut(&[View<'_>]) -> Result<()> + Send + 'static, ) -> Result<Node>

§Safety

pub fn fill(&mut self, destination: BufferView, value: u8) -> Result<Node>

pub fn copy( &mut self, destination: BufferView, source: BufferView, ) -> Result<Node>

pub fn copy_on( &mut self, destination: BufferView, source: BufferView, lane: GpuLane, ) -> Result<Node>

pub fn gpu( &mut self, kernel: &GpuKernel, bindings: &[BufferView], ) -> Result<Node>

pub unsafe fn gpu_aliasing( &mut self, kernel: &GpuKernel, bindings: &[BufferView], ) -> Result<Node>

§Safety

pub fn npu( &mut self, kernel: &NpuKernel, bindings: &[BufferView], ) -> Result<Node>

pub fn prepare(self) -> Result<ExecutableGraph>

Auto Trait Implementations§

impl !RefUnwindSafe for Graph

impl !UnwindSafe for Graph

impl Freeze for Graph

impl Send for Graph

impl Sync for Graph

impl Unpin for Graph

impl UnsafeUnpin for Graph

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> DropFlavorWrapper<T> for T

type Flavor = MayDrop

impl<T> From<T> for T

fn from(t: T) -> T

impl<T, W> HasTypeWitness<W> for Twhere W: MakeTypeWitness<Arg = T>, T: ?Sized,

const WITNESS: W = W::MAKE

impl<T> Identity for Twhere T: ?Sized,

const TYPE_EQ: TypeEq<T, <T as Identity>::Type> = TypeEq::NEW

type Type = T

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self> ⓘ

fn in_current_span(self) -> Instrumented<Self> ⓘ

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘwhere F: FnOnce(&Self) -> bool,

impl<T> PolicyExt for Twhere T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>where T: Sized + Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>where T: Sized + Policy<B, E>, P: Policy<B, E>,

impl<T> Same for T

type Output = T

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = !

fn try_from(value: U) -> Result<T, !>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

impl<T> WithSubscriber for T

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self> ⓘwhere S: Into<Dispatch>,

fn with_current_subscriber(self) -> WithDispatch<Self> ⓘ

Struct Graph

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T, W> HasTypeWitness<W> for T
where W: MakeTypeWitness<Arg = T>, T: ?Sized,

impl<T> Identity for T
where T: ?Sized,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
where F: FnOnce(&Self) -> bool,

impl<T> PolicyExt for T
where T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Sized + Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Sized + Policy<B, E>, P: Policy<B, E>,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self> ⓘ
where S: Into<Dispatch>,