tract-gpu 0.23.9

Tiny, no-nonsense, self contained, TensorFlow and ONNX inference
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

use crate::tensor::{DeviceTensor, DeviceTensorExt};
use crate::turn_handler::make_tensor_for_node_output;
use tract_core::internal::*;

pub type DispatchGdnRecurrentFn = fn(
    &DeviceTensor,
    &DeviceTensor,
    &DeviceTensor,
    &DeviceTensor,
    &DeviceTensor,
    &DeviceTensor,
    &DeviceTensor,
    &DeviceTensor,
    bool, // sigmoid_beta: beta carries raw logits, kernel applies sigmoid
) -> TractResult<()>;

#[derive(Clone, Debug)]
pub struct GpuGatedDeltaNetRecurrent {
    pub backend_name: &'static str,
    pub dispatch: DispatchGdnRecurrentFn,
    pub sigmoid_beta: bool,
}

impl PartialEq for GpuGatedDeltaNetRecurrent {
    fn eq(&self, other: &Self) -> bool {
        self.backend_name == other.backend_name && self.sigmoid_beta == other.sigmoid_beta
    }
}
impl Eq for GpuGatedDeltaNetRecurrent {}
impl std::hash::Hash for GpuGatedDeltaNetRecurrent {
    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
        self.backend_name.hash(state);
    }
}

impl Op for GpuGatedDeltaNetRecurrent {
    fn name(&self) -> StaticName {
        format!("{}GatedDeltaNetRecurrent", self.backend_name).into()
    }
    op_as_typed_op!();
}

impl EvalOp for GpuGatedDeltaNetRecurrent {
    op_out_of_plan!();

    fn eval(&self, ctx: &EvalContext, inputs: TVec<TValue>) -> TractResult<TVec<TValue>> {
        ensure!(inputs.len() == 6);
        let tensors = inputs
            .iter()
            .map(|value| value.to_device_tensor())
            .collect::<TractResult<TVec<_>>>()?;
        // Output takes the VALUE shape (with GQA groups q/k carry hk heads
        // while v carries hv = G * hk) but the QUERY dtype, matching
        // output_facts() and the CPU reference op. Each output has its own
        // arena region in the schema (multi-output node).
        let output =
            make_tensor_for_node_output(ctx, 0, tensors[0].datum_type(), tensors[2].shape())?;
        let final_state =
            make_tensor_for_node_output(ctx, 1, tensors[5].datum_type(), tensors[5].shape())?;
        (self.dispatch)(
            tensors[0],
            tensors[1],
            tensors[2],
            tensors[3],
            tensors[4],
            tensors[5],
            &output,
            &final_state,
            self.sigmoid_beta,
        )?;
        Ok(tvec![output.into_tensor().into_tvalue(), final_state.into_tensor().into_tvalue()])
    }
}

impl TypedOp for GpuGatedDeltaNetRecurrent {
    fn output_facts(&self, inputs: &[&TypedFact]) -> TractResult<TVec<TypedFact>> {
        crate::utils::facts_to_device_facts(inputs, |facts| {
            ensure!(facts.len() == 6);
            ensure!(facts[0].datum_type == DatumType::F16);
            ensure!(facts[1].datum_type == DatumType::F16);
            ensure!(facts[2].datum_type == DatumType::F16);
            ensure!(facts[3].datum_type == DatumType::F32);
            ensure!(facts[4].datum_type == DatumType::F16);
            ensure!(matches!(facts[5].datum_type, DatumType::F16 | DatumType::F32));
            // Output = value shape (hv heads), query dtype (matches the CPU
            // reference op: `out.datum_type = inputs[0].datum_type`).
            let mut out = facts[2].without_value();
            out.datum_type = facts[0].datum_type;
            Ok(tvec![out, facts[5].without_value()])
        })
        .with_context(|| format!("invalid facts for {}", self.name()))
    }
    as_op!();
}