Struct GpuDynKVCache 

pub struct GpuDynKVCache {
    pub name: String,
    pub past_sequence_fact: TypedFact,
    pub input_sequence_fact: TypedFact,
    pub axis: usize,
}

Fields

name: String

past_sequence_fact: TypedFact

input_sequence_fact: TypedFact

axis: usize

Implementations

impl GpuDynKVCache

pub fn from_tract_transformers(op: &DynKeyValueCache) -> Self

Trait Implementations

impl Clone for GpuDynKVCache

fn clone(&self) -> GpuDynKVCache

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for GpuDynKVCache

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl EvalOp for GpuDynKVCache

fn is_stateless(&self) -> bool

fn state( &self, _session: &TurnState, node_id: usize, ) -> TractResult<Option<Box<dyn OpState>>>

fn eval( &self, inputs: SmallVec<[TValue; 4]>, ) -> Result<SmallVec<[TValue; 4]>, Error>

fn eval_with_session( &self, node_id: usize, session: &TurnState, inputs: SmallVec<[TValue; 4]>, ) -> Result<SmallVec<[TValue; 4]>, Error>

impl Hash for GpuDynKVCache

fn hash<H: Hasher>(&self, state: &mut H)

Feeds this value into the given Hasher.

fn hash_slice<H>(data: &[Self], state: &mut H)

where H: Hasher, Self: Sized,

Feeds a slice of this type into the given Hasher.

impl Op for GpuDynKVCache

fn name(&self) -> StaticName

fn info(&self) -> TractResult<Vec<String>>

Short (one-line) strings giving hints on internal implementation or important configuration details to be displayed in dumps.

fn as_typed(&self) -> Option<&dyn TypedOp>

fn validation(&self) -> Validation

The kind of accuracy check that should be performed on operation when testing them.

impl PartialEq for GpuDynKVCache

fn eq(&self, other: &Self) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl TypedOp for GpuDynKVCache

fn output_facts(&self, inputs: &[&TypedFact]) -> TractResult<TVec<TypedFact>>

Deduce output facts from input facts.

fn as_op(&self) -> &dyn Op

Reinterpret the TypedOp as an Op.

fn as_op_mut(&mut self) -> &mut dyn Op

Reinterpret the TypedOp as an Op, mutably.

fn axes_mapping( &self, inputs: &[&TypedFact], outputs: &[&TypedFact], ) -> Result<AxesMapping, Error>

fn fuse( &self, _model: &Graph<TypedFact, Box<dyn TypedOp>>, _node: &Node<TypedFact, Box<dyn TypedOp>>, ) -> Result<Option<ModelPatch<TypedFact, Box<dyn TypedOp>>>, Error>

Fuse op after codegen to deal with local optimisations.

fn declutter_with_session( &self, session: &mut OptimizerSession<'_>, model: &Graph<TypedFact, Box<dyn TypedOp>>, node: &Node<TypedFact, Box<dyn TypedOp>>, ) -> Result<Option<ModelPatch<TypedFact, Box<dyn TypedOp>>>, Error>

Declutter the op to the tract_core operator set as much as possible.

fn declutter( &self, model: &Graph<TypedFact, Box<dyn TypedOp>>, node: &Node<TypedFact, Box<dyn TypedOp>>, ) -> Result<Option<ModelPatch<TypedFact, Box<dyn TypedOp>>>, Error>

Declutter the op to the tract_core operator set as much as possible.

fn cost( &self, _inputs: &[&TypedFact], ) -> Result<SmallVec<[(Cost, TDim); 4]>, Error>

Computes a cost hint of the operation.

fn input_roi( &self, model: &Graph<TypedFact, Box<dyn TypedOp>>, node: &Node<TypedFact, Box<dyn TypedOp>>, ) -> Result<Option<SmallVec<[Option<TDim>; 4]>>, Error>

Derive ROI (region of interest) expressions for this node’s inputs. Called by the PropagateRoi pass. Default returns None (no propagation). Override to introduce ROIs or bubble them through.

fn suggested_axis_changes( &self, ) -> Result<SmallVec<[(InOut, AxisOp); 4]>, Error>

fn change_axes( &self, model: &Graph<TypedFact, Box<dyn TypedOp>>, node: &Node<TypedFact, Box<dyn TypedOp>>, io: InOut, change: &AxisOp, ) -> Result<Option<AxisChangeConsequence>, Error>

fn slice( &self, patch: &mut ModelPatch<TypedFact, Box<dyn TypedOp>>, model: &Graph<TypedFact, Box<dyn TypedOp>>, node: &Node<TypedFact, Box<dyn TypedOp>>, prefix: &str, inputs: &[OutletId], output_axis: usize, start: &TDim, end: &TDim, ) -> Result<Option<SmallVec<[OutletId; 4]>>, Error>

fn quantize( &self, model: &Graph<TypedFact, Box<dyn TypedOp>>, node: &Node<TypedFact, Box<dyn TypedOp>>, dt: DatumType, scale: f32, zero_point: i32, ) -> Result<Option<Box<dyn TypedOp>>, Error>

Transforms the op in an equivalent one, operating on dt (i8 or u8).

fn concretize_dims( &self, source: &Graph<TypedFact, Box<dyn TypedOp>>, node: &Node<TypedFact, Box<dyn TypedOp>>, target: &mut Graph<TypedFact, Box<dyn TypedOp>>, mapping: &HashMap<OutletId, OutletId>, values: &SymbolValues, ) -> Result<SmallVec<[OutletId; 4]>, Error>

Transform the op into by providing a value to one or more symbols.

fn codegen( &self, model: &Graph<TypedFact, Box<dyn TypedOp>>, node: &Node<TypedFact, Box<dyn TypedOp>>, ) -> Result<Option<ModelPatch<TypedFact, Box<dyn TypedOp>>>, Error>

Translate the op into the most efficient form possible for execution.

fn nested_model_multipliers( &self, inputs: &[&TypedFact], ) -> Vec<(Cow<'static, str>, TDim)>

Nested model multipliers, with label (for profiling).

impl Eq for GpuDynKVCache