llama-cpp-sys-4 0.10.0

Low Level Bindings to llama.cpp
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"

#include <memory>
#include <openvino/core/node.hpp>
#include <openvino/core/node_output.hpp>
#include <openvino/op/add.hpp>
#include <openvino/op/constant.hpp>
#include <openvino/op/convert.hpp>
#include <openvino/op/gather.hpp>
#include <openvino/op/greater_eq.hpp>
#include <openvino/op/maximum.hpp>
#include <openvino/op/multiply.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/shape_of.hpp>
#include <openvino/op/unsqueeze.hpp>

namespace ov {
namespace frontend {
namespace ggml {
namespace op {

static ov::Output<ov::Node> reshape_add_id_input_to_2d(const ov::Output<ov::Node> & input,
                                                       const ov::PartialShape & input_shape,
                                                       const std::vector<int> & dims) {
    const auto & actual_shape = input.get_partial_shape();
    if (actual_shape.rank().is_static() && actual_shape.rank().get_length() == 2) {
        return input;
    }

    if (input_shape.rank().is_static() && input_shape.rank().get_length() == 2) {
        return input;
    }

    auto shape = std::make_shared<ov::op::v3::ShapeOf>(input, ov::element::i64);
    return std::make_shared<ov::op::v1::Reshape>(input, get_dimensions(shape, dims), false);
}

OutputVector translate_add_id(const NodeContext & context) {
    num_inputs_check(context, 3, 3);

    auto input = process_view_input_new(context, 0);
    auto bias = process_view_input_new(context, 1);
    auto ids = process_view_input_new(context, 2);

    // OpenVINO uses reversed GGML dimensions:
    //   input: [1, n_token, n_used, n_embd]
    //   bias:  [1, 1, n_expert, n_embd]
    //   ids:   [1, 1, n_token, n_used]
    // Model bias constants may already be stored as [n_expert, n_embd].
    bias = reshape_add_id_input_to_2d(bias, context.get_input_shape(1), {2, 3});
    ids = reshape_add_id_input_to_2d(ids, context.get_input_shape(2), {2, 3});

    if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) {
        ids = std::make_shared<ov::op::v0::Convert>(ids, ov::element::i32);
    }

    // an id of -1 marks a skipped slot: gather row 0 for it and zero the bias with a mask
    auto zero = ov::op::v0::Constant::create(ids.get_element_type(), ov::Shape{}, {0});
    ov::Output<ov::Node> keep = std::make_shared<ov::op::v1::GreaterEqual>(ids, zero);
    keep = std::make_shared<ov::op::v0::Convert>(keep, bias.get_element_type());
    keep = std::make_shared<ov::op::v0::Unsqueeze>(keep, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}));
    ids = std::make_shared<ov::op::v1::Maximum>(ids, zero);

    auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
    ov::Output<ov::Node> selected_bias = std::make_shared<ov::op::v8::Gather>(bias, ids, gather_axis);
    selected_bias = std::make_shared<ov::op::v1::Multiply>(selected_bias, keep);
    selected_bias = std::make_shared<ov::op::v1::Reshape>(
        selected_bias, std::make_shared<ov::op::v3::ShapeOf>(input, ov::element::i64), false);

    if (selected_bias.get_element_type() != input.get_element_type()) {
        selected_bias = std::make_shared<ov::op::v0::Convert>(selected_bias, input.get_element_type());
    }

    ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Add>(input, selected_bias);
    const auto output_type = context.get_output_type();
    if (res.get_element_type() != output_type) {
        res = std::make_shared<ov::op::v0::Convert>(res, output_type);
    }

    return rename_outputs_with_suffix({res}, context.get_name());
}

}  // namespace op
}  // namespace ggml
}  // namespace frontend
}  // namespace ov