llama-cpp-sys-4 0.3.0

Low Level Bindings to llama.cpp
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159

#include "mtp_shim.h"

#include "common.h"
#include "speculative.h"

#include <vector>

struct mtp_session {
    common_speculative_ptr spec;

    // Per-seq storage for the deprecated `prompt` pointer in
    // common_speculative_draft_params (kept alive across draft() calls).
    std::vector<llama_tokens> prompts;

    // Per-seq result buffer the draft() call writes into.
    std::vector<llama_tokens> results;

    uint32_t n_seq       = 0;
    int32_t  n_draft_max = 0;
    int32_t  n_min       = 0;
    float    p_min       = 0.0f;
};

extern "C" mtp_session * mtp_session_new(
        llama_context *             ctx_tgt,
        llama_context *             ctx_dft,
        const mtp_session_config * config) {
    if (ctx_tgt == nullptr || ctx_dft == nullptr || config == nullptr) {
        return nullptr;
    }
    if (config->n_seq == 0 || config->n_draft_max <= 0) {
        return nullptr;
    }

    common_params_speculative sparams;
    sparams.types         = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
    sparams.draft.ctx_tgt = ctx_tgt;
    sparams.draft.ctx_dft = ctx_dft;
    sparams.draft.n_max   = config->n_draft_max;
    sparams.draft.n_min   = config->n_min;
    sparams.draft.p_min   = config->p_min;

    common_speculative * raw = common_speculative_init(sparams, config->n_seq);
    if (raw == nullptr) {
        return nullptr;
    }

    auto * s = new mtp_session;
    s->spec.reset(raw);
    s->prompts.resize(config->n_seq);
    s->results.resize(config->n_seq);
    s->n_seq       = config->n_seq;
    s->n_draft_max = config->n_draft_max;
    s->n_min       = config->n_min;
    s->p_min       = config->p_min;
    return s;
}

extern "C" void mtp_session_free(mtp_session * s) {
    delete s;
}

extern "C" bool mtp_session_need_embd(const mtp_session * s) {
    if (s == nullptr) {
        return false;
    }
    return common_speculative_need_embd(s->spec.get());
}

extern "C" bool mtp_session_need_embd_pre_norm(const mtp_session * s) {
    if (s == nullptr) {
        return false;
    }
    return common_speculative_need_embd_pre_norm(s->spec.get());
}

extern "C" void mtp_session_begin(
        mtp_session *       s,
        int32_t             seq_id,
        const llama_token * prompt,
        size_t              n_prompt) {
    if (s == nullptr || seq_id < 0 || (uint32_t) seq_id >= s->n_seq) {
        return;
    }

    auto & p = s->prompts[seq_id];
    p.assign(prompt, prompt + n_prompt);
    common_speculative_begin(s->spec.get(), seq_id, p);
}

extern "C" bool mtp_session_process(
        mtp_session *       s,
        const llama_batch * batch) {
    if (s == nullptr || batch == nullptr) {
        return false;
    }
    return common_speculative_process(s->spec.get(), *batch);
}

extern "C" void mtp_session_draft(
        mtp_session * s,
        int32_t       seq_id,
        llama_pos     n_past,
        llama_token   id_last,
        llama_token * out_tokens,
        int32_t *     out_n_tokens) {
    if (s == nullptr || out_tokens == nullptr || out_n_tokens == nullptr) {
        if (out_n_tokens) *out_n_tokens = 0;
        return;
    }

    const int32_t cap = *out_n_tokens;
    *out_n_tokens = 0;

    if (seq_id < 0 || (uint32_t) seq_id >= s->n_seq) {
        return;
    }

    auto & dp = common_speculative_get_draft_params(s->spec.get(), seq_id);
    auto & result = s->results[seq_id];
    result.clear();

    dp.drafting = true;
    dp.n_max    = s->n_draft_max;
    dp.n_past   = n_past;
    dp.id_last  = id_last;
    dp.prompt   = &s->prompts[seq_id];
    dp.result   = &result;

    common_speculative_draft(s->spec.get());

    const int32_t n = (int32_t) result.size();
    const int32_t to_copy = n < cap ? n : cap;
    for (int32_t i = 0; i < to_copy; ++i) {
        out_tokens[i] = result[i];
    }
    *out_n_tokens = to_copy;
}

extern "C" void mtp_session_accept(
        mtp_session * s,
        int32_t       seq_id,
        uint16_t      n_accepted) {
    if (s == nullptr || seq_id < 0 || (uint32_t) seq_id >= s->n_seq) {
        return;
    }
    common_speculative_accept(s->spec.get(), seq_id, n_accepted);
}

extern "C" void mtp_session_print_stats(const mtp_session * s) {
    if (s == nullptr) {
        return;
    }
    common_speculative_print_stats(s->spec.get());
}

extern "C" int32_t mtp_session_n_max(const mtp_session * s) {
    return s ? s->n_draft_max : 0;
}