llama-cpp-bindings-sys 0.8.0

Low level bindings to llama.cpp
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

#include "chunked_thinking.h"

#include "llama.cpp/common/chat-auto-parser.h"
#include "llama.cpp/common/chat.h"

#include <algorithm>
#include <exception>
#include <nlohmann/json.hpp>
#include <string>
#include <string_view>

namespace marker_probes {

namespace {

constexpr std::string_view REASON_PROBE   = "__PADDLER_REASON_PROBE_3F4A8C__";
constexpr std::string_view RESPONSE_PROBE = "__PADDLER_RESPONSE_PROBE_3F4A8C__";

std::string trim_copy(std::string_view input) {
    auto first = input.find_first_not_of(" \t\r\n");
    if (first == std::string_view::npos) {
        return {};
    }
    auto last = input.find_last_not_of(" \t\r\n");
    return std::string(input.substr(first, last - first + 1));
}

bool render_template(const common_chat_template & tmpl,
                     const autoparser::generation_params & params,
                     std::string & out) {
    try {
        out = common_chat_template_direct_apply(tmpl, params);
        return true;
    } catch (const std::exception &) {
        return false;
    } catch (...) {
        return false;
    }
}

autoparser::generation_params plain_text_params() {
    autoparser::generation_params params;
    params.add_generation_prompt = false;
    params.enable_thinking = true;
    params.is_inference = false;
    params.add_inference = false;
    params.mark_input = false;
    params.messages = nlohmann::ordered_json::array({
        nlohmann::ordered_json{ { "role", "user" }, { "content", "U" } },
        nlohmann::ordered_json{ { "role", "assistant" }, { "content", std::string(RESPONSE_PROBE) } },
    });
    return params;
}

autoparser::generation_params chunked_thinking_params() {
    autoparser::generation_params params;
    params.add_generation_prompt = false;
    params.enable_thinking = true;
    params.is_inference = false;
    params.add_inference = false;
    params.mark_input = false;
    params.messages = nlohmann::ordered_json::array({
        nlohmann::ordered_json{ { "role", "user" }, { "content", "U" } },
        nlohmann::ordered_json{
            { "role", "assistant" },
            { "content", nlohmann::ordered_json::array({
                  nlohmann::ordered_json{ { "type", "thinking" }, { "thinking", std::string(REASON_PROBE) } },
                  nlohmann::ordered_json{ { "type", "text" }, { "text", std::string(RESPONSE_PROBE) } },
              }) },
        },
    });
    return params;
}

bool contains(std::string_view haystack, std::string_view needle) {
    return haystack.find(needle) != std::string_view::npos;
}

}  // namespace

probe_result chunked_thinking(const common_chat_template & tmpl) {
    probe_result result;

    std::string render_plain;
    if (!render_template(tmpl, plain_text_params(), render_plain)) {
        return result;
    }

    std::string render_chunked;
    if (!render_template(tmpl, chunked_thinking_params(), render_chunked)) {
        return result;
    }

    if (!contains(render_chunked, REASON_PROBE) || !contains(render_chunked, RESPONSE_PROBE)) {
        return result;
    }

    const std::size_t plain_size = render_plain.size();
    const std::size_t chunked_size = render_chunked.size();
    const std::size_t min_size = std::min(plain_size, chunked_size);

    std::size_t common_prefix = 0;
    while (common_prefix < min_size && render_plain[common_prefix] == render_chunked[common_prefix]) {
        ++common_prefix;
    }

    std::size_t common_suffix = 0;
    while (common_suffix < min_size - common_prefix
           && render_plain[plain_size - 1 - common_suffix] == render_chunked[chunked_size - 1 - common_suffix]) {
        ++common_suffix;
    }

    if (common_prefix + common_suffix > chunked_size) {
        return result;
    }

    std::string_view diff_slice(render_chunked);
    diff_slice = diff_slice.substr(common_prefix, chunked_size - common_prefix - common_suffix);

    auto reason_pos = diff_slice.find(REASON_PROBE);
    if (reason_pos == std::string_view::npos) {
        return result;
    }

    std::string start = trim_copy(diff_slice.substr(0, reason_pos));
    std::string end = trim_copy(diff_slice.substr(reason_pos + REASON_PROBE.size()));

    if (start.empty() || end.empty()) {
        return result;
    }
    if (contains(start, REASON_PROBE) || contains(start, RESPONSE_PROBE)) {
        return result;
    }
    if (contains(end, REASON_PROBE) || contains(end, RESPONSE_PROBE)) {
        return result;
    }

    result.start = std::move(start);
    result.end = std::move(end);
    result.found = true;
    return result;
}

}  // namespace marker_probes