ik-llama-cpp-sys 0.1.2

Low level bindings to ik_llama.cpp (ikawrakow's SOTA-quant fork)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

#pragma once

#include "ggml.h"
#include "ggml-backend.h"

#include <string>
#include <vector>
#include <unordered_map>
#include <atomic>
#include <sys/stat.h>
#include <fstream>

struct llama_model;
struct llama_model_loader;

struct tensor_reload_source {
    std::string   path;
    size_t        data_offset   = 0;
    size_t        nbytes        = 0;
    int64_t       last_mtime    = 0;
    int64_t       last_mtime_ns = 0;

    ggml_backend_buffer_t original_buffer = nullptr;
    void                * original_data   = nullptr;
    ggml_type             original_type     = GGML_TYPE_COUNT;
    size_t                original_nbytes   = 0;
    int64_t               original_ne[GGML_MAX_DIMS];
    size_t                original_nb[GGML_MAX_DIMS];

    struct split_info {
        int64_t ne[GGML_MAX_DIMS];
        size_t  nb[GGML_MAX_DIMS];
        void  * data;
        ggml_backend_buffer_t buffer;
        struct ggml_tensor * tensor = nullptr;
    };
    std::vector<split_info> original_splits;

    std::vector<std::string> sibling_names;
    ggml_split_tensor_t    * original_extra = nullptr;

    enum class reload_state {
        UNINITIALIZED,
        ON_ORIGINAL,
        DETACHED,
        FALLBACK_CPU
    };
    reload_state state = reload_state::UNINITIALIZED;
};

struct reload_info {
    std::unordered_map<std::string, tensor_reload_source> tensor_reload_sources;
    std::atomic<bool> reload_snapshots_done{false};

    reload_info(const llama_model_loader & ml);

    bool reload_tensor(const char * name, llama_model & model);
    bool reload_changed_tensors(llama_model & model);
    void snapshot_all_reload_tensors(llama_model & model);
};