rs-llama-cpp 0.1.67

Automated Rust bindings generation for LLaMA.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

#pragma once

#include "run-inference.h"
#include <stdint.h>

extern "C" {
// Generated by ChatGPT with this prompt: Create a C-compatible struct for the
// given C++ struct and write a converter function to convert the C-compatible
// struct to the C++ struct. Use char* for strings.
struct gpt_params_c {
  int32_t seed;
  int32_t n_threads;
  int32_t n_predict;
  int32_t n_ctx;
  int32_t n_batch;
  int32_t n_keep;
  int32_t n_gpu_layers;
  int32_t main_gpu;
  float tensor_split[LLAMA_MAX_DEVICES];

  int32_t top_k;
  float top_p;
  float tfs_z;
  float typical_p;
  float temp;
  float repeat_penalty;
  int32_t repeat_last_n;
  float frequency_penalty;
  float presence_penalty;
  int mirostat;
  float mirostat_tau;
  float mirostat_eta;

  char *model;
  char *model_alias;
  char *prompt;
  char *path_prompt_cache;
  char *input_prefix;
  char *input_suffix;
  // char** antiprompt;

  char *lora_adapter;
  char *lora_base;

  bool memory_f16;
  bool random_prompt;
  bool use_color;
  bool interactive;
  bool prompt_cache_all;
  bool prompt_cache_ro;

  bool embedding;
  bool interactive_first;
  bool multiline_input;

  bool instruct;
  bool penalize_nl;
  bool perplexity;
  bool use_mmap;
  bool use_mlock;
  bool mem_test;
  bool export_cgraph;
  bool verbose_prompt;
};

void rs_llama_cpp_run_inference(gpt_params_c params, token_callback callback);
}