1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
#pragma once
#include "ggml.h"
#include "llama.h"
#include <string>
#include <vector>
enum common_params_fit_status {
COMMON_PARAMS_FIT_STATUS_SUCCESS = 0, // found allocations that are projected to fit
COMMON_PARAMS_FIT_STATUS_FAILURE = 1, // could not find allocations that are projected to fit
COMMON_PARAMS_FIT_STATUS_ERROR = 2, // a hard error occurred, e.g. because no model could be found at the specified path
};
// a second model that shares the devices of the main model, e.g. a draft model
// - its context follows the context of the main model, so its memory is measured again whenever that context changes
// - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context
struct common_fit_extra_model {
const char * path_model;
llama_model_params * mparams;
llama_context_params * cparams;
bool shares_model;
};
// fits mparams and cparams to free device memory (assumes system memory is unlimited)
// - returns true if the parameters could be successfully modified to fit device memory
// - this function is NOT thread safe because it modifies the global llama logger state
// - only parameters that have the same value as in llama_default_model_params are modified
// with the exception of the context size which is modified if and only if equal to 0
common_params_fit_status common_fit_params(
const char * path_model,
llama_model_params * mparams,
llama_context_params * cparams,
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
size_t * margins, // margins of memory to leave per device in bytes
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
// print estimated memory to stdout
void common_fit_print(
const char * path_model,
llama_model_params * mparams,
llama_context_params * cparams);
void common_memory_breakdown_print(const llama_context * ctx);
// one row of the memory breakdown: a device, the host or another buffer type
struct common_memory_breakdown_row {
enum kind_t { DEVICE, HOST, BUFFER_TYPE };
kind_t kind = DEVICE;
std::string name; // device or buffer type name, "Host" for the host row
std::string description; // device description, empty otherwise
int64_t total = 0; // device memory in bytes, 0 unless kind == DEVICE
int64_t free = 0; // free device memory in bytes, 0 unless kind == DEVICE
size_t model = 0; // bytes held by this context, by kind of buffer
size_t context = 0;
size_t compute = 0;
};
// memory a model needs at the min and max context --fit may pick
struct common_memory_reports {
uint32_t n_ctx_train = 0;
uint32_t n_ctx_max = 0; // the largest context that fits the free device memory right now
uint32_t n_ctx_min = 0; // --fit-ctx (the smallest context --fit may pick), with the placement set in params
std::vector<common_memory_breakdown_row> info_max; // breakdown at n_ctx_max
std::vector<common_memory_breakdown_row> info_min; // breakdown at n_ctx_min
};
// rows of common_memory_breakdown_print: model devices first, then host
std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const llama_context * ctx);
// load model + context with no_alloc and return the memory breakdown, params are used as-is
// n_ctx_train and n_ctx are optional outputs; throws on failure
std::vector<common_memory_breakdown_row> common_fit_measure(
const char * path_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
uint32_t * n_ctx_train,
uint32_t * n_ctx,
ggml_log_level log_level);
struct common_params;
// measure min and max without allocating, params may be modified as with --fit; throws on failure
common_memory_reports common_measure_memory(common_params & params);
struct common_device_memory_data {
int64_t total;
int64_t free;
size_t model;
size_t context;
size_t compute;
};
using common_device_memory_data_vec = std::vector<common_device_memory_data>;
// Load a model + context with no_alloc and return the per-device memory breakdown.
common_device_memory_data_vec common_get_device_memory_data(
const char * path_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
uint32_t & hp_ngl,
uint32_t & hp_n_ctx_train,
uint32_t & hp_n_expert,
ggml_log_level log_level);