llama-cpp-sys-4 0.7.0

Low Level Bindings to llama.cpp
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

function(ggml_get_flags CCID CCVER)
    set(C_FLAGS "")
    set(CXX_FLAGS "")

    if (CCID MATCHES "Clang")
        set(C_FLAGS   -Wunreachable-code-break -Wunreachable-code-return)
        set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)

        if (
            (CCID STREQUAL "Clang"      AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR
            (CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0)
        )
            list(APPEND C_FLAGS -Wdouble-promotion)
        endif()
    elseif (CCID STREQUAL "GNU")
        set(C_FLAGS   -Wdouble-promotion)
        set(CXX_FLAGS -Wno-array-bounds)

        if (CCVER VERSION_GREATER_EQUAL 8.1.0)
            list(APPEND CXX_FLAGS -Wextra-semi)
        endif()
    endif()

    set(GF_C_FLAGS   ${C_FLAGS}   PARENT_SCOPE)
    set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)
endfunction()

function(ggml_get_system_arch)
    if (CMAKE_OSX_ARCHITECTURES      STREQUAL "arm64" OR
        CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
        (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
            CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
        set(GGML_SYSTEM_ARCH "ARM" PARENT_SCOPE)
    elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
            CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
            (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
            CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64|amd64)$"))
        set(GGML_SYSTEM_ARCH "x86" PARENT_SCOPE)
    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc|power")
        set(GGML_SYSTEM_ARCH "PowerPC" PARENT_SCOPE)
    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
        set(GGML_SYSTEM_ARCH "loongarch64"  PARENT_SCOPE)
    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64")
        set(GGML_SYSTEM_ARCH "riscv64" PARENT_SCOPE)
    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x")
        set(GGML_SYSTEM_ARCH "s390x" PARENT_SCOPE)
    else()
        set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE)
    endif()
endfunction()

# Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS.
function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS)
    set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16)

    string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS)
    string(STRIP   "${FA_QUANTS}" FA_QUANTS)
    if (GGML_CUDA_FA_ALL_QUANTS)
        message(WARNING "GGML_CUDA_FA_ALL_QUANTS is deprecated, use GGML_CUDA_FA_QUANTS=all instead")
        set(FA_QUANTS all)
    endif()
    if (NOT FA_QUANTS)
        message(FATAL_ERROR "GGML_CUDA_FA_QUANTS must not be empty")
    endif()

    if (FA_QUANTS STREQUAL "all")
        set(FA_COMBINATIONS "")
        foreach (TYPE_V IN LISTS FA_TYPES)
            foreach (TYPE_K IN LISTS FA_TYPES)
                list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V})
            endforeach()
        endforeach()
    else()
        set(FA_COMBINATIONS f16-f16)

        string(REPLACE "," ";" FA_SELECTED "${FA_QUANTS}")
        foreach (COMBINATION IN LISTS FA_SELECTED)
            string(STRIP   "${COMBINATION}" COMBINATION)
            if (NOT COMBINATION MATCHES "^([a-z0-9_]+)-([a-z0-9_]+)$")
                message(FATAL_ERROR "GGML_CUDA_FA_QUANTS: \"${COMBINATION}\" is not \"all\" or a <type_K>-<type_V> combination")
            endif()
            set(TYPE_K ${CMAKE_MATCH_1})
            set(TYPE_V ${CMAKE_MATCH_2})
            foreach (TYPE ${TYPE_K} ${TYPE_V})
                if (NOT TYPE IN_LIST FA_TYPES)
                    message(FATAL_ERROR
                        "GGML_CUDA_FA_QUANTS: unknown type \"${TYPE}\" in \"${COMBINATION}\", must be one of: ${FA_TYPES}")
                endif()
            endforeach()
            list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V})
        endforeach()
    endif()
    list(REMOVE_DUPLICATES FA_COMBINATIONS)

    string(REPLACE ";" "," FA_QUANTS_DEFINE "${FA_QUANTS}")
    add_compile_definitions(GGML_CUDA_FA_QUANTS="${FA_QUANTS_DEFINE}")
    foreach (TYPE_V IN LISTS FA_TYPES)
        foreach (TYPE_K IN LISTS FA_TYPES)
            if ("${TYPE_K}-${TYPE_V}" IN_LIST FA_COMBINATIONS)
                set(COMPILED 1)
            else()
                set(COMPILED 0)
            endif()
            string(TOUPPER "GGML_CUDA_FA_${TYPE_K}_${TYPE_V}" COMBINATION_DEF)
            add_compile_definitions(${COMBINATION_DEF}=${COMPILED})
        endforeach()
    endforeach()

    message(STATUS "FlashAttention K-V type combinations: ${FA_COMBINATIONS}")

    set(SRCS "")
    foreach (COMBINATION IN LISTS FA_COMBINATIONS)
        set(SRC "${DIR}/template-instances/fattn-vec-instance-${COMBINATION}.cu")
        if (NOT EXISTS "${SRC}")
            message(FATAL_ERROR "FlashAttention template instance \"${SRC}\" does not exist")
        endif()
        list(APPEND SRCS "${SRC}")
    endforeach()

    set(${OUT_SRCS} ${SRCS} PARENT_SCOPE)
endfunction()