onde 1.1.0

On-device inference engine for Apple silicon.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

package com.ondeinference

import expo.modules.kotlin.modules.Module
import expo.modules.kotlin.modules.ModuleDefinition

class OndeInferenceModule : Module() {
    companion object {
        init {
            System.loadLibrary("onde_react_native")
        }
    }

    private var enginePtr: Long = 0L

    // -- JNI native methods (implemented in Rust via #[cfg(target_os = "android")]) --

    private external fun nativeEngineCreate(): Long
    private external fun nativeEngineDestroy(engine: Long)
    private external fun nativeEngineLoadDefaultModel(
        engine: Long,
        systemPrompt: String?,
        samplingJson: String?
    ): String

    private external fun nativeEngineLoadModel(
        engine: Long,
        configJson: String,
        systemPrompt: String?,
        samplingJson: String?
    ): String

    private external fun nativeEngineUnloadModel(engine: Long): String
    private external fun nativeEngineIsLoaded(engine: Long): Boolean
    private external fun nativeEngineInfo(engine: Long): String
    private external fun nativeEngineSetSystemPrompt(engine: Long, prompt: String)
    private external fun nativeEngineClearSystemPrompt(engine: Long)
    private external fun nativeEngineSetSampling(engine: Long, samplingJson: String)
    private external fun nativeEngineHistory(engine: Long): String
    private external fun nativeEngineClearHistory(engine: Long): Long
    private external fun nativeEnginePushHistory(engine: Long, messageJson: String)
    private external fun nativeEngineSendMessage(engine: Long, message: String): String
    private external fun nativeEngineGenerate(
        engine: Long,
        messagesJson: String,
        samplingJson: String?
    ): String

    private external fun nativeDefaultModelConfig(): String
    private external fun nativeQwen251_5bConfig(): String
    private external fun nativeQwen253bConfig(): String
    private external fun nativeDefaultSamplingConfig(): String
    private external fun nativeDeterministicSamplingConfig(): String
    private external fun nativeMobileSamplingConfig(): String

    // -- Module definition --

    override fun definition() = ModuleDefinition {
        Name("OndeInference")

        OnCreate {
            enginePtr = nativeEngineCreate()
        }

        OnDestroy {
            if (enginePtr != 0L) {
                nativeEngineDestroy(enginePtr)
                enginePtr = 0L
            }
        }

        // -- Sync functions --

        Function("isLoaded") {
            if (enginePtr == 0L) return@Function false
            nativeEngineIsLoaded(enginePtr)
        }

        Function("setSystemPrompt") { prompt: String ->
            if (enginePtr != 0L) nativeEngineSetSystemPrompt(enginePtr, prompt)
        }

        Function("clearSystemPrompt") {
            if (enginePtr != 0L) nativeEngineClearSystemPrompt(enginePtr)
        }

        Function("setSampling") { samplingJson: String ->
            if (enginePtr != 0L) nativeEngineSetSampling(enginePtr, samplingJson)
        }

        Function("clearHistory") {
            if (enginePtr == 0L) return@Function 0L
            nativeEngineClearHistory(enginePtr)
        }

        Function("pushHistory") { messageJson: String ->
            if (enginePtr != 0L) nativeEnginePushHistory(enginePtr, messageJson)
        }

        // -- Config free functions (no engine needed) --

        Function("defaultModelConfig") { nativeDefaultModelConfig() }
        Function("qwen251_5bConfig") { nativeQwen251_5bConfig() }
        Function("qwen253bConfig") { nativeQwen253bConfig() }
        Function("defaultSamplingConfig") { nativeDefaultSamplingConfig() }
        Function("deterministicSamplingConfig") { nativeDeterministicSamplingConfig() }
        Function("mobileSamplingConfig") { nativeMobileSamplingConfig() }

        // -- Async functions --

        AsyncFunction("loadDefaultModel") { systemPrompt: String?, samplingJson: String? ->
            check(enginePtr != 0L) { "Engine not initialized" }
            nativeEngineLoadDefaultModel(enginePtr, systemPrompt, samplingJson)
        }

        AsyncFunction("loadModel") { configJson: String, systemPrompt: String?, samplingJson: String? ->
            check(enginePtr != 0L) { "Engine not initialized" }
            nativeEngineLoadModel(enginePtr, configJson, systemPrompt, samplingJson)
        }

        AsyncFunction("unloadModel") {
            check(enginePtr != 0L) { "Engine not initialized" }
            nativeEngineUnloadModel(enginePtr)
        }

        AsyncFunction("info") {
            check(enginePtr != 0L) { "Engine not initialized" }
            nativeEngineInfo(enginePtr)
        }

        AsyncFunction("history") {
            if (enginePtr == 0L) return@AsyncFunction "[]"
            nativeEngineHistory(enginePtr)
        }

        AsyncFunction("sendMessage") { message: String ->
            check(enginePtr != 0L) { "Engine not initialized" }
            nativeEngineSendMessage(enginePtr, message)
        }

        AsyncFunction("generate") { messagesJson: String, samplingJson: String? ->
            check(enginePtr != 0L) { "Engine not initialized" }
            nativeEngineGenerate(enginePtr, messagesJson, samplingJson)
        }
    }
}