rig/providers/together/
completion.rs

1// ================================================================
2//! Together AI Completion Integration
3//! From [Together AI Reference](https://docs.together.ai/docs/chat-overview)
4// ================================================================
5
6use crate::{
7    completion::{self, CompletionError},
8    json_utils,
9    providers::openai,
10};
11
12use super::client::{Client, together_ai_api_types::ApiResponse};
13use crate::completion::CompletionRequest;
14use crate::streaming::StreamingCompletionResponse;
15use serde_json::json;
16
17// ================================================================
18// Together Completion Models
19// ================================================================
20
21pub const YI_34B_CHAT: &str = "zero-one-ai/Yi-34B-Chat";
22pub const OLMO_7B_INSTRUCT: &str = "allenai/OLMo-7B-Instruct";
23pub const CHRONOS_HERMES_13B: &str = "Austism/chronos-hermes-13b";
24pub const ML318BR: &str = "carson/ml318br";
25pub const DOLPHIN_2_5_MIXTRAL_8X7B: &str = "cognitivecomputations/dolphin-2.5-mixtral-8x7b";
26pub const DBRX_INSTRUCT: &str = "databricks/dbrx-instruct";
27pub const DEEPSEEK_LLM_67B_CHAT: &str = "deepseek-ai/deepseek-llm-67b-chat";
28pub const DEEPSEEK_CODER_33B_INSTRUCT: &str = "deepseek-ai/deepseek-coder-33b-instruct";
29pub const PLATYPUS2_70B_INSTRUCT: &str = "garage-bAInd/Platypus2-70B-instruct";
30pub const GEMMA_2_9B_IT: &str = "google/gemma-2-9b-it";
31pub const GEMMA_2B_IT: &str = "google/gemma-2b-it";
32pub const GEMMA_2_27B_IT: &str = "google/gemma-2-27b-it";
33pub const GEMMA_7B_IT: &str = "google/gemma-7b-it";
34pub const LLAMA_3_70B_INSTRUCT_GRADIENT_1048K: &str =
35    "gradientai/Llama-3-70B-Instruct-Gradient-1048k";
36pub const MYTHOMAX_L2_13B: &str = "Gryphe/MythoMax-L2-13b";
37pub const MYTHOMAX_L2_13B_LITE: &str = "Gryphe/MythoMax-L2-13b-Lite";
38pub const LLAVA_NEXT_MISTRAL_7B: &str = "llava-hf/llava-v1.6-mistral-7b-hf";
39pub const ZEPHYR_7B_BETA: &str = "HuggingFaceH4/zephyr-7b-beta";
40pub const KOALA_7B: &str = "togethercomputer/Koala-7B";
41pub const VICUNA_7B_V1_3: &str = "lmsys/vicuna-7b-v1.3";
42pub const VICUNA_13B_V1_5_16K: &str = "lmsys/vicuna-13b-v1.5-16k";
43pub const VICUNA_13B_V1_5: &str = "lmsys/vicuna-13b-v1.5";
44pub const VICUNA_13B_V1_3: &str = "lmsys/vicuna-13b-v1.3";
45pub const KOALA_13B: &str = "togethercomputer/Koala-13B";
46pub const VICUNA_7B_V1_5: &str = "lmsys/vicuna-7b-v1.5";
47pub const CODE_LLAMA_34B_INSTRUCT: &str = "codellama/CodeLlama-34b-Instruct-hf";
48pub const LLAMA_3_8B_CHAT_HF_INT4: &str = "togethercomputer/Llama-3-8b-chat-hf-int4";
49pub const LLAMA_3_2_90B_VISION_INSTRUCT_TURBO: &str =
50    "meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo";
51pub const LLAMA_3_2_11B_VISION_INSTRUCT_TURBO: &str =
52    "meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo";
53pub const LLAMA_3_2_3B_INSTRUCT_TURBO: &str = "meta-llama/Llama-3.2-3B-Instruct-Turbo";
54pub const LLAMA_3_8B_CHAT_HF_INT8: &str = "togethercomputer/Llama-3-8b-chat-hf-int8";
55pub const LLAMA_3_1_70B_INSTRUCT_TURBO: &str = "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo";
56pub const LLAMA_2_13B_CHAT: &str = "meta-llama/Llama-2-13b-chat-hf";
57pub const LLAMA_3_70B_INSTRUCT_LITE: &str = "meta-llama/Meta-Llama-3-70B-Instruct-Lite";
58pub const LLAMA_3_8B_CHAT_HF: &str = "meta-llama/Llama-3-8b-chat-hf";
59pub const LLAMA_3_70B_CHAT_HF: &str = "meta-llama/Llama-3-70b-chat-hf";
60pub const LLAMA_3_8B_INSTRUCT_TURBO: &str = "meta-llama/Meta-Llama-3-8B-Instruct-Turbo";
61pub const LLAMA_3_8B_INSTRUCT_LITE: &str = "meta-llama/Meta-Llama-3-8B-Instruct-Lite";
62pub const LLAMA_3_1_405B_INSTRUCT_LITE_PRO: &str =
63    "meta-llama/Meta-Llama-3.1-405B-Instruct-Lite-Pro";
64pub const LLAMA_2_7B_CHAT: &str = "meta-llama/Llama-2-7b-chat-hf";
65pub const LLAMA_3_1_405B_INSTRUCT_TURBO: &str = "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo";
66pub const LLAMA_VISION_FREE: &str = "meta-llama/Llama-Vision-Free";
67pub const LLAMA_3_70B_INSTRUCT_TURBO: &str = "meta-llama/Meta-Llama-3-70B-Instruct-Turbo";
68pub const LLAMA_3_1_8B_INSTRUCT_TURBO: &str = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo";
69pub const CODE_LLAMA_7B_INSTRUCT_TOGETHER: &str = "togethercomputer/CodeLlama-7b-Instruct";
70pub const CODE_LLAMA_34B_INSTRUCT_TOGETHER: &str = "togethercomputer/CodeLlama-34b-Instruct";
71pub const CODE_LLAMA_13B_INSTRUCT: &str = "codellama/CodeLlama-13b-Instruct-hf";
72pub const CODE_LLAMA_13B_INSTRUCT_TOGETHER: &str = "togethercomputer/CodeLlama-13b-Instruct";
73pub const LLAMA_2_13B_CHAT_TOGETHER: &str = "togethercomputer/llama-2-13b-chat";
74pub const LLAMA_2_7B_CHAT_TOGETHER: &str = "togethercomputer/llama-2-7b-chat";
75pub const LLAMA_3_8B_INSTRUCT: &str = "meta-llama/Meta-Llama-3-8B-Instruct";
76pub const LLAMA_3_70B_INSTRUCT: &str = "meta-llama/Meta-Llama-3-70B-Instruct";
77pub const CODE_LLAMA_70B_INSTRUCT: &str = "codellama/CodeLlama-70b-Instruct-hf";
78pub const LLAMA_2_70B_CHAT_TOGETHER: &str = "togethercomputer/llama-2-70b-chat";
79pub const LLAMA_3_1_8B_INSTRUCT_REFERENCE: &str = "meta-llama/Meta-Llama-3.1-8B-Instruct-Reference";
80pub const LLAMA_3_1_70B_INSTRUCT_REFERENCE: &str =
81    "meta-llama/Meta-Llama-3.1-70B-Instruct-Reference";
82pub const WIZARDLM_2_8X22B: &str = "microsoft/WizardLM-2-8x22B";
83pub const MISTRAL_7B_INSTRUCT_V0_1: &str = "mistralai/Mistral-7B-Instruct-v0.1";
84pub const MISTRAL_7B_INSTRUCT_V0_2: &str = "mistralai/Mistral-7B-Instruct-v0.2";
85pub const MISTRAL_7B_INSTRUCT_V0_3: &str = "mistralai/Mistral-7B-Instruct-v0.3";
86pub const MIXTRAL_8X7B_INSTRUCT_V0_1: &str = "mistralai/Mixtral-8x7B-Instruct-v0.1";
87pub const MIXTRAL_8X22B_INSTRUCT_V0_1: &str = "mistralai/Mixtral-8x22B-Instruct-v0.1";
88pub const NOUS_HERMES_2_MIXTRAL_8X7B_DPO: &str = "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO";
89pub const NOUS_HERMES_LLAMA2_70B: &str = "NousResearch/Nous-Hermes-Llama2-70b";
90pub const NOUS_HERMES_2_MIXTRAL_8X7B_SFT: &str = "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT";
91pub const NOUS_HERMES_LLAMA2_13B: &str = "NousResearch/Nous-Hermes-Llama2-13b";
92pub const NOUS_HERMES_2_MISTRAL_DPO: &str = "NousResearch/Nous-Hermes-2-Mistral-7B-DPO";
93pub const NOUS_HERMES_LLAMA2_7B: &str = "NousResearch/Nous-Hermes-llama-2-7b";
94pub const NOUS_CAPYBARA_V1_9: &str = "NousResearch/Nous-Capybara-7B-V1p9";
95pub const HERMES_2_THETA_LLAMA_3_70B: &str = "NousResearch/Hermes-2-Theta-Llama-3-70B";
96pub const OPENCHAT_3_5: &str = "openchat/openchat-3.5-1210";
97pub const OPENORCA_MISTRAL_7B_8K: &str = "Open-Orca/Mistral-7B-OpenOrca";
98pub const QWEN_2_72B_INSTRUCT: &str = "Qwen/Qwen2-72B-Instruct";
99pub const QWEN2_5_72B_INSTRUCT_TURBO: &str = "Qwen/Qwen2.5-72B-Instruct-Turbo";
100pub const QWEN2_5_7B_INSTRUCT_TURBO: &str = "Qwen/Qwen2.5-7B-Instruct-Turbo";
101pub const QWEN1_5_110B_CHAT: &str = "Qwen/Qwen1.5-110B-Chat";
102pub const QWEN1_5_72B_CHAT: &str = "Qwen/Qwen1.5-72B-Chat";
103pub const QWEN_2_1_5B_INSTRUCT: &str = "Qwen/Qwen2-1.5B-Instruct";
104pub const QWEN_2_7B_INSTRUCT: &str = "Qwen/Qwen2-7B-Instruct";
105pub const QWEN1_5_14B_CHAT: &str = "Qwen/Qwen1.5-14B-Chat";
106pub const QWEN1_5_1_8B_CHAT: &str = "Qwen/Qwen1.5-1.8B-Chat";
107pub const QWEN1_5_32B_CHAT: &str = "Qwen/Qwen1.5-32B-Chat";
108pub const QWEN1_5_7B_CHAT: &str = "Qwen/Qwen1.5-7B-Chat";
109pub const QWEN1_5_0_5B_CHAT: &str = "Qwen/Qwen1.5-0.5B-Chat";
110pub const QWEN1_5_4B_CHAT: &str = "Qwen/Qwen1.5-4B-Chat";
111pub const SNORKEL_MISTRAL_PAIRRM_DPO: &str = "snorkelai/Snorkel-Mistral-PairRM-DPO";
112pub const SNOWFLAKE_ARCTIC_INSTRUCT: &str = "Snowflake/snowflake-arctic-instruct";
113pub const ALPACA_7B: &str = "togethercomputer/alpaca-7b";
114pub const OPENHERMES_2_MISTRAL_7B: &str = "teknium/OpenHermes-2-Mistral-7B";
115pub const OPENHERMES_2_5_MISTRAL_7B: &str = "teknium/OpenHermes-2p5-Mistral-7B";
116pub const GUANACO_65B: &str = "togethercomputer/guanaco-65b";
117pub const GUANACO_13B: &str = "togethercomputer/guanaco-13b";
118pub const GUANACO_33B: &str = "togethercomputer/guanaco-33b";
119pub const GUANACO_7B: &str = "togethercomputer/guanaco-7b";
120pub const REMM_SLERP_L2_13B: &str = "Undi95/ReMM-SLERP-L2-13B";
121pub const TOPPY_M_7B: &str = "Undi95/Toppy-M-7B";
122pub const SOLAR_10_7B_INSTRUCT_V1: &str = "upstage/SOLAR-10.7B-Instruct-v1.0";
123pub const SOLAR_10_7B_INSTRUCT_V1_INT4: &str = "togethercomputer/SOLAR-10.7B-Instruct-v1.0-int4";
124pub const WIZARDLM_13B_V1_2: &str = "WizardLM/WizardLM-13B-V1.2";
125
126// =================================================================
127// Rig Implementation Types
128// =================================================================
129
130#[derive(Clone)]
131pub struct CompletionModel {
132    pub(crate) client: Client,
133    pub model: String,
134}
135
136impl CompletionModel {
137    pub fn new(client: Client, model: &str) -> Self {
138        Self {
139            client,
140            model: model.to_string(),
141        }
142    }
143
144    pub(crate) fn create_completion_request(
145        &self,
146        completion_request: completion::CompletionRequest,
147    ) -> Result<serde_json::Value, CompletionError> {
148        let mut full_history: Vec<openai::Message> = match &completion_request.preamble {
149            Some(preamble) => vec![openai::Message::system(preamble)],
150            None => vec![],
151        };
152        if let Some(docs) = completion_request.normalized_documents() {
153            let docs: Vec<openai::Message> = docs.try_into()?;
154            full_history.extend(docs);
155        }
156        let chat_history: Vec<openai::Message> = completion_request
157            .chat_history
158            .into_iter()
159            .map(|message| message.try_into())
160            .collect::<Result<Vec<Vec<openai::Message>>, _>>()?
161            .into_iter()
162            .flatten()
163            .collect();
164
165        full_history.extend(chat_history);
166
167        let mut request = if completion_request.tools.is_empty() {
168            json!({
169                "model": self.model,
170                "messages": full_history,
171                "temperature": completion_request.temperature,
172            })
173        } else {
174            json!({
175                "model": self.model,
176                "messages": full_history,
177                "temperature": completion_request.temperature,
178                "tools": completion_request.tools.into_iter().map(openai::ToolDefinition::from).collect::<Vec<_>>(),
179                "tool_choice": "auto",
180            })
181        };
182        request = if let Some(params) = completion_request.additional_params {
183            json_utils::merge(request, params)
184        } else {
185            request
186        };
187        Ok(request)
188    }
189}
190
191impl completion::CompletionModel for CompletionModel {
192    type Response = openai::CompletionResponse;
193    type StreamingResponse = openai::StreamingCompletionResponse;
194
195    #[cfg_attr(feature = "worker", worker::send)]
196    async fn completion(
197        &self,
198        completion_request: completion::CompletionRequest,
199    ) -> Result<completion::CompletionResponse<openai::CompletionResponse>, CompletionError> {
200        let request = self.create_completion_request(completion_request)?;
201
202        let response = self
203            .client
204            .post("/v1/chat/completions")
205            .json(&request)
206            .send()
207            .await?;
208
209        if response.status().is_success() {
210            let t = response.text().await?;
211            tracing::debug!(target: "rig", "Together completion error: {}", t);
212
213            match serde_json::from_str::<ApiResponse<openai::CompletionResponse>>(&t)? {
214                ApiResponse::Ok(response) => {
215                    tracing::info!(target: "rig",
216                        "Together completion token usage: {:?}",
217                        response.usage.clone().map(|usage| format!("{usage}")).unwrap_or("N/A".to_string())
218                    );
219                    response.try_into()
220                }
221                ApiResponse::Error(err) => Err(CompletionError::ProviderError(err.error)),
222            }
223        } else {
224            Err(CompletionError::ProviderError(response.text().await?))
225        }
226    }
227
228    #[cfg_attr(feature = "worker", worker::send)]
229    async fn stream(
230        &self,
231        request: CompletionRequest,
232    ) -> Result<StreamingCompletionResponse<Self::StreamingResponse>, CompletionError> {
233        CompletionModel::stream(self, request).await
234    }
235}
rig/providers/together/completion.rs

rig/providers/together/
completion.rs
