use crate::{
ByteStream, DevinAssistantContentPart, DevinChatRequest, DevinClientMetadata, DevinContentPart,
DevinError, DevinMessage, DevinModel, DevinStopReason, DevinStreamEvent, DevinTool, FetchLike,
FetchRequest, FetchResponse, UuidGenerator,
connect::{decode_connect_frames, encode_connect_frame, read_connect_trailer_error},
constants::{
DEVIN_ASSIGN_MODEL_PATH, DEVIN_AUTH_PATH, DEVIN_CHAT_PATH, DEVIN_DEFAULT_BASE_URL,
trim_trailing_slashes,
},
fetch::{default_fetch, read_all},
json::{parse_possibly_complete_json_throttled, parse_streaming_json},
metadata_with_client_metadata, normalize_devin_session_token,
proto::{
AssignModelRequest, AssignModelResponse, CacheControlType, ChatMessagePrompt,
ChatMessageRequestType, ChatMessageSource, ChatToolCall, ChatToolChoice,
ChatToolDefinition, CompletionConfiguration, ConversationalPlannerMode,
GetChatMessageRequest, GetChatMessageResponse, GetUserJwtRequest, GetUserJwtResponse,
ImageData, ModelAssignment, PromptCacheOptions, StopReason, chat_tool_choice,
},
unary::{UnaryResponse, decode_unary_response, request_unary},
};
use async_stream::try_stream;
use futures_core::Stream;
use futures_util::StreamExt;
use indexmap::IndexMap;
use prost::Message;
use serde_json::Value;
use std::{pin::Pin, sync::Arc};
use uuid::Uuid;
pub type DevinEventStream =
Pin<Box<dyn Stream<Item = Result<DevinStreamEvent, DevinError>> + Send>>;
#[derive(Clone)]
pub struct StreamDevinChatOptions {
pub token: Option<String>,
pub base_url: String,
pub fetch: Option<FetchLike>,
pub uuid: Option<UuidGenerator>,
pub client_metadata: Option<DevinClientMetadata>,
pub model: String,
pub model_config: Option<DevinModel>,
pub messages: Vec<DevinMessage>,
pub system_prompt: Vec<String>,
pub tools: Vec<DevinTool>,
pub conversation_id: Option<String>,
pub session_id: Option<String>,
pub max_tokens: Option<u64>,
pub temperature: Option<f64>,
pub top_p: Option<f64>,
pub stop_sequences: Vec<String>,
}
impl Default for StreamDevinChatOptions {
fn default() -> Self {
Self {
token: None,
base_url: DEVIN_DEFAULT_BASE_URL.into(),
fetch: None,
uuid: None,
client_metadata: None,
model: String::new(),
model_config: None,
messages: Vec::new(),
system_prompt: Vec::new(),
tools: Vec::new(),
conversation_id: None,
session_id: None,
max_tokens: None,
temperature: None,
top_p: None,
stop_sequences: Vec::new(),
}
}
}
impl StreamDevinChatOptions {
pub(crate) fn from_request(
request: DevinChatRequest,
token: Option<String>,
base_url: String,
fetch: FetchLike,
uuid: Option<UuidGenerator>,
client_metadata: Option<DevinClientMetadata>,
) -> Self {
Self {
token,
base_url,
fetch: Some(fetch),
uuid,
client_metadata,
model: request.model,
model_config: request.model_config,
messages: request.messages,
system_prompt: request.system_prompt,
tools: request.tools,
conversation_id: request.conversation_id,
session_id: request.session_id,
max_tokens: request.max_tokens,
temperature: request.temperature,
top_p: request.top_p,
stop_sequences: request.stop_sequences,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AuthMetadata {
pub user_jwt: String,
pub api_key: String,
pub base_url: Option<String>,
}
#[derive(Clone, Debug)]
struct StreamingToolCallState {
name: String,
arguments_json: String,
last_parse_attempt_length: usize,
}
#[allow(clippy::too_many_lines)] pub fn stream_devin_chat(options: StreamDevinChatOptions) -> DevinEventStream {
Box::pin(try_stream! {
let fetch = options.fetch.clone().unwrap_or_else(default_fetch);
let configured_base_url = if options.base_url == DEVIN_DEFAULT_BASE_URL {
options.model_config.as_ref().map_or(options.base_url.as_str(), |config| config.base_url.as_str())
} else { options.base_url.as_str() };
let base_url = trim_trailing_slashes(configured_base_url);
if options.model_config.as_ref().is_some_and(|config| config.id != options.model) { Err(DevinError::protocol("model_config.id must match model"))?; }
if options.model.starts_with("fusion-") && options.model.contains("-sidekick-") && options.model_config.as_ref().and_then(|config| config.request_model_id.as_ref()).is_none() {
Err(DevinError::protocol("Fusion pairing requires a discovered model_config"))?;
}
let token = options.token.as_deref().unwrap_or_default();
let (body, assigned_model) = open_chat_response(
&options,
token,
&base_url,
fetch,
)
.await?;
let frames = decode_connect_frames(body);
futures_util::pin_mut!(frames);
let mut tool_calls = IndexMap::<String, StreamingToolCallState>::new();
let mut active_tool_call_id = None::<String>;
let mut saw_tool_call = false;
let mut latest_stop_reason = StopReason::Unspecified;
let mut response_id = None;
let mut upstream_model = assigned_model;
while let Some(frame) = frames.next().await {
let frame = frame?;
if frame.end_stream {
if let Some(error) = read_connect_trailer_error(&frame.payload) {
Err(DevinError::protocol(error))?;
}
continue;
}
let message = GetChatMessageResponse::decode(frame.payload.as_slice())
.map_err(|error| DevinError::protocol(format!(
"invalid Devin chat protobuf response: {error}"
)))?;
if !message.message_id.is_empty() { response_id = Some(message.message_id.clone()); }
if let Some(uid) = message.actual_model_uid.filter(|uid| !uid.is_empty()) { upstream_model = Some(uid); }
if !message.delta_thinking.is_empty() {
yield DevinStreamEvent::ThinkingDelta {
delta: message.delta_thinking,
signature: (!message.delta_signature.is_empty())
.then_some(message.delta_signature),
};
}
if !message.delta_text.is_empty() {
yield DevinStreamEvent::TextDelta {
delta: message.delta_text,
};
}
for call in message.delta_tool_calls {
let id = if call.id.is_empty() {
active_tool_call_id.clone()
} else {
Some(call.id.clone())
};
let Some(id) = id else {
continue;
};
let previous = tool_calls.get(&id).cloned();
let (state, delta, parsed_arguments) =
advance_streaming_tool_call(previous.as_ref(), &call);
tool_calls.insert(id.clone(), state.clone());
active_tool_call_id = Some(id.clone());
saw_tool_call = true;
if previous.is_none() {
yield DevinStreamEvent::ToolCallStart {
id: id.clone(),
name: state.name.clone(),
};
}
if call.arguments_json.is_empty() { continue; }
yield DevinStreamEvent::ToolCallDelta {
id,
delta,
arguments: parsed_arguments,
};
}
if let Ok(reason) = StopReason::try_from(message.stop_reason) {
if reason != StopReason::Unspecified {
latest_stop_reason = reason;
}
}
if let Some(usage) = message.usage {
yield DevinStreamEvent::Usage {
input_tokens: usage.input_tokens,
output_tokens: usage.output_tokens,
cache_read_tokens: usage.cache_read_tokens,
cache_write_tokens: usage.cache_write_tokens,
};
}
if message.credit_cost != 0 || message.committed_credit_cost != 0 || message.committed_acu_cost != 0.0 {
yield DevinStreamEvent::Credits { cost: message.credit_cost, committed_cost: message.committed_credit_cost, acu_cost: message.committed_acu_cost };
}
}
for (id, state) in tool_calls {
yield DevinStreamEvent::ToolCallEnd {
id,
name: state.name,
arguments: parse_streaming_json(Some(&state.arguments_json)),
};
}
yield DevinStreamEvent::Done {
reason: if saw_tool_call {
DevinStopReason::ToolUse
} else if latest_stop_reason == StopReason::MaxTokens {
DevinStopReason::Length
} else {
DevinStopReason::Stop
},
response_id, upstream_model,
};
})
}
async fn open_chat_response(
options: &StreamDevinChatOptions,
token: &str,
base_url: &str,
fetch: FetchLike,
) -> Result<(ByteStream, Option<String>), DevinError> {
let auth = fetch_auth_metadata_with_client_metadata(
token,
base_url,
Arc::clone(&fetch),
options.client_metadata.as_ref(),
)
.await?;
let chat_base_url = auth.base_url.unwrap_or_else(|| base_url.to_owned());
let uuid = options
.uuid
.clone()
.unwrap_or_else(|| Arc::new(|| Uuid::new_v4().to_string()));
let cascade_id = options
.conversation_id
.as_ref()
.or(options.session_id.as_ref())
.cloned()
.unwrap_or_else(|| uuid());
let mut turn = options.clone();
turn.conversation_id = Some(cascade_id.clone());
let router = options.model_config.as_ref().map_or_else(
|| matches!(options.model.as_str(), "adaptive" | "subagent-default"),
|config| config.model_router,
);
let assignment = if router {
Some(assign_model(options, &auth.api_key, &chat_base_url, &cascade_id, &fetch).await?)
} else {
None
};
if let Some(assignment) = &assignment {
turn.model.clone_from(&assignment.model_uid);
if let Some(config) = &mut turn.model_config {
config.request_model_id = Some(assignment.model_uid.clone());
}
}
let mut request = build_chat_request(&turn, &auth.api_key, &auth.user_jwt, uuid.as_ref());
request.model_assignment_jwt = assignment
.as_ref()
.map(|assignment| assignment.assignment_jwt.clone());
let response = fetch(FetchRequest {
url: format!("{chat_base_url}{DEVIN_CHAT_PATH}"),
method: "POST".into(),
headers: IndexMap::from([
("content-type".into(), "application/connect+proto".into()),
("connect-protocol-version".into(), "1".into()),
("connect-content-encoding".into(), "gzip".into()),
("accept-encoding".into(), "identity".into()),
("user-agent".into(), "connect-go/1.18.1 (go1.26.3)".into()),
("connect-accept-encoding".into(), "gzip".into()),
]),
body: encode_connect_frame(&request.encode_to_vec(), true, false)?,
})
.await?;
Ok((
validate_chat_response(response).await?,
assignment.map(|assignment| assignment.model_uid),
))
}
async fn validate_chat_response(response: FetchResponse) -> Result<ByteStream, DevinError> {
let status = response.status;
let status_text = response.status_text.clone();
if (200..300).contains(&status) {
return response
.body
.ok_or_else(|| DevinError::protocol("Devin chat response body is empty"));
}
let body = read_all(response).await?;
Err(crate::error::http_error(
"chat",
status,
&status_text,
&body,
))
}
pub async fn fetch_auth_metadata(
token: &str,
base_url: &str,
fetch: FetchLike,
) -> Result<AuthMetadata, DevinError> {
fetch_auth_metadata_with_client_metadata(token, base_url, fetch, None).await
}
pub async fn fetch_auth_metadata_with_client_metadata(
token: &str,
base_url: &str,
fetch: FetchLike,
client_metadata: Option<&DevinClientMetadata>,
) -> Result<AuthMetadata, DevinError> {
let mut api_key = normalize_devin_session_token(Some(token));
let mut response = request_auth(&api_key, base_url, &fetch, client_metadata).await?;
if response.status == 401 && !token.is_empty() && token != api_key {
token.clone_into(&mut api_key);
response = request_auth(&api_key, base_url, &fetch, client_metadata).await?;
}
let decoded: GetUserJwtResponse = decode_unary_response("auth", &response)?;
if decoded.user_jwt.is_empty() {
return Err(DevinError::protocol(
"Devin auth returned an empty user JWT",
));
}
let custom = decoded.custom_api_server_url.trim();
Ok(AuthMetadata {
user_jwt: decoded.user_jwt,
api_key,
base_url: (!custom.is_empty()).then(|| trim_trailing_slashes(custom)),
})
}
async fn request_auth(
token: &str,
base_url: &str,
fetch: &FetchLike,
client_metadata: Option<&DevinClientMetadata>,
) -> Result<UnaryResponse, DevinError> {
let request = GetUserJwtRequest {
metadata: Some(metadata_with_client_metadata(token, "", client_metadata)),
};
request_unary(
format!("{base_url}{DEVIN_AUTH_PATH}"),
request.encode_to_vec(),
fetch,
)
.await
}
async fn assign_model(
options: &StreamDevinChatOptions,
token: &str,
base_url: &str,
cascade_id: &str,
fetch: &FetchLike,
) -> Result<ModelAssignment, DevinError> {
let latest_user = build_chat_message_prompts(&options.messages, cascade_id)
.into_iter()
.rev()
.find(|prompt| prompt.source == ChatMessageSource::User as i32)
.map(|mut prompt| {
prompt.message_id.clear();
prompt
});
let request = AssignModelRequest {
metadata: Some(metadata_with_client_metadata(
token,
"",
options.client_metadata.as_ref(),
)),
model_router_uid: options
.model_config
.as_ref()
.and_then(|config| config.request_model_id.clone())
.unwrap_or_else(|| options.model.clone()),
cascade_id: cascade_id.into(),
chat_message_prompt: latest_user,
};
let response = request_unary(
format!("{base_url}{DEVIN_ASSIGN_MODEL_PATH}"),
request.encode_to_vec(),
fetch,
)
.await?;
let decoded: AssignModelResponse = decode_unary_response("AssignModel", &response)?;
decoded
.assignment
.filter(|assignment| {
!assignment.assignment_jwt.is_empty() && !assignment.model_uid.is_empty()
})
.ok_or_else(|| {
DevinError::protocol("Devin AssignModel returned no assignment JWT and model UID")
})
}
pub fn build_chat_request(
options: &StreamDevinChatOptions,
token: &str,
user_jwt: &str,
uuid: &dyn Fn() -> String,
) -> GetChatMessageRequest {
let cascade_id = options
.conversation_id
.as_ref()
.or(options.session_id.as_ref())
.cloned()
.unwrap_or_else(uuid);
let chat_model_uid = options
.model_config
.as_ref()
.and_then(|config| config.request_model_id.clone())
.unwrap_or_else(|| options.model.clone());
let google_schema = chat_model_uid.to_lowercase().contains("gemini");
GetChatMessageRequest {
metadata: Some(metadata_with_client_metadata(
token,
user_jwt,
options.client_metadata.as_ref(),
)),
prompt: options.system_prompt.join("\n\n"),
chat_message_prompts: build_chat_message_prompts(&options.messages, &cascade_id),
chat_model_uid,
model_assignment_jwt: None,
request_type: ChatMessageRequestType::Cascade as i32,
planner_mode: ConversationalPlannerMode::Default as i32,
tool_choice: Some(ChatToolChoice {
choice: Some(chat_tool_choice::Choice::OptionName("auto".into())),
}),
system_prompt_cache_options: Some(PromptCacheOptions {
r#type: CacheControlType::Ephemeral as i32,
}),
disable_parallel_tool_calls: !options
.model_config
.as_ref()
.is_some_and(|config| config.supports_parallel_tool_calls),
cascade_id,
execution_id: uuid(),
configuration: Some(CompletionConfiguration {
num_completions: 1,
max_tokens: options.max_tokens.unwrap_or_else(|| {
options
.model_config
.as_ref()
.and_then(|config| u64::try_from(config.max_tokens).ok())
.unwrap_or(64_000)
}),
max_newlines: 400,
temperature: options.temperature.unwrap_or(1.0),
first_temperature: 0.0,
top_k: 40,
top_p: float32_widen(options.top_p.unwrap_or(0.95)),
stop_patterns: options.stop_sequences.clone(),
fim_eot_prob_threshold: 0.0,
}),
tools: options
.tools
.iter()
.map(|tool| ChatToolDefinition {
name: tool.name.clone(),
description: tool.description.clone(),
json_schema_string: if google_schema {
crate::schema::normalize_google_schema(&tool.input_schema).to_string()
} else {
tool.input_schema.to_string()
},
strict: tool.strict,
})
.collect(),
}
}
pub fn build_chat_message_prompts(
messages: &[DevinMessage],
cascade_id: &str,
) -> Vec<ChatMessagePrompt> {
let mut ids = IndexMap::<String, String>::new();
let mut used = std::collections::HashSet::new();
for message in messages {
if let DevinMessage::Assistant { content, .. } = message {
for part in content {
if let DevinAssistantContentPart::ToolCall { id, .. } = part {
normalize_tool_id(id, &mut ids, &mut used);
}
}
}
}
messages
.iter()
.enumerate()
.map(|(index, message)| match message {
DevinMessage::User { content } | DevinMessage::Developer { content } => {
let normalized = normalize_content(content);
ChatMessagePrompt {
message_id: crate::deterministic_uuid(&format!(
"{cascade_id}\0{index}\0{}",
if matches!(message, DevinMessage::User { .. }) {
"user"
} else {
"developer"
}
)),
source: ChatMessageSource::User as i32,
prompt: normalized.0,
images: normalized.1,
..Default::default()
}
}
DevinMessage::Assistant {
content,
response_id,
} => {
let mut normalized = normalize_assistant_content(content);
for call in &mut normalized.tool_calls {
call.id = normalize_tool_id(&call.id, &mut ids, &mut used);
}
ChatMessagePrompt {
message_id: response_id.clone().unwrap_or_else(|| {
crate::deterministic_uuid(&format!("{cascade_id}\0{index}\0assistant"))
}),
source: ChatMessageSource::System as i32,
prompt: normalized.text,
thinking: normalized.thinking,
signature: normalized.signature,
signature_type: String::new(),
tool_calls: normalized.tool_calls,
..Default::default()
}
}
DevinMessage::Tool {
tool_call_id,
content,
is_error,
} => {
let normalized = normalize_content(content);
ChatMessagePrompt {
message_id: crate::deterministic_uuid(&format!(
"{cascade_id}\0{index}\0tool\0{tool_call_id}"
)),
source: ChatMessageSource::Tool as i32,
tool_call_id: normalize_tool_id(tool_call_id, &mut ids, &mut used),
tool_result_is_error: *is_error,
prompt: normalized.0,
images: normalized.1,
..Default::default()
}
}
})
.filter(|prompt| {
prompt.source != ChatMessageSource::System as i32
|| !prompt.prompt.is_empty()
|| !prompt.thinking.is_empty()
|| !prompt.signature.is_empty()
|| !prompt.tool_calls.is_empty()
})
.collect()
}
#[allow(clippy::cast_possible_truncation)]
fn float32_widen(value: f64) -> f64 {
f64::from(value as f32)
}
fn normalize_tool_id(
id: &str,
ids: &mut IndexMap<String, String>,
used: &mut std::collections::HashSet<String>,
) -> String {
if let Some(known) = ids.get(id) {
return known.clone();
}
let mut normalized = id
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '_' || c == '-' {
c
} else {
'_'
}
})
.take(40)
.collect::<String>();
if normalized.is_empty() {
normalized = "call".into();
}
if used.contains(&normalized) {
normalized = format!(
"{}_{}",
&normalized[..normalized.len().min(7)],
crate::deterministic_uuid(id).replace('-', "")
);
}
while used.contains(&normalized) {
normalized = crate::deterministic_uuid(&format!("{id}\0{normalized}")).replace('-', "");
}
used.insert(normalized.clone());
ids.insert(id.into(), normalized.clone());
normalized
}
fn advance_streaming_tool_call(
previous: Option<&StreamingToolCallState>,
call: &ChatToolCall,
) -> (StreamingToolCallState, String, Option<Value>) {
let previous_json = previous.map_or("", |state| state.arguments_json.as_str());
let arguments_json = if call.arguments_json.starts_with(previous_json) {
call.arguments_json.clone()
} else {
format!("{previous_json}{}", call.arguments_json)
};
let parsed = parse_possibly_complete_json_throttled(
&arguments_json,
previous.map_or(0, |state| state.last_parse_attempt_length),
);
let state = StreamingToolCallState {
name: if call.name.is_empty() {
previous.map_or_else(String::new, |state| state.name.clone())
} else {
call.name.clone()
},
arguments_json: arguments_json.clone(),
last_parse_attempt_length: parsed.as_ref().map_or_else(
|| previous.map_or(0, |state| state.last_parse_attempt_length),
|parsed| parsed.attempted_length,
),
};
let delta = arguments_json
.get(previous_json.len()..)
.unwrap_or_default()
.to_owned();
(state, delta, parsed.and_then(|parsed| parsed.value))
}
fn normalize_content(content: &[DevinContentPart]) -> (String, Vec<ImageData>) {
content.iter().fold(
(String::new(), Vec::new()),
|(mut text, mut images), part| {
match part {
DevinContentPart::Text { text: part } => text.push_str(part),
DevinContentPart::Image { data, mime_type } => images.push(ImageData {
base64_data: data.clone(),
mime_type: mime_type.clone(),
}),
}
(text, images)
},
)
}
struct NormalizedAssistant {
text: String,
thinking: String,
signature: String,
tool_calls: Vec<ChatToolCall>,
}
fn normalize_assistant_content(content: &[DevinAssistantContentPart]) -> NormalizedAssistant {
content.iter().fold(
NormalizedAssistant {
text: String::new(),
thinking: String::new(),
signature: String::new(),
tool_calls: Vec::new(),
},
|mut normalized, part| {
match part {
DevinAssistantContentPart::Text { text } => {
normalized.text.push_str(text);
}
DevinAssistantContentPart::Thinking {
thinking,
thinking_signature,
} => {
normalized.thinking.push_str(thinking);
if normalized.signature.is_empty() {
normalized.signature = thinking_signature.clone().unwrap_or_default();
}
}
DevinAssistantContentPart::ToolCall {
id,
name,
arguments,
} => normalized.tool_calls.push(ChatToolCall {
id: id.clone(),
name: name.clone(),
arguments_json: arguments.to_string(),
}),
}
normalized
},
)
}