use crate::{
config::SkillSuggestionSettings,
skills::SkillDiscovery,
typesafe::{JevAnswer, JevQuestion, TypeSafeClient},
};
use std::{collections::BTreeMap, io::Read, path::PathBuf};
const SHORTLIST: usize = 3;
const EXCERPT_CHARS: usize = 700;
const SKILL_READ_MAX_BYTES: u64 = 16 * 1024;
const MAX_RANKED_SKILLS: usize = 255;
const RANK_INSTRUCTIONS: &str = "Which of these skills, if any, is the right one to load to help with the user's latest request?";
const RERANK_INSTRUCTIONS: &str = "Exactly one of these skills is the right one to load for the user's latest request. Which one? Read what each actually does, not just its name.";
const GATE_QUESTIONS: [(&str, &str, bool); 3] = [
(
"acts_on_user_system",
"Is the assistant being asked to act on the user's files, accounts, devices, or online services, rather than only to explain or advise?",
false,
),
(
"would_follow_documented_procedure",
"Would a careful expert answering this consult a specific documented procedure or set of commands, rather than answering from general understanding?",
false,
),
(
"prose_suffices",
"Could a knowledgeable generalist fully satisfy this request in prose, with no tools, no documentation, and no access to the user's files or accounts?",
true,
),
];
#[derive(Debug, Clone, PartialEq)]
struct SkillCandidate {
name: String,
description: String,
path: PathBuf,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct SkillSuggestionContext {
settings: SkillSuggestionSettings,
candidates: Vec<SkillCandidate>,
}
#[derive(Debug, Clone, PartialEq)]
pub(super) struct SkillSuggestion {
pub(super) skill: String,
pub(super) gate: f64,
pub(super) fit: f64,
}
impl SkillSuggestion {
pub(super) fn hint(&self) -> String {
skill_relevance_hint(&self.skill)
}
}
pub(super) fn skill_relevance_hint(skill: &str) -> String {
format!(
"<skill_relevance>\nRelevant to the current request: {skill}. Ignore this if it does not fit what the user actually asked for.\n</skill_relevance>"
)
}
impl SkillSuggestionContext {
pub(crate) fn from_settings(
settings: &SkillSuggestionSettings,
skills: &SkillDiscovery,
) -> Option<Self> {
if !settings.enabled {
return None;
}
let candidates = skills
.skills
.values()
.take(MAX_RANKED_SKILLS)
.map(|skill| SkillCandidate {
name: skill.name.clone(),
description: skill
.frontmatter
.get("description")
.map(|description| description.trim().to_string())
.unwrap_or_default(),
path: skill.path.clone(),
})
.collect::<Vec<_>>();
(candidates.len() >= 2).then(|| Self {
settings: settings.clone(),
candidates,
})
}
pub(super) fn prompt_names_skill(&self, prompt: &str) -> bool {
self.candidates.iter().any(|candidate| {
prompt.match_indices('$').any(|(index, _)| {
prompt[index + 1..]
.strip_prefix(candidate.name.as_str())
.is_some_and(|rest| {
!rest.chars().next().is_some_and(|next| {
next.is_alphanumeric() || next == '-' || next == '_'
})
})
})
})
}
pub(super) fn suggest(&self, prompt: &str) -> anyhow::Result<Option<SkillSuggestion>> {
let client = TypeSafeClient::from_environment()?;
let state = serde_json::json!({"request": prompt}).to_string();
let mut questions = BTreeMap::from([(
"which".to_string(),
JevQuestion::Choice {
instructions: RANK_INSTRUCTIONS.into(),
criteria: self
.candidates
.iter()
.map(|candidate| (candidate.name.clone(), candidate.description.clone().into()))
.collect(),
},
)]);
for (key, instructions, _) in GATE_QUESTIONS {
questions.insert(
format!("gate::{key}"),
JevQuestion::Noul {
instructions: instructions.into(),
criteria: None,
},
);
}
let mut answers = client.ask_many(&state, questions)?.value;
let gate = GATE_QUESTIONS
.iter()
.map(|(key, _, inverted)| {
let value = read_noul(answers.remove(&format!("gate::{key}")), key)?;
Ok(if *inverted { 1.0 - value } else { value })
})
.collect::<anyhow::Result<Vec<_>>>()?
.iter()
.sum::<f64>()
/ GATE_QUESTIONS.len() as f64;
if gate < self.settings.gate_threshold {
return Ok(None);
}
let Some(JevAnswer::Choice { probabilities, .. }) = answers.remove("which") else {
anyhow::bail!("Jev skill ranking omitted choice answer");
};
let mut ranked = probabilities.into_iter().collect::<Vec<_>>();
ranked.sort_by(|left, right| right.1.total_cmp(&left.1));
let shortlist = ranked
.into_iter()
.take(SHORTLIST)
.filter_map(|(name, _)| {
self.candidates
.iter()
.find(|candidate| candidate.name == name)
})
.collect::<Vec<_>>();
if shortlist.is_empty() {
return Ok(None);
}
let mut questions = BTreeMap::new();
if shortlist.len() >= 2 {
questions.insert(
"which".to_string(),
JevQuestion::Choice {
instructions: RERANK_INSTRUCTIONS.into(),
criteria: shortlist
.iter()
.map(|candidate| {
(
candidate.name.clone(),
format!("{} — {}", candidate.description, skill_excerpt(candidate))
.into(),
)
})
.collect(),
},
);
}
for candidate in &shortlist {
questions.insert(
format!("fits::{}", candidate.name),
JevQuestion::Noul {
instructions: format!(
"Does the skill '{}' do the specific thing the user's request asks for? It is described as: {}",
candidate.name, candidate.description
).into(),
criteria: None,
},
);
}
let mut answers = client.ask_many(&state, questions)?.value;
let winner = match answers.remove("which") {
Some(JevAnswer::Choice { choice, .. }) => choice,
Some(_) => anyhow::bail!("Jev skill re-check returned wrong answer type"),
None if shortlist.len() == 1 => shortlist[0].name.clone(),
None => anyhow::bail!("Jev skill re-check omitted choice answer"),
};
let fits = shortlist
.iter()
.map(|candidate| {
read_noul(answers.remove(&format!("fits::{}", candidate.name)), "fits")
})
.collect::<anyhow::Result<Vec<_>>>()?;
let best_fit = fits.iter().copied().fold(0.0_f64, f64::max);
if best_fit < self.settings.fit_threshold
|| !shortlist.iter().any(|candidate| candidate.name == winner)
{
return Ok(None);
}
Ok(Some(SkillSuggestion {
skill: winner,
gate,
fit: best_fit,
}))
}
}
fn read_noul(answer: Option<JevAnswer>, name: &str) -> anyhow::Result<f64> {
match answer {
Some(JevAnswer::Noul { noul }) if (0.0..=1.0).contains(&noul) => Ok(noul),
Some(_) => anyhow::bail!("Jev skill suggestion returned invalid {name} answer"),
None => anyhow::bail!("Jev skill suggestion omitted {name} answer"),
}
}
fn skill_excerpt(candidate: &SkillCandidate) -> String {
let mut text = String::new();
let read = std::fs::File::open(&candidate.path)
.and_then(|file| file.take(SKILL_READ_MAX_BYTES).read_to_string(&mut text));
if read.is_err() {
return String::new();
}
let body = text
.strip_prefix("---\n")
.and_then(|rest| rest.split_once("\n---\n").map(|(_, body)| body))
.unwrap_or(&text);
body.trim().chars().take(EXCERPT_CHARS).collect()
}