use std::{
io::{Read, Write},
net::{TcpListener, TcpStream},
path::{Path, PathBuf},
sync::{Arc, Mutex},
thread,
time::{Duration, SystemTime},
};
use basis::{
Bound, CancellationToken, CollectingSink, Compaction, ContextConfig, Event, MemoryConfig,
RunError, RunFailure, Runtime, TurnOptions, Workspace, WorkspaceBuilder, hooks::HooksConfig,
skills::SkillsConfig, store, templates::TemplatesConfig, tools::declared::ToolsConfig,
};
use mentra::{BuiltinProvider, ModelSelector};
#[tokio::test]
async fn compacting_a_conversation_reports_what_it_replaced_on_the_stream() {
let endpoint = ScriptedEndpoint::start();
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_runtime(endpoint.runtime(store_dir.path()))
.open()
.await
.expect("opens");
let mut run = workspace.prepare("go").expect("mints");
run.execute(CollectingSink::default())
.await
.expect("the scripted turn runs");
let mut sink = CollectingSink::default();
let compacted = run
.compact(Some("hold on to the migration plan"), &mut sink)
.await
.expect("the compacting pass runs")
.expect("a conversation with a turn in it has something to compact");
assert!(
compacted.replaced_items > 0,
"a pass that replaced nothing is not a pass: {compacted:?}"
);
assert_eq!(
compacted.transcript_len,
run.history().len(),
"the reported length has to be the transcript the next turn will send"
);
let agent_id = run.agent_id().to_string();
let events = sink.into_events();
assert!(
matches!(&events[0], Event::CompactionStarted { agent_id: id } if *id == agent_id),
"{events:?}"
);
match &events[1] {
Event::CompactionCompleted {
agent_id: id,
replaced_items,
transcript_len,
..
} => {
assert_eq!(*id, agent_id);
assert_eq!(*replaced_items, compacted.replaced_items);
assert_eq!(*transcript_len, compacted.transcript_len);
}
other => panic!("expected a completed compaction, got {other:?}"),
}
assert_eq!(events.len(), 2, "and nothing else: {events:?}");
}
#[tokio::test]
async fn the_instruction_is_added_to_what_the_summarizer_is_already_told() {
let endpoint = ScriptedEndpoint::start();
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_runtime(endpoint.runtime(store_dir.path()))
.open()
.await
.expect("opens");
let mut run = workspace.prepare("go").expect("mints");
run.execute(CollectingSink::default())
.await
.expect("the scripted turn runs");
run.compact(
Some("hold on to the migration plan"),
&mut CollectingSink::default(),
)
.await
.expect("the compacting pass runs");
let asked = endpoint.requests().join("\n");
assert!(
asked.contains("hold on to the migration plan"),
"the caller's instruction never reached the summarizer: {asked}"
);
assert!(
asked.contains("compaction"),
"and it must arrive inside mentra's own compaction instructions: {asked}"
);
}
#[tokio::test]
async fn a_compaction_that_fails_says_so_on_the_stream() {
let endpoint = ScriptedEndpoint::start_refusing_compaction();
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_runtime(endpoint.runtime(store_dir.path()))
.open()
.await
.expect("opens");
let mut run = workspace.prepare("go").expect("mints");
run.execute(CollectingSink::default())
.await
.expect("the ordinary turn is still answered");
let mut sink = CollectingSink::default();
let failure = run
.compact(None, &mut sink)
.await
.expect_err("the summarizing call is refused");
let events = sink.into_events();
match &events[..] {
[Event::Error { message, .. }] => assert!(
failure.to_string().contains(message.as_str()),
"the stream must carry the failure the caller was handed: \
{message:?} against {failure}"
),
other => panic!("expected one error on the stream, got {other:?}"),
}
}
#[tokio::test]
async fn a_compaction_past_its_deadline_never_reaches_the_summarizer() {
let endpoint = ScriptedEndpoint::start();
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_runtime(endpoint.runtime(store_dir.path()))
.open()
.await
.expect("opens");
let mut run = workspace.prepare("go").expect("mints");
run.execute(CollectingSink::default())
.await
.expect("the scripted turn runs");
let transcript_before = run.history().len();
let asked_before = endpoint.requests().len();
let mut sink = CollectingSink::default();
let failure = run
.compact_with_options(
None,
&mut sink,
TurnOptions::default()
.with_absolute_deadline(SystemTime::now() - Duration::from_secs(1)),
)
.await
.expect_err("a pass past its deadline does not run");
assert!(
matches!(
failure,
RunError::Runtime(mentra::error::RuntimeError::DeadlineExceeded)
),
"the bound must reach the caller as the bound, not as a summarizer \
failure: {failure:?}"
);
assert_eq!(
run.history().len(),
transcript_before,
"an abandoned pass must leave the conversation exactly as it found it"
);
assert_eq!(
endpoint.requests().len(),
asked_before,
"and must not have spent a model call on its way to giving up"
);
match &sink.into_events()[..] {
[
Event::Error {
recoverable,
message,
},
] => {
assert!(!recoverable, "a bound is not something to retry into");
assert_eq!(message, "deadline exceeded");
}
other => panic!("expected one bound on the stream, got {other:?}"),
}
}
#[tokio::test]
async fn a_cancelled_compaction_reports_the_cancel_rather_than_a_summarizer_failure() {
let endpoint = ScriptedEndpoint::start();
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_runtime(endpoint.runtime(store_dir.path()))
.open()
.await
.expect("opens");
let mut run = workspace.prepare("go").expect("mints");
run.execute(CollectingSink::default())
.await
.expect("the scripted turn runs");
let transcript_before = run.history().len();
let asked_before = endpoint.requests().len();
let (options, token) = TurnOptions::cancellable();
token.cancel();
let mut sink = CollectingSink::default();
let failure = run
.compact_with_options(Some("keep the migration plan"), &mut sink, options)
.await
.expect_err("an already-cancelled pass does not run");
assert!(
matches!(
failure,
RunError::Runtime(mentra::error::RuntimeError::Cancelled)
),
"{failure:?}"
);
assert_eq!(run.history().len(), transcript_before);
assert_eq!(endpoint.requests().len(), asked_before);
match &sink.into_events()[..] {
[
Event::Error {
recoverable,
message,
},
] => {
assert!(!recoverable);
assert_eq!(message, "operation cancelled");
}
other => panic!("expected one cancellation on the stream, got {other:?}"),
}
}
#[tokio::test]
async fn an_unbounded_pass_is_what_compact_still_asks_for() {
let endpoint = ScriptedEndpoint::start();
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_runtime(endpoint.runtime(store_dir.path()))
.open()
.await
.expect("opens");
let mut run = workspace.prepare("go").expect("mints");
run.execute(CollectingSink::default())
.await
.expect("the scripted turn runs");
let compacted = run
.compact_with_options(None, &mut CollectingSink::default(), TurnOptions::default())
.await
.expect("an unbounded pass runs")
.expect("there is older history to summarize");
assert!(compacted.replaced_items > 0, "{compacted:?}");
}
#[tokio::test]
async fn the_older_verb_inherits_the_deadline_the_run_was_configured_with() {
let endpoint = ScriptedEndpoint::start();
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_runtime(endpoint.runtime(store_dir.path()))
.open()
.await
.expect("opens");
let mut run = workspace.prepare("go").expect("mints");
run.execute(CollectingSink::default())
.await
.expect("the scripted turn runs");
let mut run = run.with_bounds(
TurnOptions::default().with_absolute_deadline(SystemTime::now() - Duration::from_secs(1)),
);
let transcript_before = run.history().len();
let asked_before = endpoint.requests().len();
let mut sink = CollectingSink::default();
let failure = run
.compact(Some("keep the migration plan"), &mut sink)
.await
.expect_err("a run already past its deadline does not get a summarizing pass");
assert!(
matches!(
failure,
RunError::Runtime(mentra::error::RuntimeError::DeadlineExceeded)
),
"{failure:?}"
);
assert_eq!(run.history().len(), transcript_before);
assert_eq!(
endpoint.requests().len(),
asked_before,
"a pass refused by a bound is a pass the provider never hears about"
);
match &sink.into_events()[..] {
[
Event::Error {
recoverable,
message,
},
] => {
assert!(!recoverable);
assert_eq!(message, "deadline exceeded");
}
other => panic!("expected one deadline on the stream, got {other:?}"),
}
}
#[tokio::test]
async fn a_deadline_reached_inside_an_automatic_pass_is_the_runs_own_bound() {
let endpoint = ScriptedEndpoint::start_stalling_compaction();
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_compaction(eager_compaction())
.with_runtime(endpoint.runtime(store_dir.path()))
.open()
.await
.expect("opens");
let mut run = workspace.prepare("go").expect("mints");
run.execute(CollectingSink::default())
.await
.expect("the first turn has nothing older to summarize and is answered");
let report = tokio::time::timeout(
PROMPTLY,
run.send_with_options(
"again",
CollectingSink::default(),
basis::AllowAll,
TurnOptions::default().with_deadline(DEADLINE_INSIDE_THE_STALL),
),
)
.await
.expect("a bounded pass must not wait for a summarizer that never answers")
.expect("a bound ends the run, it does not break it");
assert_eq!(
report.stopped_by,
Some(Bound::Deadline),
"the run's own bound, named — not a summarizer failure: {report:?}"
);
assert!(matches!(
report.failure.as_ref(),
Some(RunFailure::DeadlineExceeded)
));
assert_eq!(
endpoint.summarizing_requests(),
1,
"the bound has to have landed inside the pass, not before it"
);
assert_eq!(
endpoint.turn_requests(),
1,
"and the second turn never got as far as its own model request"
);
}
#[tokio::test]
async fn a_cancel_during_an_automatic_pass_ends_the_run_as_a_cancellation() {
let (options, token) = TurnOptions::cancellable();
let endpoint = ScriptedEndpoint::start_cancelling_on_compaction(token);
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_compaction(eager_compaction())
.with_runtime(endpoint.runtime(store_dir.path()))
.open()
.await
.expect("opens");
let mut run = workspace.prepare("go").expect("mints");
run.execute(CollectingSink::default())
.await
.expect("the first turn has nothing older to summarize and is answered");
let report = tokio::time::timeout(
PROMPTLY,
run.send_with_options("again", CollectingSink::default(), basis::AllowAll, options),
)
.await
.expect("a cancelled pass must not run to completion")
.expect("cancelling ends the run, it does not break it");
assert!(matches!(
report.failure.as_ref(),
Some(RunFailure::Cancelled)
));
assert_eq!(report.stopped_by, None);
assert!(!report.succeeded());
assert_eq!(endpoint.summarizing_requests(), 1);
assert_eq!(
endpoint.turn_requests(),
1,
"the cancelled pass ends the run rather than being degraded past, so \
the second turn's own model request never goes out"
);
}
#[tokio::test]
async fn a_conversation_with_nothing_to_compact_says_so_and_emits_nothing() {
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_runtime(closed_port(store_dir.path()))
.open()
.await
.expect("opens");
let mut sink = CollectingSink::default();
let compacted = workspace
.prepare("go")
.expect("mints")
.compact(None, &mut sink)
.await
.expect("an empty conversation is not an error");
assert_eq!(compacted, None);
assert!(
sink.into_events().is_empty(),
"nothing happened, so nothing is announced"
);
}
#[tokio::test]
async fn renaming_a_session_is_what_a_later_listing_reports() {
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_runtime(closed_port(store_dir.path()))
.open()
.await
.expect("opens");
let mut run = workspace.prepare("go").expect("mints");
let agent_id = run.agent_id().to_string();
run.set_name("the parser fix").expect("renames");
let listed = store::list_in(store_dir.path(), dir.path()).expect("lists");
let named = listed
.iter()
.find(|session| session.agent_id == agent_id)
.expect("the conversation this workspace minted");
assert_eq!(named.name, "the parser fix");
}
#[tokio::test]
async fn a_forgotten_conversation_is_neither_listed_nor_resumable() {
let dir = tempfile::tempdir().expect("tempdir");
let store_dir = tempfile::tempdir().expect("tempdir");
let workspace = offline(dir.path())
.with_runtime(closed_port(store_dir.path()))
.open()
.await
.expect("opens");
let kept = workspace
.prepare("keep me")
.expect("mints")
.agent_id()
.to_string();
let deleted = {
let run = workspace.prepare("forget me").expect("mints");
run.agent_id().to_string()
};
store::forget_in(store_dir.path(), &deleted).expect("deletes");
assert_eq!(
store::list_in(store_dir.path(), dir.path())
.expect("lists")
.into_iter()
.map(|session| session.agent_id)
.collect::<Vec<_>>(),
vec![kept],
"the one that was forgotten must be gone and the other must not"
);
assert!(
workspace.resume(&deleted, "again").is_err(),
"and there is nothing left to pick back up"
);
}
#[tokio::test]
async fn forgetting_a_conversation_that_was_never_there_is_not_an_error() {
let store_dir = tempfile::tempdir().expect("tempdir");
store::forget_in(store_dir.path(), "agent-nobody-ever-minted")
.expect("deleting nothing deletes nothing");
}
fn eager_compaction() -> Compaction {
Compaction::default()
.with_auto_threshold_tokens(Some(1))
.with_auto_threshold_percent(None)
}
const PROMPTLY: Duration = Duration::from_secs(10);
const DEADLINE_INSIDE_THE_STALL: Duration = Duration::from_secs(2);
fn offline(workspace: &Path) -> WorkspaceBuilder {
Workspace::builder(workspace)
.with_context(ContextConfig {
file_name: "AGENTS.md".to_string(),
global_dir: None,
walk_parents: false,
})
.with_skills(SkillsConfig {
workspace_subdir: Some(PathBuf::from(".basis/skills")),
shared_workspace_dir: true,
global_dir: None,
shared_home_dir: false,
})
.with_templates(TemplatesConfig {
workspace_subdir: PathBuf::from(".basis/templates"),
global_dir: None,
})
.with_hooks(HooksConfig {
workspace_file: PathBuf::from(".basis/hooks.json"),
global_dir: None,
})
.with_tools(ToolsConfig {
workspace_file: PathBuf::from(".basis/tools.json"),
global_dir: None,
})
.with_memory(MemoryConfig::disabled())
}
fn runtime_at(store_dir: &Path, base_url: &str) -> Arc<Runtime> {
Arc::new(
Runtime::builder()
.with_provider(BuiltinProvider::OpenAI)
.with_api_key("test-key")
.with_base_url(base_url)
.with_model(ModelSelector::Id("test-model".to_string()))
.with_store_dir(store_dir)
.build()
.expect("the runtime builds without contacting anything"),
)
}
fn closed_port(store_dir: &Path) -> Arc<Runtime> {
runtime_at(store_dir, "http://127.0.0.1:1/v1")
}
struct ScriptedEndpoint {
base_url: String,
requests: Arc<Mutex<Vec<String>>>,
}
impl ScriptedEndpoint {
fn start() -> Self {
Self::start_with(WhenSummarizing::default())
}
fn start_refusing_compaction() -> Self {
Self::start_with(WhenSummarizing {
refuse: true,
..WhenSummarizing::default()
})
}
fn start_stalling_compaction() -> Self {
Self::start_with(WhenSummarizing {
stall: true,
..WhenSummarizing::default()
})
}
fn start_cancelling_on_compaction(token: CancellationToken) -> Self {
Self::start_with(WhenSummarizing {
stall: true,
cancels: Some(token),
..WhenSummarizing::default()
})
}
fn start_with(when_summarizing: WhenSummarizing) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind test endpoint");
let address = listener.local_addr().expect("read endpoint address");
let requests = Arc::new(Mutex::new(Vec::new()));
let recorded = Arc::clone(&requests);
let behaviour = Arc::new(when_summarizing);
thread::spawn(move || {
while let Ok((stream, _)) = listener.accept() {
let recorded = Arc::clone(&recorded);
let behaviour = Arc::clone(&behaviour);
thread::spawn(move || answer(stream, &recorded, &behaviour));
}
});
Self {
base_url: format!("http://{address}/"),
requests,
}
}
fn summarizing_requests(&self) -> usize {
self.requests()
.iter()
.filter(|request| request.contains(COMPACTION_SYSTEM_PROMPT))
.count()
}
fn turn_requests(&self) -> usize {
self.requests()
.iter()
.filter(|request| {
request.starts_with("POST") && !request.contains(COMPACTION_SYSTEM_PROMPT)
})
.count()
}
fn runtime(&self, store_dir: &Path) -> Arc<Runtime> {
runtime_at(store_dir, &self.base_url)
}
fn requests(&self) -> Vec<String> {
self.requests
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone()
}
}
#[derive(Default)]
struct WhenSummarizing {
refuse: bool,
stall: bool,
cancels: Option<CancellationToken>,
}
const STALL: Duration = Duration::from_secs(30);
const COMPACTION_SYSTEM_PROMPT: &str = "You are a coding-session compaction engine";
fn answer(mut stream: TcpStream, recorded: &Mutex<Vec<String>>, when: &WhenSummarizing) {
let request = read_http_request(&mut stream);
let summarizing = request.contains(COMPACTION_SYSTEM_PROMPT);
let refused = summarizing && when.refuse;
recorded
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.push(request);
if summarizing {
if let Some(token) = &when.cancels {
token.cancel();
}
if when.stall {
thread::sleep(STALL);
return;
}
}
let response = if refused {
let body = r#"{"error":{"message":"summarizing is not available","type":"invalid_request_error"}}"#;
format!(
"HTTP/1.1 400 Bad Request\r\nconnection: close\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{body}",
body.len()
)
} else {
let body = sse_body();
format!(
"HTTP/1.1 200 OK\r\nconnection: close\r\ncontent-type: text/event-stream\r\ncontent-length: {}\r\n\r\n{body}",
body.len()
)
};
let _ = stream.write_all(response.as_bytes());
}
fn sse_body() -> String {
[
r#"{"id":"chatcmpl_1","model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","content":"done"}}]}"#,
r#"{"id":"chatcmpl_1","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#,
"[DONE]",
]
.iter()
.map(|event| format!("data: {event}\n\n"))
.collect()
}
fn read_http_request(stream: &mut TcpStream) -> String {
let mut bytes = Vec::new();
let mut buffer = [0_u8; 4096];
let mut header_end = None;
let mut content_length = 0_usize;
while let Ok(read) = stream.read(&mut buffer) {
if read == 0 {
break;
}
bytes.extend_from_slice(&buffer[..read]);
if header_end.is_none()
&& let Some(index) = bytes.windows(4).position(|window| window == b"\r\n\r\n")
{
let end = index + 4;
header_end = Some(end);
content_length = String::from_utf8_lossy(&bytes[..end])
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().unwrap_or_default())
})
.unwrap_or_default();
}
if header_end.is_some_and(|end| bytes.len() >= end + content_length) {
break;
}
}
String::from_utf8_lossy(&bytes).into_owned()
}