use super::apply_stop_string_holdback;
#[test]
fn stop_string_spanning_chunk_boundary_does_not_leak() {
let stops = vec!["<|im_start|>".to_string()];
let buffer_len = "<|im_start|>".len() - 1;
let mut acc = String::new();
let mut emitted = 0usize;
let mut triggered = false;
let out = apply_stop_string_holdback(
"hello ",
&stops,
buffer_len,
&mut acc,
&mut emitted,
&mut triggered,
);
assert_eq!(out, "");
assert_eq!(acc, "hello ");
assert!(!triggered);
let out = apply_stop_string_holdback(
"<|im_st",
&stops,
buffer_len,
&mut acc,
&mut emitted,
&mut triggered,
);
assert_eq!(out, "he");
assert!(!out.contains("<|im_st"), "partial stop leaked to client");
assert!(!triggered);
let out = apply_stop_string_holdback(
"art|>",
&stops,
buffer_len,
&mut acc,
&mut emitted,
&mut triggered,
);
assert_eq!(out, "llo ");
assert_eq!(acc, "hello ");
assert!(triggered);
let total = String::new() + "" + "he" + "llo ";
assert_eq!(total, "hello ");
assert!(!total.contains("<|im_st"));
assert!(!total.contains("<|im_start|>"));
}
#[test]
fn no_stop_strings_is_zero_behavior_change() {
let stops: Vec<String> = Vec::new();
let buffer_len = 0usize;
let mut acc = String::new();
let mut emitted = 0usize;
let mut triggered = false;
let out = apply_stop_string_holdback(
"hello ",
&stops,
buffer_len,
&mut acc,
&mut emitted,
&mut triggered,
);
assert_eq!(out, "hello ");
assert!(!triggered);
let out = apply_stop_string_holdback(
"world",
&stops,
buffer_len,
&mut acc,
&mut emitted,
&mut triggered,
);
assert_eq!(out, "world");
assert!(!triggered);
let out = apply_stop_string_holdback(
"<|im_start|>",
&stops,
buffer_len,
&mut acc,
&mut emitted,
&mut triggered,
);
assert_eq!(out, "<|im_start|>");
assert!(!triggered);
}
#[test]
fn utf8_boundary_safety_in_holdback() {
let stops = vec!["STOP".to_string()];
let buffer_len = 3usize;
let mut acc = String::new();
let mut emitted = 0usize;
let mut triggered = false;
let out = apply_stop_string_holdback(
"aébc",
&stops,
buffer_len,
&mut acc,
&mut emitted,
&mut triggered,
);
assert_eq!(out, "a");
assert!(out.is_char_boundary(out.len()));
assert!(!triggered);
}