use std::fs;
use std::io::{Read, Write};
use std::os::fd::AsFd;
use std::path::{Path, PathBuf};
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
use std::thread;
use std::time::{Duration, Instant};
use nix::fcntl::{FcntlArg, OFlag, fcntl};
use nix::poll::{PollFd, PollFlags, PollTimeout, poll};
use serde::Deserialize;
use thiserror::Error;
use tiktoken::CoreBpe;
const REQUEST_MAGIC: &[u8; 4] = b"OPF1";
const RESPONSE_MAGIC: &[u8; 4] = b"OPR1";
const OP_INFER: u8 = 1;
const OP_PING: u8 = 2;
const OP_SHUTDOWN: u8 = 3;
const MAX_SEQUENCES: usize = 128;
const MAX_SEQUENCE_TOKENS: usize = 131_072;
const MAX_SEQUENCE_BYTES: usize = 16 * 1024 * 1024;
const MAX_TOKENS_PER_REQUEST: usize = MAX_SEQUENCES * MAX_SEQUENCE_TOKENS;
const MAX_ERROR_BYTES: usize = 64 * 1024;
const PIPE_WRITE_CHUNK: usize = 4 * 1024;
pub const CLIENT_VERSION: &str = env!("CARGO_PKG_VERSION");
#[derive(Debug, Error)]
pub enum Error {
#[error("failed to start or communicate with the MLX server: {0}")]
Io(#[from] std::io::Error),
#[error("invalid protocol response: {0}")]
Protocol(String),
#[error("MLX server error: {0}")]
Server(String),
#[error("tokenizer error: {0}")]
Tokenizer(String),
#[error("failed to read Viterbi calibration")]
CalibrationIo(#[source] std::io::Error),
#[error("invalid Viterbi calibration")]
CalibrationJson(#[source] serde_json::Error),
#[error("invalid MLX server configuration: {0}")]
Configuration(&'static str),
#[error(
"lossless chunking is unavailable for a {required_tokens}-token input with a {chunk_tokens}-token trace; use detect_chunked_batch_approximate only when boundary-dependent output is acceptable"
)]
LosslessChunkingUnavailable {
required_tokens: usize,
chunk_tokens: usize,
},
#[error("MLX server {operation} timed out after {timeout:?}")]
Timeout {
operation: &'static str,
timeout: Duration,
},
#[error("MLX server is no longer available after an earlier failure")]
Unavailable,
}
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub enum Label {
AccountNumber,
PrivateAddress,
PrivateDate,
PrivateEmail,
PrivatePerson,
PrivatePhone,
PrivateUrl,
Secret,
}
impl Label {
pub fn as_str(self) -> &'static str {
match self {
Self::AccountNumber => "account_number",
Self::PrivateAddress => "private_address",
Self::PrivateDate => "private_date",
Self::PrivateEmail => "private_email",
Self::PrivatePerson => "private_person",
Self::PrivatePhone => "private_phone",
Self::PrivateUrl => "private_url",
Self::Secret => "secret",
}
}
pub fn placeholder(self) -> &'static str {
match self {
Self::AccountNumber => "<ACCOUNT_NUMBER>",
Self::PrivateAddress => "<PRIVATE_ADDRESS>",
Self::PrivateDate => "<PRIVATE_DATE>",
Self::PrivateEmail => "<PRIVATE_EMAIL>",
Self::PrivatePerson => "<PRIVATE_PERSON>",
Self::PrivatePhone => "<PRIVATE_PHONE>",
Self::PrivateUrl => "<PRIVATE_URL>",
Self::Secret => "<SECRET>",
}
}
fn from_index(index: u8) -> Option<Self> {
Some(match index {
0 => Self::AccountNumber,
1 => Self::PrivateAddress,
2 => Self::PrivateDate,
3 => Self::PrivateEmail,
4 => Self::PrivatePerson,
5 => Self::PrivatePhone,
6 => Self::PrivateUrl,
7 => Self::Secret,
_ => return None,
})
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Span {
pub label: Label,
pub start: usize,
pub end: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Detection {
pub spans: Vec<Span>,
}
impl Detection {
pub fn redact(&self, text: &str) -> String {
let mut spans = self
.spans
.iter()
.filter(|span| {
span.end <= text.len()
&& span.end > span.start
&& text.is_char_boundary(span.start)
&& text.is_char_boundary(span.end)
})
.cloned()
.collect::<Vec<_>>();
spans.sort_by_key(|span| (span.start, span.end, span.label));
let mut merged: Vec<Span> = Vec::with_capacity(spans.len());
for span in spans {
if let Some(previous) = merged.last_mut()
&& span.start < previous.end
{
previous.end = previous.end.max(span.end);
if span.label == Label::Secret {
previous.label = Label::Secret;
}
continue;
}
merged.push(span);
}
let mut output = String::with_capacity(text.len());
let mut cursor = 0;
for span in merged {
output.push_str(&text[cursor..span.start]);
output.push_str(span.label.placeholder());
cursor = span.end;
}
output.push_str(&text[cursor..]);
output
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct FixedTraceShape {
pub batch: usize,
pub tokens: usize,
}
#[derive(Clone, Debug)]
pub struct ServerOptions {
pub memory_limit_gb: Option<f64>,
pub cache_limit_gb: Option<f64>,
pub max_batch_tokens: usize,
pub viterbi_biases: Option<[f32; 6]>,
pub viterbi_calibration_path: Option<PathBuf>,
pub startup_timeout: Duration,
pub io_timeout: Duration,
pub shutdown_timeout: Duration,
pub prewarm_fixed_trace: bool,
pub log_memory_stats: bool,
}
impl Default for ServerOptions {
fn default() -> Self {
Self {
memory_limit_gb: None,
cache_limit_gb: Some(0.5),
max_batch_tokens: 32_768,
viterbi_biases: None,
viterbi_calibration_path: None,
startup_timeout: Duration::from_secs(60),
io_timeout: Duration::from_secs(5 * 60),
shutdown_timeout: Duration::from_secs(2),
prewarm_fixed_trace: true,
log_memory_stats: false,
}
}
}
pub struct PrivacyFilter {
child: Child,
input: ChildStdin,
output: ChildStdout,
tokenizer: &'static CoreBpe,
fixed_trace_tokens: Option<usize>,
io_timeout: Duration,
shutdown_timeout: Duration,
available: bool,
}
impl PrivacyFilter {
pub fn spawn(
server_executable: impl AsRef<Path>,
model: impl AsRef<Path>,
options: ServerOptions,
) -> Result<Self, Error> {
Self::spawn_inner(server_executable.as_ref(), model.as_ref(), options, None)
}
pub fn spawn_fixed(
server_executable: impl AsRef<Path>,
model: impl AsRef<Path>,
options: ServerOptions,
fixed_trace: FixedTraceShape,
) -> Result<Self, Error> {
Self::spawn_inner(
server_executable.as_ref(),
model.as_ref(),
options,
Some(fixed_trace),
)
}
fn spawn_inner(
server_executable: &Path,
model: &Path,
options: ServerOptions,
fixed_trace: Option<FixedTraceShape>,
) -> Result<Self, Error> {
validate_memory_limit(options.memory_limit_gb)?;
validate_cache_limit(options.cache_limit_gb)?;
validate_fixed_trace(fixed_trace, options.max_batch_tokens)?;
let viterbi_biases = resolve_viterbi_biases(model, &options)?;
let mut command = Command::new(server_executable);
command.arg("--model").arg(model);
if let Some(limit) = options.memory_limit_gb {
command.arg("--memory-limit-gb").arg(limit.to_string());
}
command.arg("--cache-limit-gb").arg(
options
.cache_limit_gb
.map_or_else(|| "0".to_owned(), |limit| limit.to_string()),
);
command
.arg("--max-batch-tokens")
.arg(options.max_batch_tokens.to_string());
if let Some(shape) = fixed_trace {
command
.arg("--fixed-trace-batch")
.arg(shape.batch.to_string())
.arg("--fixed-trace-tokens")
.arg(shape.tokens.to_string());
if options.prewarm_fixed_trace {
command.arg("--prewarm-fixed-trace");
}
}
if options.log_memory_stats {
command.arg("--log-memory-stats");
}
let bias_names = [
"--bias-background-stay",
"--bias-background-to-start",
"--bias-inside-to-continue",
"--bias-inside-to-end",
"--bias-end-to-background",
"--bias-end-to-start",
];
for (name, value) in bias_names.into_iter().zip(viterbi_biases) {
if value != 0.0 {
command.arg(name).arg(value.to_string());
}
}
let mut child = command
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.spawn()?;
let input = child
.stdin
.take()
.ok_or_else(|| Error::Protocol("missing child stdin".into()))?;
let output = child
.stdout
.take()
.ok_or_else(|| Error::Protocol("missing child stdout".into()))?;
let tokenizer = tiktoken::get_encoding("o200k_base")
.ok_or_else(|| Error::Tokenizer("o200k_base is unavailable".into()))?;
let mut filter = Self {
child,
input,
output,
tokenizer,
fixed_trace_tokens: fixed_trace.map(|shape| shape.tokens),
io_timeout: options.io_timeout,
shutdown_timeout: options.shutdown_timeout,
available: true,
};
if let Err(error) =
set_nonblocking(&filter.input).and_then(|()| set_nonblocking(&filter.output))
{
filter.terminate();
return Err(Error::Io(error));
}
if let Err(error) = filter.ping(options.startup_timeout) {
filter.terminate();
return Err(error);
}
Ok(filter)
}
pub fn detect_batch(&mut self, texts: &[&str]) -> Result<Vec<Detection>, Error> {
if !self.available {
return Err(Error::Unavailable);
}
let mut output = Vec::with_capacity(texts.len());
for chunk in texts.chunks(MAX_SEQUENCES) {
let prepared = prepare_request(self.tokenizer, chunk)?;
match self.exchange_batch(chunk, &prepared.encoded, prepared.total, &prepared.bytes) {
Ok(detections) => output.extend(detections),
Err(error) => {
self.terminate();
return Err(error);
}
}
}
Ok(output)
}
pub fn detect_chunked_batch(
&mut self,
texts: &[&str],
chunk_tokens: usize,
overlap_tokens: usize,
) -> Result<Vec<Detection>, Error> {
if !self.available {
return Err(Error::Unavailable);
}
validate_chunk_configuration(chunk_tokens, overlap_tokens)?;
let trace_tokens = lossless_trace_tokens(chunk_tokens, self.fixed_trace_tokens);
validate_lossless_chunk_request(self.tokenizer, texts, trace_tokens)?;
self.detect_batch(texts)
}
pub fn detect_chunked_batch_approximate(
&mut self,
texts: &[&str],
chunk_tokens: usize,
overlap_tokens: usize,
) -> Result<Vec<Detection>, Error> {
if !self.available {
return Err(Error::Unavailable);
}
validate_chunk_configuration(chunk_tokens, overlap_tokens)?;
validate_approximate_trace_tokens(chunk_tokens, self.fixed_trace_tokens)?;
let mut combined = vec![Vec::new(); texts.len()];
let mut pending = Vec::with_capacity(MAX_SEQUENCES);
let mut pending_tokens = 0usize;
for (source_index, text) in texts.iter().enumerate() {
let encoded = Encoded::new(self.tokenizer, text)?;
for chunk in encoded.overlapping_chunks(chunk_tokens, overlap_tokens)? {
let tokens = chunk.tokens.len();
if !pending.is_empty()
&& (pending.len() == MAX_SEQUENCES
|| pending_tokens.saturating_add(tokens) > MAX_TOKENS_PER_REQUEST)
{
self.exchange_chunk_batch(&mut pending, &mut combined)?;
pending_tokens = 0;
}
pending_tokens = pending_tokens
.checked_add(tokens)
.ok_or_else(|| Error::Protocol("chunked batch token count overflow".into()))?;
pending.push(EncodedChunk {
source_index,
text,
encoded: chunk,
});
}
}
self.exchange_chunk_batch(&mut pending, &mut combined)?;
Ok(combined
.into_iter()
.map(|mut spans| {
spans.sort_by_key(|span| (span.start, span.end, span.label));
spans.dedup();
Detection { spans }
})
.collect())
}
fn exchange_chunk_batch(
&mut self,
pending: &mut Vec<EncodedChunk<'_>>,
combined: &mut [Vec<Span>],
) -> Result<(), Error> {
if pending.is_empty() {
return Ok(());
}
let mut source_indices = Vec::with_capacity(pending.len());
let mut texts = Vec::with_capacity(pending.len());
let mut encoded = Vec::with_capacity(pending.len());
for chunk in pending.drain(..) {
source_indices.push(chunk.source_index);
texts.push(chunk.text);
encoded.push(chunk.encoded);
}
let prepared = prepare_encoded_request(encoded)?;
let detections =
match self.exchange_batch(&texts, &prepared.encoded, prepared.total, &prepared.bytes) {
Ok(detections) => detections,
Err(error) => {
self.terminate();
return Err(error);
}
};
for (source_index, detection) in source_indices.into_iter().zip(detections) {
combined[source_index].extend(detection.spans);
}
Ok(())
}
fn exchange_batch(
&mut self,
texts: &[&str],
encoded: &[Encoded],
total: usize,
request: &[u8],
) -> Result<Vec<Detection>, Error> {
let deadline = deadline_after(self.io_timeout);
write_all_before(
&mut self.input,
request,
deadline,
self.io_timeout,
"request write",
)?;
let (status, count, declared_labels) =
read_response_header_before(&mut self.output, deadline, self.io_timeout)?;
if status != 0 {
if declared_labels as usize > MAX_ERROR_BYTES {
return Err(Error::Protocol("server error message is too large".into()));
}
let mut message = vec![0; declared_labels as usize];
read_exact_before(
&mut self.output,
&mut message,
deadline,
self.io_timeout,
"error response read",
)?;
return Err(Error::Server(
String::from_utf8_lossy(&message).into_owned(),
));
}
if count as usize != encoded.len() || declared_labels as usize != total {
return Err(Error::Protocol(
"response dimensions do not match request".into(),
));
}
let mut lengths = Vec::with_capacity(count as usize);
for expected in 0..count {
let id = read_u64_before(&mut self.output, deadline, self.io_timeout)?;
let length = read_u32_before(&mut self.output, deadline, self.io_timeout)?;
let _reserved = read_u32_before(&mut self.output, deadline, self.io_timeout)?;
if id != expected as u64 || length as usize != encoded[expected as usize].tokens.len() {
return Err(Error::Protocol(
"response sequence metadata mismatch".into(),
));
}
lengths.push(length as usize);
}
let mut detections = Vec::with_capacity(encoded.len());
for ((text, item), length) in texts.iter().zip(encoded).zip(lengths) {
let mut labels = vec![0u8; length];
read_exact_before(
&mut self.output,
&mut labels,
deadline,
self.io_timeout,
"inference response read",
)?;
detections.push(Detection {
spans: decode_spans(text, item, &labels)?,
});
}
Ok(detections)
}
pub fn detect(&mut self, text: &str) -> Result<Detection, Error> {
self.detect_batch(&[text])?
.pop()
.ok_or_else(|| Error::Protocol("empty response".into()))
}
fn ping(&mut self, timeout: Duration) -> Result<(), Error> {
let request = control_request(OP_PING);
let deadline = deadline_after(timeout);
write_all_before(
&mut self.input,
&request,
deadline,
timeout,
"startup ping write",
)?;
let (status, count, labels) =
read_response_header_before(&mut self.output, deadline, timeout)?;
if status != 0 || count != 0 || labels != 0 {
return Err(Error::Protocol("invalid startup ping response".into()));
}
Ok(())
}
fn terminate(&mut self) {
self.available = false;
let _ = self.child.kill();
let _ = self.child.wait();
}
}
impl Drop for PrivacyFilter {
fn drop(&mut self) {
if !self.available {
return;
}
let request = control_request(OP_SHUTDOWN);
let deadline = deadline_after(self.shutdown_timeout);
let _ = write_all_before(
&mut self.input,
&request,
deadline,
self.shutdown_timeout,
"shutdown write",
);
loop {
match self.child.try_wait() {
Ok(Some(_)) => return,
Ok(None) if Instant::now() < deadline => {
thread::sleep(Duration::from_millis(10));
}
Ok(None) | Err(_) => break,
}
}
self.terminate();
}
}
struct Encoded {
tokens: Vec<u32>,
token_starts: Vec<usize>,
token_ends: Vec<usize>,
}
struct PreparedRequest {
encoded: Vec<Encoded>,
total: usize,
bytes: Vec<u8>,
}
struct EncodedChunk<'a> {
source_index: usize,
text: &'a str,
encoded: Encoded,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct CalibrationArtifact {
operating_points: CalibrationOperatingPoints,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct CalibrationOperatingPoints {
default: CalibrationPoint,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct CalibrationPoint {
biases: CalibrationBiases,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct CalibrationBiases {
transition_bias_background_stay: f32,
transition_bias_background_to_start: f32,
transition_bias_inside_to_continue: f32,
transition_bias_inside_to_end: f32,
transition_bias_end_to_background: f32,
transition_bias_end_to_start: f32,
}
impl Encoded {
fn new(tokenizer: &CoreBpe, text: &str) -> Result<Self, Error> {
if text.len() > MAX_SEQUENCE_BYTES {
return Err(Error::Protocol(format!(
"sequence exceeds {MAX_SEQUENCE_BYTES} bytes"
)));
}
if text.len() > MAX_SEQUENCE_TOKENS
&& tokenizer.count_with_special_tokens(text) > MAX_SEQUENCE_TOKENS
{
return Err(Error::Protocol(format!(
"sequence exceeds {MAX_SEQUENCE_TOKENS} tokens"
)));
}
let tokens = tokenizer.encode_with_special_tokens(text);
if tokens.len() > MAX_SEQUENCE_TOKENS {
return Err(Error::Protocol(format!(
"sequence exceeds {MAX_SEQUENCE_TOKENS} tokens"
)));
}
let mut token_starts = Vec::with_capacity(tokens.len());
let mut token_ends = Vec::with_capacity(tokens.len());
let mut decoded = Vec::with_capacity(text.len());
for &token in &tokens {
token_starts.push(decoded.len());
decoded.extend(tokenizer.decode(&[token]));
token_ends.push(decoded.len());
}
if decoded != text.as_bytes() {
return Err(Error::Tokenizer(
"token bytes do not reconstruct the source text".into(),
));
}
Ok(Self {
tokens,
token_starts,
token_ends,
})
}
fn overlapping_chunks(
&self,
chunk_tokens: usize,
overlap_tokens: usize,
) -> Result<Vec<Self>, Error> {
validate_chunk_configuration(chunk_tokens, overlap_tokens)?;
if self.tokens.is_empty() {
return Ok(vec![Self {
tokens: Vec::new(),
token_starts: Vec::new(),
token_ends: Vec::new(),
}]);
}
let stride = chunk_tokens - overlap_tokens;
let mut chunks = Vec::with_capacity(self.tokens.len().div_ceil(stride));
let mut start = 0;
loop {
let end = (start + chunk_tokens).min(self.tokens.len());
chunks.push(Self {
tokens: self.tokens[start..end].to_vec(),
token_starts: self.token_starts[start..end].to_vec(),
token_ends: self.token_ends[start..end].to_vec(),
});
if end == self.tokens.len() {
break;
}
start += stride;
}
Ok(chunks)
}
}
fn prepare_request(tokenizer: &CoreBpe, texts: &[&str]) -> Result<PreparedRequest, Error> {
let mut encoded = Vec::with_capacity(texts.len());
for text in texts {
encoded.push(Encoded::new(tokenizer, text)?);
}
prepare_encoded_request(encoded)
}
fn prepare_encoded_request(encoded: Vec<Encoded>) -> Result<PreparedRequest, Error> {
let total = encoded.iter().try_fold(0usize, |total, item| {
total
.checked_add(item.tokens.len())
.filter(|&count| count <= MAX_TOKENS_PER_REQUEST)
.ok_or_else(|| Error::Protocol("batch is too large".into()))
})?;
let mut bytes = Vec::with_capacity(16 + encoded.len() * 16 + total * 4);
bytes.extend_from_slice(REQUEST_MAGIC);
bytes.extend_from_slice(&[OP_INFER, 0, 0, 0]);
bytes.extend_from_slice(&(encoded.len() as u32).to_le_bytes());
bytes.extend_from_slice(&(total as u32).to_le_bytes());
for (index, item) in encoded.iter().enumerate() {
bytes.extend_from_slice(&(index as u64).to_le_bytes());
bytes.extend_from_slice(&(item.tokens.len() as u32).to_le_bytes());
bytes.extend_from_slice(&0u32.to_le_bytes());
}
for item in &encoded {
for &token in &item.tokens {
bytes.extend_from_slice(&token.to_le_bytes());
}
}
Ok(PreparedRequest {
encoded,
total,
bytes,
})
}
fn validate_chunk_configuration(chunk_tokens: usize, overlap_tokens: usize) -> Result<(), Error> {
if chunk_tokens == 0 || chunk_tokens > MAX_SEQUENCE_TOKENS || overlap_tokens > chunk_tokens / 2
{
return Err(Error::Configuration(
"chunk_tokens must be within the model context and at least twice overlap_tokens",
));
}
Ok(())
}
fn lossless_trace_tokens(
requested_chunk_tokens: usize,
fixed_trace_tokens: Option<usize>,
) -> usize {
fixed_trace_tokens.map_or(requested_chunk_tokens, |fixed_trace_tokens| {
requested_chunk_tokens.min(fixed_trace_tokens)
})
}
fn validate_approximate_trace_tokens(
chunk_tokens: usize,
fixed_trace_tokens: Option<usize>,
) -> Result<(), Error> {
if let Some(fixed_trace_tokens) = fixed_trace_tokens
&& chunk_tokens > fixed_trace_tokens
{
return Err(Error::Configuration(
"chunk_tokens must not exceed the configured fixed trace token length",
));
}
Ok(())
}
fn validate_lossless_chunk_request(
tokenizer: &CoreBpe,
texts: &[&str],
chunk_tokens: usize,
) -> Result<(), Error> {
for text in texts {
if text.len() > MAX_SEQUENCE_BYTES {
return Err(Error::Protocol(format!(
"sequence exceeds {MAX_SEQUENCE_BYTES} bytes"
)));
}
let tokens = tokenizer.count_with_special_tokens(text);
if tokens > MAX_SEQUENCE_TOKENS {
return Err(Error::Protocol(format!(
"sequence exceeds {MAX_SEQUENCE_TOKENS} tokens"
)));
}
if tokens > chunk_tokens {
return Err(Error::LosslessChunkingUnavailable {
required_tokens: tokens,
chunk_tokens,
});
}
}
Ok(())
}
fn resolve_viterbi_biases(model: &Path, options: &ServerOptions) -> Result<[f32; 6], Error> {
if options.viterbi_biases.is_some() && options.viterbi_calibration_path.is_some() {
return Err(Error::Configuration(
"viterbi_biases and viterbi_calibration_path are mutually exclusive",
));
}
let biases = match options.viterbi_biases {
Some(biases) => biases,
None => {
let default_path = model
.parent()
.unwrap_or_else(|| Path::new("."))
.join("viterbi_calibration.json");
let path = options
.viterbi_calibration_path
.as_deref()
.unwrap_or(&default_path);
load_viterbi_biases(path)?
}
};
if biases.iter().any(|value| !value.is_finite()) {
return Err(Error::Configuration(
"Viterbi calibration biases must be finite",
));
}
Ok(biases)
}
fn load_viterbi_biases(path: &Path) -> Result<[f32; 6], Error> {
let bytes = fs::read(path).map_err(Error::CalibrationIo)?;
let artifact: CalibrationArtifact =
serde_json::from_slice(&bytes).map_err(Error::CalibrationJson)?;
let biases = artifact.operating_points.default.biases;
Ok([
biases.transition_bias_background_stay,
biases.transition_bias_background_to_start,
biases.transition_bias_inside_to_continue,
biases.transition_bias_inside_to_end,
biases.transition_bias_end_to_background,
biases.transition_bias_end_to_start,
])
}
fn decode_spans(text: &str, encoded: &Encoded, labels: &[u8]) -> Result<Vec<Span>, Error> {
if labels.len() != encoded.tokens.len() {
return Err(Error::Protocol(
"label count differs from token count".into(),
));
}
let mut token_spans = Vec::<(Label, usize, usize)>::new();
let mut open: Option<(Label, usize)> = None;
for (index, &class) in labels.iter().enumerate() {
if class == 0 {
if let Some((label, start)) = open.take() {
token_spans.push((label, start, index));
}
continue;
}
if class > 32 {
return Err(Error::Protocol(format!("invalid label class {class}")));
}
let label = Label::from_index((class - 1) / 4).unwrap();
match (class - 1) % 4 {
0 => {
if let Some((prior, start)) = open.replace((label, index)) {
token_spans.push((prior, start, index));
}
}
1 => {
if open.map(|x| x.0) != Some(label)
&& let Some((prior, start)) = open.replace((label, index))
{
token_spans.push((prior, start, index));
}
}
2 => {
let start = match open.take() {
Some((open_label, start)) if open_label == label => start,
Some((prior, start)) => {
token_spans.push((prior, start, index));
index
}
None => index,
};
token_spans.push((label, start, index + 1));
}
3 => {
if let Some((prior, start)) = open.take() {
token_spans.push((prior, start, index));
}
token_spans.push((label, index, index + 1));
}
_ => unreachable!(),
}
}
if let Some((label, start)) = open {
token_spans.push((label, start, labels.len()));
}
let mut spans = Vec::with_capacity(token_spans.len());
for (label, token_start, token_end) in token_spans {
if token_start >= token_end {
continue;
}
let mut start = encoded.token_starts[token_start];
let mut end = encoded.token_ends[token_end - 1];
while start > 0 && !text.is_char_boundary(start) {
start -= 1;
}
while end < text.len() && !text.is_char_boundary(end) {
end += 1;
}
while start < end {
let ch = text[start..end].chars().next().unwrap();
if !ch.is_whitespace() {
break;
}
start += ch.len_utf8();
}
while end > start {
let ch = text[start..end].chars().next_back().unwrap();
if !ch.is_whitespace() {
break;
}
end -= ch.len_utf8();
}
if start < end {
spans.push(Span { label, start, end });
}
}
spans.sort_by_key(|span| {
(
span.start,
std::cmp::Reverse(span.end - span.start),
span.label,
)
});
spans.dedup();
Ok(spans)
}
fn control_request(opcode: u8) -> [u8; 16] {
let mut request = [0u8; 16];
request[..4].copy_from_slice(REQUEST_MAGIC);
request[4] = opcode;
request
}
fn deadline_after(timeout: Duration) -> Instant {
Instant::now()
.checked_add(timeout)
.unwrap_or_else(Instant::now)
}
fn validate_memory_limit(memory_limit_gb: Option<f64>) -> Result<(), Error> {
let Some(memory_limit_gb) = memory_limit_gb else {
return Ok(());
};
const BYTES_PER_GIB: f64 = 1_073_741_824.0;
let bytes = memory_limit_gb * BYTES_PER_GIB;
if !memory_limit_gb.is_finite()
|| memory_limit_gb <= 0.0
|| !bytes.is_finite()
|| bytes < 1.0
|| bytes >= usize::MAX as f64
{
return Err(Error::Configuration(
"memory_limit_gb must be finite, positive, at least one byte, and fit in size_t",
));
}
Ok(())
}
fn validate_cache_limit(cache_limit_gb: Option<f64>) -> Result<(), Error> {
let Some(cache_limit_gb) = cache_limit_gb else {
return Ok(());
};
const BYTES_PER_GIB: f64 = 1_073_741_824.0;
let bytes = cache_limit_gb * BYTES_PER_GIB;
if !cache_limit_gb.is_finite()
|| cache_limit_gb <= 0.0
|| !bytes.is_finite()
|| bytes < 1.0
|| bytes >= usize::MAX as f64
{
return Err(Error::Configuration(
"cache_limit_gb must be finite, positive, at least one byte, and fit in size_t",
));
}
Ok(())
}
fn validate_fixed_trace(
fixed_trace: Option<FixedTraceShape>,
max_batch_tokens: usize,
) -> Result<(), Error> {
if max_batch_tokens == 0 {
return Err(Error::Configuration(
"max_batch_tokens must be greater than zero",
));
}
let Some(shape) = fixed_trace else {
return Ok(());
};
let Some(padded_tokens) = shape.batch.checked_mul(shape.tokens) else {
return Err(Error::Configuration(
"fixed trace padded-token count overflows usize",
));
};
if shape.batch == 0 || shape.tokens == 0 || padded_tokens > max_batch_tokens {
return Err(Error::Configuration(
"fixed trace dimensions must be positive and fit max_batch_tokens",
));
}
Ok(())
}
fn wait_for_io(
fd: &impl AsFd,
events: PollFlags,
deadline: Instant,
configured_timeout: Duration,
operation: &'static str,
) -> Result<(), Error> {
loop {
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return Err(Error::Timeout {
operation,
timeout: configured_timeout,
});
}
let timeout = PollTimeout::try_from(remaining).unwrap_or(PollTimeout::MAX);
let mut descriptors = [PollFd::new(fd.as_fd(), events)];
match poll(&mut descriptors, timeout) {
Ok(0) => {
return Err(Error::Timeout {
operation,
timeout: configured_timeout,
});
}
Ok(_) => return Ok(()),
Err(nix::errno::Errno::EINTR) => continue,
Err(errno) => {
return Err(Error::Io(std::io::Error::from_raw_os_error(errno as i32)));
}
}
}
}
fn set_nonblocking(fd: &impl AsFd) -> std::io::Result<()> {
let flags = fcntl(fd, FcntlArg::F_GETFL)
.map(OFlag::from_bits_truncate)
.map_err(|errno| std::io::Error::from_raw_os_error(errno as i32))?;
fcntl(fd, FcntlArg::F_SETFL(flags | OFlag::O_NONBLOCK))
.map(|_| ())
.map_err(|errno| std::io::Error::from_raw_os_error(errno as i32))
}
fn write_all_before(
writer: &mut (impl Write + AsFd),
mut bytes: &[u8],
deadline: Instant,
configured_timeout: Duration,
operation: &'static str,
) -> Result<(), Error> {
while !bytes.is_empty() {
wait_for_io(
writer,
PollFlags::POLLOUT,
deadline,
configured_timeout,
operation,
)?;
let chunk_len = bytes.len().min(PIPE_WRITE_CHUNK);
match writer.write(&bytes[..chunk_len]) {
Ok(0) => return Err(Error::Io(std::io::ErrorKind::WriteZero.into())),
Ok(written) => bytes = &bytes[written..],
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => continue,
Err(error) => return Err(Error::Io(error)),
}
}
Ok(())
}
fn read_exact_before(
reader: &mut ChildStdout,
mut bytes: &mut [u8],
deadline: Instant,
configured_timeout: Duration,
operation: &'static str,
) -> Result<(), Error> {
while !bytes.is_empty() {
wait_for_io(
reader,
PollFlags::POLLIN,
deadline,
configured_timeout,
operation,
)?;
match reader.read(bytes) {
Ok(0) => return Err(Error::Io(std::io::ErrorKind::UnexpectedEof.into())),
Ok(read) => bytes = &mut bytes[read..],
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => continue,
Err(error) => return Err(Error::Io(error)),
}
}
Ok(())
}
fn read_u32_before(
reader: &mut ChildStdout,
deadline: Instant,
timeout: Duration,
) -> Result<u32, Error> {
let mut bytes = [0u8; 4];
read_exact_before(reader, &mut bytes, deadline, timeout, "response read")?;
Ok(u32::from_le_bytes(bytes))
}
fn read_u64_before(
reader: &mut ChildStdout,
deadline: Instant,
timeout: Duration,
) -> Result<u64, Error> {
let mut bytes = [0u8; 8];
read_exact_before(reader, &mut bytes, deadline, timeout, "response read")?;
Ok(u64::from_le_bytes(bytes))
}
fn read_response_header_before(
reader: &mut ChildStdout,
deadline: Instant,
timeout: Duration,
) -> Result<(u8, u32, u32), Error> {
let mut header = [0u8; 16];
read_exact_before(
reader,
&mut header,
deadline,
timeout,
"response header read",
)?;
if &header[..4] != RESPONSE_MAGIC {
return Err(Error::Protocol("bad response magic".into()));
}
Ok(parse_response_header(header))
}
fn parse_response_header(header: [u8; 16]) -> (u8, u32, u32) {
let count = u32::from_le_bytes([header[8], header[9], header[10], header[11]]);
let labels = u32::from_le_bytes([header[12], header[13], header[14], header[15]]);
(header[4], count, labels)
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use std::fs::File;
use std::os::unix::fs::PermissionsExt;
#[test]
fn tokenizer_round_trips_and_tracks_split_unicode() {
let tokenizer = tiktoken::get_encoding("o200k_base").unwrap();
let text = "Jordan お誕生日 jordan@example.com";
let encoded = Encoded::new(tokenizer, text).unwrap();
assert_eq!(encoded.token_starts.len(), encoded.tokens.len());
assert_eq!(encoded.token_ends.last().copied(), Some(text.len()));
}
#[test]
fn tokenizer_preserves_official_o200k_special_tokens() {
let tokenizer = tiktoken::get_encoding("o200k_base").unwrap();
let text = "before<|endoftext|>alice@example.com";
let encoded = Encoded::new(tokenizer, text).unwrap();
let special_index = encoded
.tokens
.iter()
.position(|&token| token == 199_999)
.expect("o200k endoftext token should be preserved");
assert_eq!(encoded.tokens, tokenizer.encode_with_special_tokens(text));
assert_ne!(encoded.tokens, tokenizer.encode(text));
assert_eq!(
&text[encoded.token_starts[special_index]..encoded.token_ends[special_index]],
"<|endoftext|>"
);
}
#[test]
fn token_chunks_preserve_global_offsets_and_exact_overlap() {
let tokenizer = tiktoken::get_encoding("o200k_base").unwrap();
let text = format!("{}終わり", "word ".repeat(200));
let encoded = Encoded::new(tokenizer, &text).unwrap();
let chunks = encoded.overlapping_chunks(32, 8).unwrap();
assert!(chunks.len() > 1);
assert_eq!(chunks[0].token_starts[0], 0);
assert_eq!(
chunks.last().and_then(|chunk| chunk.token_ends.last()),
Some(&text.len())
);
for chunk in &chunks {
assert!(chunk.tokens.len() <= 32);
assert_eq!(chunk.tokens.len(), chunk.token_starts.len());
assert_eq!(chunk.tokens.len(), chunk.token_ends.len());
}
for pair in chunks.windows(2) {
assert_eq!(
&pair[0].tokens[pair[0].tokens.len() - 8..],
&pair[1].tokens[..8]
);
}
}
#[test]
fn token_chunk_configuration_is_bounded() {
let tokenizer = tiktoken::get_encoding("o200k_base").unwrap();
let encoded = Encoded::new(tokenizer, "hello").unwrap();
for (tokens, overlap) in [(0, 0), (8, 5), (8, 8), (8, 9)] {
assert!(matches!(
encoded.overlapping_chunks(tokens, overlap),
Err(Error::Configuration(_))
));
}
}
#[test]
fn lossless_chunking_rejects_an_input_that_requires_splitting() {
let tokenizer = tiktoken::get_encoding("o200k_base").unwrap();
let text = "word ".repeat(200);
let required_tokens = tokenizer.count_with_special_tokens(&text);
assert!(validate_lossless_chunk_request(tokenizer, &["short"], 32).is_ok());
assert!(matches!(
validate_lossless_chunk_request(tokenizer, &[&text], 32),
Err(Error::LosslessChunkingUnavailable {
required_tokens: actual,
chunk_tokens: 32,
}) if actual == required_tokens
));
}
#[test]
fn chunking_validation_uses_the_actual_fixed_trace_length() {
let tokenizer = tiktoken::get_encoding("o200k_base").unwrap();
let text = "word ".repeat(200);
let required_tokens = tokenizer.count_with_special_tokens(&text);
assert!(required_tokens > 128);
let trace_tokens = lossless_trace_tokens(4_096, Some(128));
assert_eq!(trace_tokens, 128);
assert!(matches!(
validate_lossless_chunk_request(tokenizer, &[&text], trace_tokens),
Err(Error::LosslessChunkingUnavailable {
required_tokens: actual,
chunk_tokens: 128,
}) if actual == required_tokens
));
assert!(matches!(
validate_approximate_trace_tokens(4_096, Some(128)),
Err(Error::Configuration(_))
));
assert!(validate_approximate_trace_tokens(128, Some(128)).is_ok());
assert!(validate_approximate_trace_tokens(4_096, None).is_ok());
}
#[test]
fn bioes_to_unicode_safe_redaction() {
let tokenizer = tiktoken::get_encoding("o200k_base").unwrap();
let text = "Call Alice Smith today";
let encoded = Encoded::new(tokenizer, text).unwrap();
let mut labels = vec![0; encoded.tokens.len()];
let first = encoded
.token_starts
.iter()
.zip(&encoded.token_ends)
.position(|(&start, &end)| start <= 4 && end > 4)
.unwrap();
let last = encoded.token_ends.iter().position(|&x| x >= 16).unwrap();
if first == last {
labels[first] = 20;
} else {
labels[first] = 17;
labels[last] = 19;
for item in &mut labels[first + 1..last] {
*item = 18;
}
}
let detection = Detection {
spans: decode_spans(text, &encoded, &labels).unwrap(),
};
assert_eq!(detection.redact(text), "Call <PRIVATE_PERSON> today");
}
#[test]
fn unicode_boundary_expansion_preserves_overlapping_labels() {
let text = "éx";
let encoded = Encoded {
tokens: vec![1, 2, 3],
token_starts: vec![0, 1, 2],
token_ends: vec![1, 2, 3],
};
let spans = decode_spans(text, &encoded, &[20, 13, 15]).unwrap();
assert_eq!(spans.len(), 2);
assert!(spans.contains(&Span {
label: Label::PrivatePerson,
start: 0,
end: 2,
}));
assert!(spans.contains(&Span {
label: Label::PrivateEmail,
start: 0,
end: 3,
}));
}
#[test]
fn redaction_skips_spans_that_split_utf8_characters() {
let text = "café";
let detection = Detection {
spans: vec![Span {
label: Label::PrivatePerson,
start: 3,
end: 4,
}],
};
assert_eq!(detection.redact(text), text);
}
#[test]
fn redaction_unions_overlapping_spans_without_leaking_the_suffix() {
let text = "abcdefghij";
let detection = Detection {
spans: vec![
Span {
label: Label::PrivateEmail,
start: 0,
end: 5,
},
Span {
label: Label::PrivatePerson,
start: 4,
end: 10,
},
],
};
assert_eq!(detection.redact(text), "<PRIVATE_EMAIL>");
}
#[test]
fn oversized_input_is_rejected_during_local_request_preparation() {
let tokenizer = tiktoken::get_encoding("o200k_base").unwrap();
let text = "x ".repeat(MAX_SEQUENCE_TOKENS + 1);
let error = prepare_request(tokenizer, &[&text])
.err()
.expect("oversized input should be rejected");
assert!(matches!(error, Error::Protocol(_)));
}
#[test]
fn byte_limit_rejects_input_before_token_vector_allocation() {
let tokenizer = tiktoken::get_encoding("o200k_base").unwrap();
let text = "x".repeat(MAX_SEQUENCE_BYTES + 1);
let error = Encoded::new(tokenizer, &text)
.err()
.expect("oversized byte input should be rejected");
assert!(matches!(error, Error::Protocol(_)));
}
#[test]
fn memory_limit_must_convert_to_a_positive_size_t() {
for limit in [0.0, -1.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
assert!(matches!(
validate_memory_limit(Some(limit)),
Err(Error::Configuration(_))
));
}
let size_t_overflow = (usize::MAX as f64) / 1_073_741_824.0;
assert!(matches!(
validate_memory_limit(Some(size_t_overflow)),
Err(Error::Configuration(_))
));
assert!(validate_memory_limit(None).is_ok());
assert!(validate_memory_limit(Some(0.5)).is_ok());
}
#[test]
fn cache_limit_is_independent_and_defaults_to_half_a_gibibyte() {
assert_eq!(ServerOptions::default().cache_limit_gb, Some(0.5));
assert!(validate_cache_limit(None).is_ok());
assert!(validate_cache_limit(Some(0.5)).is_ok());
for limit in [0.0, -1.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
assert!(matches!(
validate_cache_limit(Some(limit)),
Err(Error::Configuration(_))
));
}
}
#[test]
fn fixed_trace_must_be_positive_and_fit_the_padded_token_budget() {
assert!(ServerOptions::default().prewarm_fixed_trace);
assert!(validate_fixed_trace(None, 32_768).is_ok());
for shape in [
FixedTraceShape {
batch: 0,
tokens: 1024,
},
FixedTraceShape {
batch: 8,
tokens: 0,
},
FixedTraceShape {
batch: 8,
tokens: 1024,
},
] {
assert!(matches!(
validate_fixed_trace(Some(shape), 4096),
Err(Error::Configuration(_))
));
}
assert!(
validate_fixed_trace(
Some(FixedTraceShape {
batch: 4,
tokens: 1024,
}),
4096
)
.is_ok()
);
}
#[test]
fn default_options_load_sibling_calibration_in_official_bias_order() {
let directory = tempfile::tempdir().unwrap();
let calibration = directory.path().join("viterbi_calibration.json");
fs::write(
&calibration,
r#"{
"operating_points": {
"default": {
"biases": {
"transition_bias_background_stay": 1.0,
"transition_bias_background_to_start": 2.0,
"transition_bias_inside_to_continue": 3.0,
"transition_bias_inside_to_end": 4.0,
"transition_bias_end_to_background": 5.0,
"transition_bias_end_to_start": 6.0
}
}
}
}"#,
)
.unwrap();
let options = ServerOptions::default();
let model = directory.path().join("model.mlxfn");
assert_eq!(
resolve_viterbi_biases(&model, &options).unwrap(),
[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
);
}
#[test]
fn nonblocking_write_honors_timeout_when_pipe_is_full() {
let (_reader, writer) = nix::unistd::pipe().unwrap();
let mut writer = File::from(writer);
set_nonblocking(&writer).unwrap();
let payload = vec![0u8; 4 * 1024 * 1024];
let started = Instant::now();
let timeout = Duration::from_millis(50);
let error = write_all_before(
&mut writer,
&payload,
deadline_after(timeout),
timeout,
"test write",
)
.unwrap_err();
assert!(matches!(error, Error::Timeout { .. }));
assert!(started.elapsed() < Duration::from_secs(2));
}
#[test]
fn parses_bounded_response_header_fields() {
let mut header = [0u8; 16];
header[..4].copy_from_slice(RESPONSE_MAGIC);
header[4] = 1;
header[8..12].copy_from_slice(&7u32.to_le_bytes());
header[12..16].copy_from_slice(&11u32.to_le_bytes());
assert_eq!(parse_response_header(header), (1, 7, 11));
}
#[test]
fn spawn_times_out_when_server_never_becomes_ready() {
let directory = tempfile::tempdir().unwrap();
let server = directory.path().join("unresponsive-server");
fs::write(&server, "#!/bin/sh\nexec sleep 10\n").unwrap();
let mut permissions = fs::metadata(&server).unwrap().permissions();
permissions.set_mode(0o700);
fs::set_permissions(&server, permissions).unwrap();
let options = ServerOptions {
viterbi_biases: Some([0.0; 6]),
startup_timeout: Duration::from_millis(50),
shutdown_timeout: Duration::from_millis(50),
..ServerOptions::default()
};
let started = Instant::now();
let error = PrivacyFilter::spawn(&server, "unused-model", options)
.err()
.expect("unresponsive process should fail startup");
assert!(matches!(error, Error::Timeout { .. }));
assert!(started.elapsed() < Duration::from_secs(2));
}
}