pub const JSON: &str = "json";
pub const F32_BASE64: &str = "f32-base64";
pub struct Encoded {
pub blobs: Vec<String>,
pub json: Vec<Vec<f32>>,
pub meta: Option<(String, usize)>,
}
pub fn encode_vectors(vectors: &[Vec<f32>], encoding: &str) -> Result<Encoded, String> {
match encoding {
JSON => Ok(Encoded {
blobs: Vec::new(),
json: vectors.to_vec(),
meta: None,
}),
F32_BASE64 => {
let dim = vectors.first().map_or(0, |v| v.len());
let mut blobs = Vec::with_capacity(vectors.len());
for v in vectors {
if v.len() != dim {
return Err("all vectors must share one dimension for f32-base64".into());
}
let mut raw = Vec::with_capacity(dim * 4);
for &x in v {
raw.extend_from_slice(&x.to_le_bytes());
}
blobs.push(base64_encode(&raw));
}
Ok(Encoded {
blobs,
json: Vec::new(),
meta: Some((F32_BASE64.to_string(), dim)),
})
}
other => Err(format!("unknown vector encoding {other:?}")),
}
}
pub fn decode_f32_base64(blobs: &[String], dimension: Option<usize>) -> Result<Vec<Vec<f32>>, String> {
let mut out = Vec::with_capacity(blobs.len());
for blob in blobs {
let raw = base64_decode(blob)?;
if raw.len() % 4 != 0 {
return Err("f32-base64 blob length is not a multiple of 4 bytes".into());
}
let n = raw.len() / 4;
if let Some(d) = dimension {
if n != d {
return Err(format!("blob has {n} floats, expected dimension {d}"));
}
}
let mut v = Vec::with_capacity(n);
for chunk in raw.chunks_exact(4) {
v.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]));
}
out.push(v);
}
Ok(out)
}
const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
fn base64_encode(raw: &[u8]) -> String {
let mut out = String::with_capacity((raw.len() + 2) / 3 * 4);
for chunk in raw.chunks(3) {
let b0 = chunk[0] as u32;
let b1 = *chunk.get(1).unwrap_or(&0) as u32;
let b2 = *chunk.get(2).unwrap_or(&0) as u32;
let n = (b0 << 16) | (b1 << 8) | b2;
out.push(B64[((n >> 18) & 63) as usize] as char);
out.push(B64[((n >> 12) & 63) as usize] as char);
out.push(if chunk.len() > 1 { B64[((n >> 6) & 63) as usize] as char } else { '=' });
out.push(if chunk.len() > 2 { B64[(n & 63) as usize] as char } else { '=' });
}
out
}
fn base64_decode(s: &str) -> Result<Vec<u8>, String> {
fn val(c: u8) -> Result<u32, String> {
match c {
b'A'..=b'Z' => Ok((c - b'A') as u32),
b'a'..=b'z' => Ok((c - b'a' + 26) as u32),
b'0'..=b'9' => Ok((c - b'0' + 52) as u32),
b'+' => Ok(62),
b'/' => Ok(63),
_ => Err(format!("invalid base64 byte {c:?}")),
}
}
let bytes: Vec<u8> = s.bytes().filter(|&c| c != b'=' && !c.is_ascii_whitespace()).collect();
let mut out = Vec::with_capacity(bytes.len() / 4 * 3);
for chunk in bytes.chunks(4) {
let mut acc = 0u32;
let mut bits = 0u32;
for &c in chunk {
acc = (acc << 6) | val(c)?;
bits += 6;
if bits >= 8 {
bits -= 8;
out.push(((acc >> bits) & 0xff) as u8);
}
}
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn f32_base64_round_trips() {
let vecs = vec![vec![1.5f32, -2.25, 0.0, 3.125], vec![0.5, 0.5, 0.5, 0.5]];
let enc = encode_vectors(&vecs, F32_BASE64).unwrap();
assert_eq!(enc.meta, Some(("f32-base64".to_string(), 4)));
let back = decode_f32_base64(&enc.blobs, Some(4)).unwrap();
assert_eq!(back, vecs);
}
#[test]
fn json_passes_through() {
let vecs = vec![vec![1.0f32, 2.0, 3.0]];
let enc = encode_vectors(&vecs, JSON).unwrap();
assert!(enc.meta.is_none());
assert_eq!(enc.json, vecs);
}
#[test]
fn base64_matches_reference_vectors() {
assert_eq!(base64_encode(b""), "");
assert_eq!(base64_encode(b"f"), "Zg==");
assert_eq!(base64_encode(b"fo"), "Zm8=");
assert_eq!(base64_encode(b"foo"), "Zm9v");
assert_eq!(base64_encode(b"foobar"), "Zm9vYmFy");
assert_eq!(base64_decode("Zm9vYmFy").unwrap(), b"foobar");
assert_eq!(base64_decode("Zm8=").unwrap(), b"fo");
}
#[test]
fn ragged_rejected_and_dimension_enforced() {
assert!(encode_vectors(&[vec![1.0, 2.0], vec![3.0]], F32_BASE64).is_err());
let enc = encode_vectors(&[vec![1.0f32, 2.0, 3.0, 4.0]], F32_BASE64).unwrap();
assert!(decode_f32_base64(&enc.blobs, Some(8)).is_err());
}
}