tokenfold_core/
codec.rs

1use serde::{Deserialize, Serialize};
2use serde_json::Value;
3
4use crate::TokenFoldError;
5use crate::transforms::{json_dict, json_fold, log_fold};
6
7#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
8#[serde(rename_all = "snake_case")]
9pub enum OutputEncoding {
10    #[default]
11    Json,
12    Toon,
13}
14
15#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
16#[serde(rename_all = "snake_case")]
17pub enum DecodeFormat {
18    #[default]
19    Auto,
20    Json,
21    Toon,
22    Text,
23}
24
25pub fn encode_toon(input: &[u8]) -> Result<Vec<u8>, TokenFoldError> {
26    let value: Value = serde_json::from_slice(input).map_err(|e| {
27        TokenFoldError::InvalidInput(format!("invalid JSON for TOON encoding: {e}"))
28    })?;
29    let encoded = toon_format::encode_default(&value)
30        .map_err(|e| TokenFoldError::InternalError(format!("TOON encoding failed: {e}")))?;
31    let decoded: Value = toon_format::decode_default(&encoded)
32        .map_err(|e| TokenFoldError::InternalError(format!("TOON verification failed: {e}")))?;
33    if decoded != value {
34        return Err(TokenFoldError::InternalError(
35            "TOON round-trip verification mismatch".to_string(),
36        ));
37    }
38    Ok(encoded.into_bytes())
39}
40
41pub fn decode(input: &[u8], from: DecodeFormat) -> Result<Vec<u8>, TokenFoldError> {
42    let from = match from {
43        DecodeFormat::Auto => detect_decode_format(input)?,
44        explicit => explicit,
45    };
46    match from {
47        DecodeFormat::Json => decode_json(input),
48        DecodeFormat::Toon => {
49            let text = std::str::from_utf8(input).map_err(|e| {
50                TokenFoldError::InvalidInput(format!("TOON is not valid UTF-8: {e}"))
51            })?;
52            let value: Value = toon_format::decode_default(text)
53                .map_err(|e| TokenFoldError::InvalidInput(format!("invalid TOON: {e}")))?;
54            decode_json(&serde_json::to_vec(&value).map_err(|e| {
55                TokenFoldError::InternalError(format!("failed to serialize decoded TOON: {e}"))
56            })?)
57        }
58        DecodeFormat::Text => {
59            let text = std::str::from_utf8(input).map_err(|e| {
60                TokenFoldError::InvalidInput(format!("text is not valid UTF-8: {e}"))
61            })?;
62            Ok(log_fold::unfold_log(text).into_bytes())
63        }
64        DecodeFormat::Auto => unreachable!("auto is resolved above"),
65    }
66}
67
68fn decode_json(input: &[u8]) -> Result<Vec<u8>, TokenFoldError> {
69    let undicted = json_dict::undict_json(input)
70        .map_err(|e| TokenFoldError::InvalidInput(format!("invalid Tokenfold JSON frame: {e}")))?;
71    json_fold::unfold_json(&undicted)
72        .map_err(|e| TokenFoldError::InvalidInput(format!("invalid Tokenfold JSON frame: {e}")))
73}
74
75fn detect_decode_format(input: &[u8]) -> Result<DecodeFormat, TokenFoldError> {
76    let text = std::str::from_utf8(input).map_err(|e| {
77        TokenFoldError::InvalidInput(format!("encoded input is not valid UTF-8: {e}"))
78    })?;
79    if text.starts_with("__tf_logfold1__\n") {
80        return Ok(DecodeFormat::Text);
81    }
82    let json = serde_json::from_slice::<Value>(input).is_ok();
83    let toon = toon_format::decode_default::<Value>(text).is_ok();
84    match (json, toon) {
85        (true, false) => Ok(DecodeFormat::Json),
86        (false, true) => Ok(DecodeFormat::Toon),
87        (true, true) => Err(TokenFoldError::InvalidInput(
88            "encoded input is ambiguous; pass --from json or --from toon".to_string(),
89        )),
90        (false, false) => Err(TokenFoldError::InvalidInput(
91            "cannot detect encoded input; pass --from json, toon, or text".to_string(),
92        )),
93    }
94}
95
96#[cfg(test)]
97mod tests {
98    use super::*;
99
100    #[test]
101    fn toon_round_trip_then_reverses_tokenfold_frames() {
102        let source = br#"[{"id":1,"name":"Ada"},{"id":2,"name":"Lin"}]"#;
103        let folded = json_fold::fold_json(source).unwrap();
104        let toon = encode_toon(&folded).unwrap();
105        let decoded = decode(&toon, DecodeFormat::Toon).unwrap();
106        assert_eq!(
107            serde_json::from_slice::<Value>(&decoded).unwrap(),
108            serde_json::from_slice::<Value>(source).unwrap()
109        );
110    }
111
112    #[test]
113    fn auto_rejects_unknown_input() {
114        assert!(decode(b"not valid {", DecodeFormat::Auto).is_err());
115    }
116}
tokenfold_core/codec.rs

tokenfold_core/
codec.rs
