serde_llsd/de/

notation.rs

//! #  de/notation -- de-serialize LLSD, "notation" form.
//!
//!  Library for serializing and de-serializing data in
//!  Linden Lab Structured Data format.
//!
//!  Format documentation is at http://wiki.secondlife.com/wiki/LLSD
//!
//!  Notation format.
//!  Similar to JSON, but not compatible
//!
//! Notation format comes in two forms - bytes, and UTF-8 characters.
//! UTF-8 format is always valid UTF-8 strings, and can be encapsulated
//! inside XML if desired. This format is used inside SL/OS for "gltf material overrides".
//!
//! Byte string form is binary bytes, and cannot be encapsulated inside XML.
//! It can contain raw binary fields of the form b(NN)"rawbytes".
//! and raw strings of the form s(NN)"rawstring".
//! This form is used inside SL/OS for script uploads. We think.
//
//  Animats
//  June, 2023.
//  License: LGPL.
//
use crate::LLSDValue;
use anyhow::{anyhow, Error};
use std::collections::HashMap;
use core::iter::{Peekable};
use core::str::{Chars, Bytes};
use uuid::{Uuid};
use chrono::DateTime;
use base64::Engine;

//
//  Constants
//
/// Notation LLSD prefix
pub const LLSDNOTATIONPREFIX: &str = "<? llsd/notation ?>\n"; 
/// Sentinel, must match exactly.
pub const LLSDNOTATIONSENTINEL: &str = LLSDNOTATIONPREFIX;

/// Exported parse from bytes.
pub fn from_bytes(b: &[u8]) -> Result<LLSDValue, Error> {
    LLSDStreamBytes::parse(b)
}

/// Exported parse from str.
pub fn from_str(s: &str) -> Result<LLSDValue, Error> {
    LLSDStreamChars::parse(s)
}

/// An LLSD stream. May be either a UTF-8 stream or a byte stream.
/// Generic trait.
trait LLSDStream<C, S> {
    /// Get next char/byte
    fn next(&mut self) -> Option<C>;
    
    /// Get next char/byte, result
    fn next_ok(&mut self) -> Result<C, Error> {
        if let Some(ch) = self.next() {
            Ok(ch)
        } else {
            Err(anyhow!("Unexpected end of input parsing Notation"))
        }           
    }
    
    /// Peek at next char/byte
    fn peek(&mut self) -> Option<&C>;
    
    //  Peek at next char, as result
    fn peek_ok(&mut self) -> Result<&C, Error> {
        if let Some(ch) = self.peek() {
            Ok(ch)
        } else {
            Err(anyhow!("Unexpected end of input parsing Notation"))
        }           
    }
    
    /// Convert into char
    fn into_char(ch: &C) -> char;
    
    /// Consume whitespace. Next char will be non-whitespace.
    //  Need to treat explicit "\n" as whitespace.
    fn consume_whitespace(&mut self) -> Result<(), Error> {
        while let Some(ch) = self.peek() {
            match Self::into_char(ch) {
                ' ' | '\n' => { let _ = self.next(); },                 // ignore leading white space
                '\\' => {
                    let _ = self.next();                                // consume backslash
                    let ch = Self::into_char(&self.next_ok()?);         // expecting 'n'
                    if ch != 'n' {                                      // Explicit "\n" is normal white space
                        return Err(anyhow!("Unexpected escape sequence \"\\{}\" where white space expected.", ch));
                    }   
                }
                _ => break
            }
        }
        Ok(())  
    }
    
    /// Consume expected non-whitespace char
    fn consume_char(&mut self, expected_ch: char) -> Result<(), Error> {
        self.consume_whitespace()?;
        let ch = Self::into_char(&self.next_ok()?);
        if ch == expected_ch {
            Ok(())
        } else {
            Err(anyhow!("Expected '{}', found '{}'.", expected_ch, ch))
        }
    }

    /// Parse "iNNN"
    fn parse_integer(&mut self) -> Result<LLSDValue, Error> {
        let mut s = String::with_capacity(20);  // pre-allocate; can still grow
        //  Accumulate numeric chars.
        while let Some(ch) = self.peek() {
            match Self::into_char(ch) {
                '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9'|'+'|'-' => s.push(Self::into_char(&self.next().unwrap())),
                 _ => break
            }
        }
        //  Digits accmulated, use standard conversion
        Ok(LLSDValue::Integer(s.parse::<i32>()?))
    }
        /// Parse "rNNN".
    //  Does "notation" allow exponents?
    fn parse_real(&mut self) -> Result<LLSDValue, Error> {
        let mut s = String::with_capacity(20);  // pre-allocate; can still grow
        //  Accumulate numeric chars.
        //  This will not accept NaN.
        while let Some(ch) = self.peek() {
            match Self::into_char(ch) {
                '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9'|'+'|'-'|'.' => s.push(Self::into_char(&self.next().unwrap())),
                 _ => break
            }
        }
        //  Digits accmulated, use standard conversion
        Ok(LLSDValue::Real(s.parse::<f64>()?))
    }
    
    /// Parse Boolean
    fn parse_boolean(&mut self, first_char: char) -> Result<LLSDValue, Error> {
        //  Accumulate next word
        let mut s = String::with_capacity(4);
        s.push(first_char);     // we already had the first character.        
        loop {              
            if let Some(ch) = self.peek() {
                if Self::into_char(ch).is_alphabetic() {
                    s.push(Self::into_char(&self.next().unwrap()));
                    continue
                }
            }
            break;
        }
        //  Check for all the allowed Boolean forms.
        match s.as_str() {
            "f" | "F" | "false" | "FALSE" => Ok(LLSDValue::Boolean(false)),
            "t" | "T" | "true" | "TRUE" => Ok(LLSDValue::Boolean(true)),
            _ => Err(anyhow!("Parsing Boolean, got {}", s)) 
        }
    }
    /// Parse string. "ABC" or 'ABC', with '\' as escape.
    /// Allowed escapes are \\, \", \', and \n
    /// Does not parse the numeric count prefix form.
    fn parse_quoted_string(&mut self, delim: char) -> Result<String, Error> {
        self.consume_whitespace()?;
        let mut s = String::with_capacity(128);             // allocate reasonably large size for typical string.
        loop {
            let ch = Self::into_char(&self.next_ok()?);     // next char, must be present
            if ch == delim { break }                        // end of string
            if ch == '\\' {                                 // escape
                let ch = Self::into_char(&self.next_ok()?); // next char, must be present
                match ch {
                    '\\' | '\'' | '\"' => s.push(ch),       // escapable characters
                    'n' => s.push('\n'),                    // backslash n becomes newline
                    _ => { return Err(anyhow!("Unexpected escape sequence \"\\{}\" within quoted string.", ch)); }
                }
            } else {
                s.push(ch)
            }
        }
        String::shrink_to_fit(&mut s);                      // release wasted space
        Ok(s)
    }   
    /// Parse date string per RFC 1339.
    fn parse_date(&mut self) -> Result<LLSDValue, Error> {
        if let Some(delim) = self.next() {
            if Self::into_char(&delim) == '"' || Self::into_char(&delim) == '\'' {
                let s = self.parse_quoted_string(Self::into_char(&delim))?;
                let naive_date =  DateTime::parse_from_rfc3339(&s)?; // parse date per RFC 3339.
                Ok(LLSDValue::Date(naive_date.timestamp())) // seconds since UNIX epoch.
            } else {
                Err(anyhow!("URI did not begin with '\"'"))
            }
        } else {
            Err(anyhow!("URI at end of file."))
        }
    }
    
    /// Parse URI string per rfc 1738
    fn parse_uri(&mut self) -> Result<LLSDValue, Error> {
        if let Some(delim) = self.next() {
            if Self::into_char(&delim) == '"' || Self::into_char(&delim) == '\'' {
                let s = self.parse_quoted_string(Self::into_char(&delim))?;
                Ok(LLSDValue::URI(urlencoding::decode(&s)?.to_string()))
            } else {
                Err(anyhow!("URI did not begin with '\"'"))
            }
        } else {
            Err(anyhow!("URI at end of file."))
        }
    }    
    /// Parse UUID. No quotes
    fn parse_uuid(&mut self) -> Result<LLSDValue, Error> {
        const UUID_LEN: usize = "c69b29b1-8944-58ae-a7c5-2ca7b23e22fb".len();   // just to get the length of a standard format UUID.
        let mut s = String::with_capacity(UUID_LEN);
        for _ in 0..UUID_LEN {
            s.push(Self::into_char(&(self.next().ok_or(anyhow!("EOF parsing UUID"))?)));
        }
        Ok(LLSDValue::UUID(Uuid::parse_str(&s)?))
    }

    /// Parse "{ 'key' : value, 'key' : value ... }
    fn parse_map(&mut self) -> Result<LLSDValue, Error> {
        let mut kvmap = HashMap::new();                         // building map
        loop {
            self.consume_whitespace()?;
            let key =  {
                let ch = Self::into_char(&self.next_ok()?);
                match ch {
                    '}' => { break } // end of map, may be empty.
                    '\'' | '"' => self.parse_quoted_string(ch)?, 
                    _ => { return Err(anyhow!("Map key began with {} instead of quote.", ch)); }
                }
            };
            self.consume_char(':')?;
            let value = self.parse_value()?;           // value of key:value
            kvmap.insert(key, value);
            //  Check for comma indicating more items.
            self.consume_whitespace()?;
            if Self::into_char(self.peek_ok()?) == ',' {
                let _ = self.next();    // consume comma, continue with next field
            }
        }
        Ok(LLSDValue::Map(kvmap))
    }
        
    /// Parse "[ value, value ... ]"
    /// At this point, the '[' has been consumed.
    /// At successful return, the ending ']' has been consumed.
    fn parse_array(&mut self) -> Result<LLSDValue, Error> {
        let mut array_items = Vec::new();
        //  Accumulate array elements.
        loop {
            //  Check for end of items
            self.consume_whitespace()?;
            let ch = Self::into_char(self.peek_ok()?);
            if ch == ']' {
                let _ = self.next(); break;    // end of array, may be empty.
            }
            array_items.push(self.parse_value()?);          // parse next value
            //  Check for comma indicating more items.
            self.consume_whitespace()?;
            if Self::into_char(self.peek_ok()?) == ',' {
                let _ = self.next();    // consume comma, continue with next field
            }           
        }
        Ok(LLSDValue::Array(array_items))               // return array
    }
    
    fn parse_binary(&mut self) -> Result<LLSDValue, Error>; // passed down to next level
    
    fn parse_sized_string(&mut self) -> Result<LLSDValue, Error>; // passed down to next level
        
    
    /// Parse one value - real, integer, map, etc. Recursive.
    /// This is the top level of the parser
    fn parse_value(&mut self) -> Result<LLSDValue, Error> {
        self.consume_whitespace()?;                      // ignore leading white space
        let ch = Self::into_char(&self.next_ok()?);
        match ch {
            '!' => { Ok(LLSDValue::Undefined) }         // "Undefined" as a value
            '0' => { Ok(LLSDValue::Boolean(false)) }    // false
            '1' => { Ok(LLSDValue::Boolean(true)) }     // true
            'f' | 'F' => { self.parse_boolean(ch) }     // false, all alpha forms
            't' | 'T' => { self.parse_boolean(ch) }     // true, all alpha forms
            '{' => { self.parse_map() }                 // map
            '[' => { self.parse_array() }               // array
            'i' => { self.parse_integer() }             // integer
            'r' => { self.parse_real() }                // real
            'd' => { self.parse_date() }                // date
            'u' => { self.parse_uuid() }                // UUID
            'l' => { self.parse_uri() }                 // URI
            'b' => { self.parse_binary() }              // binary
            's' => { self.parse_sized_string() }        // string with explicit size
            '"' => { Ok(LLSDValue::String(self.parse_quoted_string(ch)?)) }  // string, double quoted
            '\'' => { Ok(LLSDValue::String(self.parse_quoted_string(ch)?)) }  // string, double quoted
            //  ***MORE*** add cases for UUID, URL, date, and binary.
            _ => { Err(anyhow!("Unexpected character: {:?}", ch)) } // error
        }
    }
}

/// Stream, composed of UTF-8 chars.
struct LLSDStreamChars<'a> {
    /// Stream is composed of peekable UTF-8 chars
    cursor: Peekable<Chars<'a>>,
}

impl LLSDStream<char, Peekable<Chars<'_>>> for LLSDStreamChars<'_> {
    /// Get next UTF-8 char.
    fn next(&mut self) -> Option<char> {
        self.cursor.next()
    }
    /// Peek at next UTF-8 char.
    fn peek(&mut self) -> Option<&char> {
        self.cursor.peek()
    }
    /// Into char, which is a null conversion
    fn into_char(ch: &char) -> char {
        *ch
    }  
    
    /// Won't work.
    fn parse_binary(&mut self) -> Result<LLSDValue, Error> {
        Err(anyhow!("Byte-counted binary data inside UTF-8 won't work."))
    }
    
    /// Won't work.
    fn parse_sized_string(&mut self) -> Result<LLSDValue, Error> {
        Err(anyhow!("Byte-counted string data inside UTF-8 won't work."))
    }
}

impl LLSDStreamChars<'_> {
    /// Parse LLSD string expressed in notation format into an LLSDObject tree. No header.
    /// Strng form
    pub fn parse(notation_str: &str) -> Result<LLSDValue, Error> {
        let mut stream = LLSDStreamChars { cursor: notation_str.chars().peekable() };
        match stream.parse_value() {
            Ok(v) => Ok(v),
            Err(e) => {
                //  Useful error message
                let s = beginning_to_iterator(notation_str, &stream.cursor);
                Err(anyhow!("LLSD notation string parse error: {:?}. Parse got this far: {}", e, s))
            }
        }
    }
}

/// Stream, composed of raw bytes.
struct LLSDStreamBytes<'a> {
    /// Stream is composed of peekable bytes.
    cursor: Peekable<std::slice::Iter<'a, u8>>,
}

impl LLSDStream<u8, Peekable<Bytes<'_>>> for LLSDStreamBytes<'_> {
    /// Get next byte.
    fn next(&mut self) -> Option<u8> {
        self.cursor.next().copied()
    }
    /// Peek at next byte.
    fn peek(&mut self) -> Option<&u8> {
        self.cursor.peek().copied()
    }
    /// Into char, which is a real conversion to a UTF-8 char.
    fn into_char(ch: &u8) -> char {
        (*ch).into()
    }
    
    /// Parse binary value.
    /// Format is b16"value" or b64"value" or b(cnt)"value".
    /// Putting text in this format is just wrong, yet the LL example does it.
    /// This conversion may fail for non-ASCII input.
    //
    //  The LL parser for this is at
    //  https://github.com/secondlife/viewer/blob/ec4135da63a3f3877222fba4ecb59b15650371fe/indra/llcommon/llsdserialize.cpp#L789
    //  That reads N bytes from the input as a byte stream. We only do this for byte streams, not Strings.
    //
    fn parse_binary(&mut self) -> Result<LLSDValue, Error> {
        if let Some(ch) = self.peek() {
            match Self::into_char(ch) {
                '(' => {
                    let cnt = self.parse_number_in_parentheses()?;
                    self.consume_char('"')?;
                    let s = self.next_chunk(cnt)?;
                    self.consume_char('"')?;     // count must be correct or this will fail.
                    Ok(LLSDValue::Binary(s))     // not sure about this
                }                 
                '1' => {
                    self.consume_char('1')?;
                    self.consume_char('6')?;          // base 16
                    self.consume_char('"')?;          // begin quote
                    let mut s = self.parse_quoted_string('"')?;
                    s.retain(|c| !c.is_whitespace());
                    Ok(LLSDValue::Binary(hex::decode(s)?))
                }
                '6' => {
                    self.consume_char('6')?;
                    self.consume_char('4')?;
                    self.consume_char('"')?;          // begin quote
                    let mut s = self.parse_quoted_string('"')?;
                    s.retain(|c| !c.is_whitespace());
                    println!("Base 64 decode input: \"{}\"", s);    // ***TEMP***
                    let bytes = base64::engine::general_purpose::STANDARD.decode(s)?;
                    Ok(LLSDValue::Binary(bytes))
                }
                _ => Err(anyhow!("Binary value started with {} instead of (, 1, or 6", ch))   
            } 
        } else {
            Err(anyhow!("Binary value started with EOF"))   
        }
    }
    
    /// Parse sized string.
    /// Format is s(NNN)"string"
    fn parse_sized_string(&mut self) -> Result<LLSDValue, Error> {
        let cnt = self.parse_number_in_parentheses()?;
        //  At this point, we are supposed to have a quoted string of ASCII characters.
        //  If this can be validy converted as UTF-8, it will be accepted.
        self.consume_char('"')?;
        let s = self.next_chunk(cnt)?;
        self.consume_char('"')?;
        Ok(LLSDValue::String(String::from_utf8(s)?))
    }
}

impl LLSDStreamBytes<'_> {
    /// Parse LLSD string expressed in notation format into an LLSDObject tree. No header.
    /// Bytes form.
    pub fn parse(notation_bytes: &[u8]) -> Result<LLSDValue, Error> {
        let mut stream = LLSDStreamBytes { cursor: notation_bytes.iter().peekable() };
        stream.parse_value()
    }

    /// Parse (NNN), which is used for length information.
    fn parse_number_in_parentheses(&mut self) -> Result<usize, Error> {
        self.consume_char('(')?;
        let val = self.parse_integer()?;
        self.consume_char(')')?;   
        if let LLSDValue::Integer(v) = val {
            Ok(v as usize)
        } else {
            panic!("Integer parse did not return an integer.");
        }
    }
    
    /// Read chunk of N bytes.
    fn next_chunk(&mut self, cnt: usize) -> Result<Vec<u8>, Error> {
        let mut s = Vec::with_capacity(cnt);
        //  next_chunk, for getting N chars, doesn't work yet.
        for _ in 0..cnt {
            s.push(self.next_ok()?);
        }
        Ok(s)
    }

}

//  Utility functions
/// Extract the part of a string from the beginning to an iterator.
fn beginning_to_iterator<'a>(orig: &'a str, pos: &Peekable<Chars>) -> &'a str {
    let suffix: String = pos.clone().collect();
    println!("Suffix: {}", suffix);
    if let Some(s) = orig.strip_suffix(&suffix) {
        s
    } else {
        orig
    }
}

#[test]
/// Unit tests
fn notationparse1() {
    let s1 = "\"ABC☺DEF\"".to_string();  // string, including quotes, with emoji.
    let mut stream1 = LLSDStreamChars { cursor: s1.chars().peekable() };
    stream1.consume_char('"').unwrap(); // leading quote
    let v1 = stream1.parse_quoted_string('"').unwrap();
    assert_eq!(v1, "ABC☺DEF");
}

#[test]
fn notationparse2() {
    //  Linden Lab documented test data from wiki. Compatibility test use only.
    const TESTNOTATION2: &str = r#"
[
  {'destination':l"http://secondlife.com"}, 
  {'version':i1}, 
  {
    'agent_id':u3c115e51-04f4-523c-9fa6-98aff1034730, 
    'session_id':u2c585cec-038c-40b0-b42e-a25ebab4d132, 
    'circuit_code':i1075, 
    'first_name':'Phoenix', 
    'last_name':'Linden',
    'position':[r70.9247,r254.378,r38.7304], 
    'look_at':[r-0.043753,r-0.999042,r0], 
    'granters':[ua2e76fcd-9360-4f6d-a924-000000000003],
    'attachment_data':
    [
      {
        'attachment_point':i2,
        'item_id':ud6852c11-a74e-309a-0462-50533f1ef9b3,
        'asset_id':uc69b29b1-8944-58ae-a7c5-2ca7b23e22fb
      },
      {
        'attachment_point':i10, 
        'item_id':uff852c22-a74e-309a-0462-50533f1ef900,
        'asset_id':u5868dd20-c25a-47bd-8b4c-dedc99ef9479
      }
    ]
  }
]
"#;
    let parsed_s = LLSDStreamChars::parse(TESTNOTATION2);
    println!("Parse of string form {}: \n{:#?}", TESTNOTATION2, parsed_s);
    let parsed_b = LLSDStreamBytes::parse(TESTNOTATION2.as_bytes());
    println!("Parse of byte form: {:#?}", parsed_b);
    assert_eq!(parsed_s.unwrap(), parsed_b.unwrap());
}

#[test]
fn notationparse3() {
    //  Linden Lab documented test data from wiki. Compatibility test use only.
    const TESTNOTATION3: &str = r#"
[
  {
    'creation-date':d"2007-03-15T18:30:18Z", 
    'creator-id':u3c115e51-04f4-523c-9fa6-98aff1034730
  },
  s(10)"0123456789",
  "Where's the beef?",
  'Over here.',  
  b(158)"default
{
    state_entry()
    {
        llSay(0, "Hello, Avatar!");
    }

    touch_start(integer total_number)
    {
        llSay(0, "Touched.");
    }
}",
  b64"AABAAAAAAAAAAAIAAAA//wAAP/8AAADgAAAA5wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AABkAAAAZAAAAAAAAAAAAAAAZAAAAAAAAAABAAAAAAAAAAAAAAAAAAAABQAAAAEAAAAQAAAAAAAA
AAUAAAAFAAAAABAAAAAAAAAAPgAAAAQAAAAFAGNbXgAAAABgSGVsbG8sIEF2YXRhciEAZgAAAABc
XgAAAAhwEQjRABeVAAAABQBjW14AAAAAYFRvdWNoZWQuAGYAAAAAXF4AAAAIcBEI0QAXAZUAAEAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" 
]
"#;
    let parsed_b = from_bytes(TESTNOTATION3.as_bytes()).unwrap();
    println!("Parse of byte form: {:#?}", parsed_b);
    let parsed_b = from_str(TESTNOTATION3);
    assert!(parsed_b.is_err());             // not allowed to have b(158) or s(10) in string mode.
    println!("Parse of string form: {:#?}", parsed_b);
}

#[test]
fn notationparse4() {
    //  This is a "material override".
    const TESTNOTATION4: &str = r#"
        {'gltf_json':['{\"asset\":{\"version\":\"2.0\"},\"images\":[{\"uri\":\"5748decc-f629-461c-9a36-a35a221fe21f\"},
            {\"uri\":\"5748decc-f629-461c-9a36-a35a221fe21f\"}],\"materials\":[{\"occlusionTexture\":{\"index\":1},\"pbrMetallicRoughness\":{\"metallicRoughnessTexture\":{\"index\":0},\"roughnessFactor\":0.20000000298023224}}],\"textures\":[{\"source\":0},
            {\"source\":1}]}\\n'],'local_id':i8893800,'object_id':u6ac43d70-80eb-e526-ec91-110b4116293e,'region_handle_x':i342016,'region_handle_y':i343552,'sides':[i0]}"
"#;
    let parsed_b = from_bytes(TESTNOTATION4.as_bytes());
    println!("Parse of byte form: {:#?}", parsed_b);
    let local_id = *parsed_b.unwrap().as_map().unwrap().get("local_id").unwrap().as_integer().unwrap();
    assert_eq!(local_id, 8893800); // validate local ID
    let parsed_b = from_str(TESTNOTATION4);
    println!("Parse of str form: {:#?}", parsed_b);
    let local_id = *parsed_b.unwrap().as_map().unwrap().get("local_id").unwrap().as_integer().unwrap();
    assert_eq!(local_id, 8893800); // validate local ID
}

#[test]
fn notationparse5() {
    //  Parse with error. Note unwanted backslash in "images".
    const TESTNOTATION5: &str = r#"
        {'gltf_json':['{\"asset\":{\"version\":\"2.0\"},\"im\ages\":[{\"uri\":\"5748decc-f629-461c-9a36-a35a221fe21f\"},
            {\"uri\":\"5748decc-f629-461c-9a36-a35a221fe21f\"}],\"materials\":[{\"occlusionTexture\":{\"index\":1},\"pbrMetallicRoughness\":{\"metallicRoughnessTexture\":{\"index\":0},\"roughnessFactor\":0.20000000298023224}}],\"textures\":[{\"source\":0},
            {\"source\":1}]}\\n'],'local_id':i8893800,'object_id':u6ac43d70-80eb-e526-ec91-110b4116293e,'region_handle_x':i342016,'region_handle_y':i343552,'sides':[i0]}"
"#;
    println!("Test notationparse5");
    let parsed_b = from_str(TESTNOTATION5);
    println!("Parse of byte form: {:#?}", parsed_b);
    assert!(parsed_b.is_err());
}