clipasm 0.3.2

A strict, typed language for compiling and rendering Video and Audio graphs
Documentation
use crate::diagnostic::{BuiltinDiagnostic, Diagnostic, Result};
use crate::source::{SourceFile, SourceSpan};

#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct Token {
    pub(crate) kind: TokenKind,
    pub(crate) span: SourceSpan,
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) enum TokenKind {
    Identifier(String),
    Number(String),
    String(String),
    Newline,
    Dollar,
    At,
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    LeftBracket,
    RightBracket,
    LeftAngle,
    RightAngle,
    Colon,
    DoubleColon,
    Comma,
    Equal,
    Plus,
    Minus,
    Star,
    Slash,
    Percent,
    DotDot,
    End,
}

pub(crate) fn lex(source: SourceFile) -> Result<Vec<Token>> {
    Lexer::new(source).lex()
}

struct Lexer {
    source: SourceFile,
    offset: usize,
    line: usize,
    column: usize,
}

impl Lexer {
    fn new(source: SourceFile) -> Self {
        Self {
            source,
            offset: 0,
            line: 1,
            column: 1,
        }
    }

    fn lex(mut self) -> Result<Vec<Token>> {
        let mut tokens = Vec::new();
        while let Some(character) = self.peek() {
            match character {
                ' ' | '\t' | '\r' => self.advance(),
                '\n' => {
                    let span = self.span();
                    self.advance();
                    tokens.push(Token {
                        kind: TokenKind::Newline,
                        span,
                    });
                }
                '#' => self.skip_comment(),
                '"' => tokens.push(self.string()?),
                character if is_identifier_start(character) => tokens.push(self.identifier()),
                character if character.is_ascii_digit() => tokens.push(self.number()),
                _ => tokens.push(self.punctuation()?),
            }
        }
        tokens.push(Token {
            kind: TokenKind::End,
            span: self.span(),
        });
        Ok(tokens)
    }

    fn identifier(&mut self) -> Token {
        let span = self.span();
        let start = self.offset;
        self.advance();
        while self.peek().is_some_and(is_identifier_continue) {
            self.advance();
        }
        Token {
            kind: TokenKind::Identifier(self.source.text()[start..self.offset].to_owned()),
            span,
        }
    }

    fn number(&mut self) -> Token {
        let span = self.span();
        let start = self.offset;
        while self
            .peek()
            .is_some_and(|character| character.is_ascii_digit())
        {
            self.advance();
        }
        if self.peek() == Some('.')
            && self
                .peek_next()
                .is_some_and(|character| character.is_ascii_digit())
        {
            self.advance();
            while self
                .peek()
                .is_some_and(|character| character.is_ascii_digit())
            {
                self.advance();
            }
        }
        Token {
            kind: TokenKind::Number(self.source.text()[start..self.offset].to_owned()),
            span,
        }
    }

    fn string(&mut self) -> Result<Token> {
        let span = self.span();
        self.advance();
        let mut value = String::new();
        loop {
            match self.peek() {
                Some('"') => {
                    self.advance();
                    return Ok(Token {
                        kind: TokenKind::String(value),
                        span,
                    });
                }
                Some('\\') => {
                    self.advance();
                    let escaped = match self.peek() {
                        Some('"') => '"',
                        Some('\\') => '\\',
                        Some('n') => '\n',
                        Some('r') => '\r',
                        Some('t') => '\t',
                        Some(character) => {
                            return Err(Diagnostic::builtin(
                                BuiltinDiagnostic::InvalidEscape,
                                format!("unsupported string escape `\\{character}`"),
                                self.span(),
                            ));
                        }
                        None => return Err(Self::unterminated_string(span)),
                    };
                    self.advance();
                    value.push(escaped);
                }
                Some('\n') | None => return Err(Self::unterminated_string(span)),
                Some(character) => {
                    self.advance();
                    value.push(character);
                }
            }
        }
    }

    fn punctuation(&mut self) -> Result<Token> {
        let span = self.span();
        let character = self.peek().expect("punctuation begins at a character");
        if character == ':' && self.peek_next() == Some(':') {
            self.advance();
            self.advance();
            return Ok(Token {
                kind: TokenKind::DoubleColon,
                span,
            });
        }
        if character == '.' && self.peek_next() == Some('.') {
            self.advance();
            self.advance();
            return Ok(Token {
                kind: TokenKind::DotDot,
                span,
            });
        }
        let kind = match character {
            '$' => TokenKind::Dollar,
            '@' => TokenKind::At,
            '(' => TokenKind::LeftParen,
            ')' => TokenKind::RightParen,
            '{' => TokenKind::LeftBrace,
            '}' => TokenKind::RightBrace,
            '[' => TokenKind::LeftBracket,
            ']' => TokenKind::RightBracket,
            '<' => TokenKind::LeftAngle,
            '>' => TokenKind::RightAngle,
            ':' => TokenKind::Colon,
            ',' => TokenKind::Comma,
            '=' => TokenKind::Equal,
            '+' => TokenKind::Plus,
            '-' => TokenKind::Minus,
            '*' => TokenKind::Star,
            '/' => TokenKind::Slash,
            '%' => TokenKind::Percent,
            _ => {
                return Err(Diagnostic::builtin(
                    BuiltinDiagnostic::InvalidToken,
                    format!("unexpected character `{character}`"),
                    span,
                ));
            }
        };
        self.advance();
        Ok(Token { kind, span })
    }

    fn skip_comment(&mut self) {
        while self.peek().is_some_and(|character| character != '\n') {
            self.advance();
        }
    }

    fn unterminated_string(span: SourceSpan) -> Diagnostic {
        Diagnostic::builtin(
            BuiltinDiagnostic::UnterminatedString,
            "string literal is missing its closing quote",
            span,
        )
    }

    fn span(&self) -> SourceSpan {
        SourceSpan::at(self.source.clone(), self.line, self.column)
    }

    fn peek(&self) -> Option<char> {
        self.source.text()[self.offset..].chars().next()
    }

    fn peek_next(&self) -> Option<char> {
        self.source.text()[self.offset..].chars().nth(1)
    }

    fn advance(&mut self) {
        let character = self.peek().expect("advance requires a character");
        self.offset += character.len_utf8();
        if character == '\n' {
            self.line += 1;
            self.column = 1;
        } else {
            self.column += 1;
        }
    }
}

fn is_identifier_start(character: char) -> bool {
    character == '_' || character.is_ascii_alphabetic()
}

fn is_identifier_continue(character: char) -> bool {
    matches!(character, '_' | '-') || character.is_ascii_alphanumeric()
}

#[cfg(test)]
mod tests {
    use super::*;

    fn kinds(source: &str) -> Vec<TokenKind> {
        lex(SourceFile::new("test.clipasm", source))
            .expect("valid source")
            .into_iter()
            .map(|token| token.kind)
            .collect()
    }

    #[test]
    fn lexes_native_surface_without_program_knowledge() {
        assert_eq!(
            kinds("clipasm 1\n@visible clip<Audio> {\n  trim(300ms..800ms) as result # note\n}\n"),
            vec![
                TokenKind::Identifier("clipasm".to_owned()),
                TokenKind::Number("1".to_owned()),
                TokenKind::Newline,
                TokenKind::At,
                TokenKind::Identifier("visible".to_owned()),
                TokenKind::Identifier("clip".to_owned()),
                TokenKind::LeftAngle,
                TokenKind::Identifier("Audio".to_owned()),
                TokenKind::RightAngle,
                TokenKind::LeftBrace,
                TokenKind::Newline,
                TokenKind::Identifier("trim".to_owned()),
                TokenKind::LeftParen,
                TokenKind::Number("300".to_owned()),
                TokenKind::Identifier("ms".to_owned()),
                TokenKind::DotDot,
                TokenKind::Number("800".to_owned()),
                TokenKind::Identifier("ms".to_owned()),
                TokenKind::RightParen,
                TokenKind::Identifier("as".to_owned()),
                TokenKind::Identifier("result".to_owned()),
                TokenKind::Newline,
                TokenKind::RightBrace,
                TokenKind::Newline,
                TokenKind::End,
            ]
        );

        assert_eq!(
            kinds("clipasm 1\nmy-program as output-name\n"),
            vec![
                TokenKind::Identifier("clipasm".to_owned()),
                TokenKind::Number("1".to_owned()),
                TokenKind::Newline,
                TokenKind::Identifier("my-program".to_owned()),
                TokenKind::Identifier("as".to_owned()),
                TokenKind::Identifier("output-name".to_owned()),
                TokenKind::Newline,
                TokenKind::End,
            ]
        );
    }

    #[test]
    fn decodes_strings_and_tracks_locations() {
        let tokens = lex(SourceFile::new(
            "test.clipasm",
            "\nimage(\"a\\n\\\"b.png\")\n",
        ))
        .expect("valid source");
        assert_eq!(tokens[1].span.line, 2);
        assert_eq!(tokens[1].span.column, 1);
        assert_eq!(tokens[3].kind, TokenKind::String("a\n\"b.png".to_owned()));
    }

    #[test]
    fn lexes_decimal_arithmetic_units_and_repeated_postfixes() {
        assert_eq!(
            kinds("(1.25 + $offset)ms / 2%%"),
            vec![
                TokenKind::LeftParen,
                TokenKind::Number("1.25".to_owned()),
                TokenKind::Plus,
                TokenKind::Dollar,
                TokenKind::Identifier("offset".to_owned()),
                TokenKind::RightParen,
                TokenKind::Identifier("ms".to_owned()),
                TokenKind::Slash,
                TokenKind::Number("2".to_owned()),
                TokenKind::Percent,
                TokenKind::Percent,
                TokenKind::End,
            ]
        );

        assert_eq!(
            kinds("$duration- 100ms"),
            vec![
                TokenKind::Dollar,
                TokenKind::Identifier("duration-".to_owned()),
                TokenKind::Number("100".to_owned()),
                TokenKind::Identifier("ms".to_owned()),
                TokenKind::End,
            ]
        );
    }

    #[test]
    fn rejects_unknown_characters_and_unterminated_strings() {
        let invalid =
            lex(SourceFile::new("test.clipasm", "image(!)")).expect_err("invalid character");
        assert_eq!(invalid.code, "E_INVALID_TOKEN");
        assert_eq!(invalid.span.column, 7);

        let unterminated = lex(SourceFile::new("test.clipasm", "image(\"missing)"))
            .expect_err("unterminated string");
        assert_eq!(unterminated.code, "E_UNTERMINATED_STRING");
        assert_eq!(unterminated.span.column, 7);
    }
}