scon-core 0.1.0

Strict, deterministic configuration parsing and Serde support for SCON
Documentation
use crate::ast::Span;

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct SourcePosition {
    pub line: usize,
    pub character: usize,
    pub byte: usize,
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct SourceRange {
    pub start: SourcePosition,
    pub end: SourcePosition,
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Utf16Position {
    pub line: usize,
    pub character: usize,
}

#[derive(Clone, Debug)]
pub struct LineIndex {
    line_starts: Vec<usize>,
}

impl LineIndex {
    pub fn new(source: &str) -> Self {
        let mut line_starts = vec![0];
        for (index, byte) in source.bytes().enumerate() {
            if byte == b'\n' {
                line_starts.push(index + 1);
            }
        }
        Self { line_starts }
    }

    pub fn source_position(&self, source: &str, byte: usize) -> SourcePosition {
        let byte = byte.min(source.len());
        let line = self.line_for_byte(byte);
        let line_start = self.line_starts[line];
        SourcePosition {
            line,
            character: source[line_start..byte].chars().count(),
            byte,
        }
    }

    pub fn byte_for_line_character(
        &self,
        source: &str,
        line: usize,
        character: usize,
    ) -> Option<usize> {
        let line_start = *self.line_starts.get(line)?;
        let line_end = self.line_end(source, line);
        let mut chars_seen = 0usize;
        for (offset, _) in source[line_start..line_end].char_indices() {
            if chars_seen == character {
                return Some(line_start + offset);
            }
            chars_seen += 1;
        }
        (chars_seen == character).then_some(line_end)
    }

    pub fn utf16_position(&self, source: &str, byte: usize) -> Utf16Position {
        let byte = byte.min(source.len());
        let line = self.line_for_byte(byte);
        let line_start = self.line_starts[line];
        Utf16Position {
            line,
            character: source[line_start..byte].chars().map(char::len_utf16).sum(),
        }
    }

    pub fn byte_for_utf16_position(
        &self,
        source: &str,
        line: usize,
        character: usize,
    ) -> Option<usize> {
        let line_start = *self.line_starts.get(line)?;
        let line_end = self.line_end(source, line);
        let mut units_seen = 0usize;
        for (offset, ch) in source[line_start..line_end].char_indices() {
            if units_seen == character {
                return Some(line_start + offset);
            }
            units_seen += ch.len_utf16();
            if units_seen > character {
                return None;
            }
        }
        (units_seen == character).then_some(line_end)
    }

    pub fn source_range(&self, source: &str, span: Span) -> SourceRange {
        SourceRange {
            start: self.source_position(source, span.start_byte),
            end: self.source_position(source, span.end_byte),
        }
    }

    fn line_for_byte(&self, byte: usize) -> usize {
        match self.line_starts.binary_search(&byte) {
            Ok(line) => line,
            Err(next) => next.saturating_sub(1),
        }
    }

    fn line_end(&self, source: &str, line: usize) -> usize {
        self.line_starts
            .get(line + 1)
            .map(|next_start| next_start.saturating_sub(1))
            .unwrap_or(source.len())
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum CommentKind {
    Hash,
    SlashSlash,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum TokenKind {
    Identifier,
    String,
    Number,
    Keyword,
    Punctuation,
    Comment(CommentKind),
    Whitespace,
    Newline,
    Unknown,
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Token {
    pub kind: TokenKind,
    pub span: Span,
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Comment {
    pub kind: CommentKind,
    pub text: String,
    pub range: SourceRange,
}

pub fn collect_tokens(source: &str) -> Vec<Token> {
    let bytes = source.as_bytes();
    let mut tokens = Vec::new();
    let mut pos = 0usize;
    while pos < bytes.len() {
        let start = pos;
        match bytes[pos] {
            b' ' | b'\t' => {
                pos += 1;
                while matches!(bytes.get(pos), Some(b' ' | b'\t')) {
                    pos += 1;
                }
                tokens.push(token(TokenKind::Whitespace, start, pos));
            }
            b'\n' => {
                pos += 1;
                tokens.push(token(TokenKind::Newline, start, pos));
            }
            b'\r' if bytes.get(pos + 1) == Some(&b'\n') => {
                pos += 2;
                tokens.push(token(TokenKind::Newline, start, pos));
            }
            b'#' => {
                pos = consume_comment(bytes, pos);
                tokens.push(token(TokenKind::Comment(CommentKind::Hash), start, pos));
            }
            b'/' if bytes.get(pos + 1) == Some(&b'/') => {
                pos = consume_comment(bytes, pos);
                tokens.push(token(
                    TokenKind::Comment(CommentKind::SlashSlash),
                    start,
                    pos,
                ));
            }
            b'"' => {
                pos = consume_string(bytes, pos);
                tokens.push(token(TokenKind::String, start, pos));
            }
            b'-' | b'0'..=b'9' => {
                pos += 1;
                while matches!(
                    bytes.get(pos),
                    Some(b'0'..=b'9' | b'.' | b'e' | b'E' | b'+' | b'-')
                ) {
                    pos += 1;
                }
                tokens.push(token(TokenKind::Number, start, pos));
            }
            ch if is_ident_start(ch) => {
                pos += 1;
                while bytes.get(pos).is_some_and(|ch| is_ident_continue(*ch)) {
                    pos += 1;
                }
                let text = &source[start..pos];
                let kind = if matches!(text, "include" | "true" | "false" | "null") {
                    TokenKind::Keyword
                } else {
                    TokenKind::Identifier
                };
                tokens.push(token(kind, start, pos));
            }
            b'{' | b'}' | b'[' | b']' | b'=' | b',' | b'.' | b'$' => {
                pos += 1;
                tokens.push(token(TokenKind::Punctuation, start, pos));
            }
            _ => {
                pos += source[pos..]
                    .chars()
                    .next()
                    .map(char::len_utf8)
                    .unwrap_or(1);
                tokens.push(token(TokenKind::Unknown, start, pos));
            }
        }
    }
    tokens
}

pub fn comments_from_tokens(
    source: &str,
    line_index: &LineIndex,
    tokens: &[Token],
) -> Vec<Comment> {
    tokens
        .iter()
        .filter_map(|token| {
            let TokenKind::Comment(kind) = token.kind else {
                return None;
            };
            Some(Comment {
                kind,
                text: source[token.span.start_byte..token.span.end_byte].to_string(),
                range: line_index.source_range(source, token.span),
            })
        })
        .collect()
}

fn token(kind: TokenKind, start: usize, end: usize) -> Token {
    Token {
        kind,
        span: Span::new(start, end),
    }
}

fn consume_comment(bytes: &[u8], mut pos: usize) -> usize {
    while bytes
        .get(pos)
        .is_some_and(|ch| !matches!(ch, b'\n' | b'\r'))
    {
        pos += 1;
    }
    pos
}

fn consume_string(bytes: &[u8], mut pos: usize) -> usize {
    pos += 1;
    let mut escaped = false;
    while let Some(byte) = bytes.get(pos).copied() {
        pos += 1;
        if escaped {
            escaped = false;
            continue;
        }
        match byte {
            b'\\' => escaped = true,
            b'"' => break,
            b'\n' | b'\r' => break,
            _ => {}
        }
    }
    pos
}

fn is_ident_start(ch: u8) -> bool {
    ch.is_ascii_alphabetic() || ch == b'_'
}

fn is_ident_continue(ch: u8) -> bool {
    ch.is_ascii_alphanumeric() || ch == b'_' || ch == b'-'
}