buildlang 1.15.0

BuildLang (buildc): an effects-oriented compiler with a verified C path, shader output, experimental backends, opt-in linear types, and re-checkable accountability + scientific-runtime receipts
Documentation
use super::super::document::Document;
use super::classify::{classify_identifier, next_non_ws, pending_after_identifier};

const COMMENT_TYPE: u32 = 7;
const STRING_TYPE: u32 = 8;
const NUMBER_TYPE: u32 = 9;
const OPERATOR_TYPE: u32 = 10;

#[derive(Debug, Clone, Copy)]
struct Token {
    line: u32,
    start: u32,
    length: u32,
    token_type: u32,
    modifiers: u32,
}

pub(super) fn scan_document(doc: &Document) -> Vec<u32> {
    let mut tokens = Vec::new();
    for line in 0..doc.line_count() {
        if let Some(text) = doc.line(line as u32) {
            scan_line(line as u32, text, &mut tokens);
        }
    }
    encode_tokens(&tokens)
}

fn scan_line(line_number: u32, line: &str, tokens: &mut Vec<Token>) {
    let mut index = 0;
    let mut pending = None;
    let mut previous_was_dot = false;
    while index < line.len() {
        let Some(ch) = line[index..].chars().next() else {
            break;
        };
        if ch.is_whitespace() {
            index += ch.len_utf8();
            continue;
        }
        if line[index..].starts_with("//") {
            push_token(
                line_number,
                line,
                index,
                line.len(),
                COMMENT_TYPE,
                0,
                tokens,
            );
            break;
        }
        if ch == '"' {
            let end = string_end(line, index);
            push_token(line_number, line, index, end, STRING_TYPE, 0, tokens);
            index = end;
            previous_was_dot = false;
            continue;
        }
        if ch.is_ascii_digit() {
            let end = consume_number(line, index);
            push_token(line_number, line, index, end, NUMBER_TYPE, 0, tokens);
            index = end;
            previous_was_dot = false;
            continue;
        }
        if is_identifier_start(ch) {
            let end = consume_identifier(line, index);
            let ident = &line[index..end];
            let macro_end = if next_non_ws(line, end) == Some((end, '!')) {
                end + 1
            } else {
                end
            };
            let (token_type, modifiers) =
                classify_identifier(ident, pending, previous_was_dot, line, macro_end);
            push_token(
                line_number,
                line,
                index,
                macro_end,
                token_type,
                modifiers,
                tokens,
            );
            pending = pending_after_identifier(ident);
            index = macro_end;
            previous_was_dot = false;
            continue;
        }
        if let Some(operator) = match_operator(&line[index..]) {
            let end = index + operator.len();
            push_token(line_number, line, index, end, OPERATOR_TYPE, 0, tokens);
            index = end;
            previous_was_dot = operator == ".";
            continue;
        }
        index += ch.len_utf8();
        previous_was_dot = false;
    }
}

fn push_token(
    line_number: u32,
    line: &str,
    start: usize,
    end: usize,
    token_type: u32,
    modifiers: u32,
    tokens: &mut Vec<Token>,
) {
    if end <= start {
        return;
    }
    tokens.push(Token {
        line: line_number,
        start: utf16_len(&line[..start]),
        length: utf16_len(&line[start..end]),
        token_type,
        modifiers,
    });
}

fn encode_tokens(tokens: &[Token]) -> Vec<u32> {
    let mut data = Vec::with_capacity(tokens.len() * 5);
    let mut previous_line = 0;
    let mut previous_start = 0;
    for token in tokens {
        let delta_line = token.line - previous_line;
        let delta_start = if delta_line == 0 {
            token.start - previous_start
        } else {
            token.start
        };
        data.extend([
            delta_line,
            delta_start,
            token.length,
            token.token_type,
            token.modifiers,
        ]);
        previous_line = token.line;
        previous_start = token.start;
    }
    data
}

fn string_end(line: &str, start: usize) -> usize {
    let mut escaped = false;
    for (offset, ch) in line[start + 1..].char_indices() {
        if escaped {
            escaped = false;
        } else if ch == '\\' {
            escaped = true;
        } else if ch == '"' {
            return start + 1 + offset + ch.len_utf8();
        }
    }
    line.len()
}

fn consume_number(line: &str, start: usize) -> usize {
    consume_while(line, start, |ch| ch.is_ascii_digit() || ch == '.')
}

fn consume_identifier(line: &str, start: usize) -> usize {
    consume_while(line, start, is_identifier_continue)
}

fn consume_while(line: &str, start: usize, predicate: fn(char) -> bool) -> usize {
    let mut end = start;
    for ch in line[start..].chars() {
        if !predicate(ch) {
            break;
        }
        end += ch.len_utf8();
    }
    end
}

fn match_operator(rest: &str) -> Option<&'static str> {
    ["==", "!=", "<=", ">=", "&&", "||", "->", "=>", "::"]
        .into_iter()
        .chain([
            "+", "-", "*", "/", "%", "=", "<", ">", "!", ".", ":", ";", ",", "|", "&",
        ])
        .find(|operator| rest.starts_with(operator))
}

fn is_identifier_start(ch: char) -> bool {
    ch == '_' || ch.is_ascii_alphabetic()
}

fn is_identifier_continue(ch: char) -> bool {
    is_identifier_start(ch) || ch.is_ascii_digit()
}

fn utf16_len(text: &str) -> u32 {
    text.chars().map(|ch| ch.len_utf16() as u32).sum()
}