df_ls_lexical_analysis 0.3.0-rc.1

use super::*;
use df_ls_diagnostics::DiagnosticsInfo;

/// Tokenize an individual token argument
/// The type of argument is determined in this function.
pub(crate) fn tokenize_token_argument(
    tok_help: &mut TokenizerHelper,
    regex_list: &RegexList,
    diagnostics: &mut DiagnosticsInfo,
    token_arguments_id: u64,
    allow_pipe_arguments: bool,
) -> TokenizerResult {
    // To speed up we only continue checking if we do not have an exact match.
    let mut exact_match_found = false;

    // Check largest first (pipe_arguments or string) and char
    // Pipe Arguments
    let (token_pipe_arguments, pipe_args_len) = if allow_pipe_arguments {
        match_token_pipe_arguments(tok_help, regex_list)?
    } else {
        // If pipe argument is not allowed, with a `len` of `0` it will never be the biggest.
        // And if all are `0` (which can not happen), an int will match the longest.
        (TokenMatchStatus::NoMatch, 0)
    };

    // String (if not `allow_pipe_arguments`)
    // If `allow_pipe_arguments` then we can skip this for later.
    let (token_string, string_len) = if !allow_pipe_arguments {
        match_token_string(tok_help, regex_list)?
    } else {
        (TokenMatchStatus::NoMatch, 0)
    };

    // And check char (when calculating exact match),
    // because it might be longer then String, for example in `'['`.
    //
    // Char
    //
    // Because chars can contain `[` is might match something larger then string.
    // Because of this we always check char if `'` is next char
    //
    // Extra check for speed improvement.
    // Char is not very common so only check if next is `'`.
    let (token_char, char_len) = if tok_help.check_if_next_char_match('\'') {
        // Then we are in a pipe argument, the character `'|'` and `'['` is not allowed.
        // See issue #116 and #117 for more info.
        if !allow_pipe_arguments
            && (tok_help.check_if_next_chars_match(&['\'', '|', '\''])
                || tok_help.check_if_next_chars_match(&['\'', '[', '\'']))
        {
            (TokenMatchStatus::NoMatch, 0)
        } else {
            match_token_char(tok_help, regex_list)?
        }
    } else {
        (TokenMatchStatus::NoMatch, 0)
    };

    // Get longest match and use that.
    let exact_match_len = *vec![string_len, pipe_args_len, char_len]
        .iter()
        .max()
        .unwrap();

    // Int
    //
    // Extra check for speed improvement.
    // Check if next character matches any of the given characters,
    // if not, don't run regex.
    let (token_int, int_len) = if tok_help
        .check_if_next_char_matches_any_of(&['-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'])
    {
        match_token_int(tok_help, regex_list)?
    } else {
        (TokenMatchStatus::NoMatch, 0)
    };
    if int_len == exact_match_len {
        exact_match_found = true;
    }

    // Arg N
    let (token_arg_n, arg_n_len) = if !exact_match_found {
        // Extra check for speed improvement.
        // Arg N is not common so only check if next starts with `ARG`.
        if tok_help.check_if_next_chars_match(&['A', 'R', 'G']) {
            match_token_arg_n(tok_help, regex_list)?
        } else {
            (TokenMatchStatus::NoMatch, 0)
        }
    } else {
        (TokenMatchStatus::NoMatch, 0)
    };
    if arg_n_len == exact_match_len {
        exact_match_found = true;
    }

    // Reference
    let (token_ref, ref_len) = if !exact_match_found {
        // Extra check for speed improvement.
        // This is only a sight improvement when we parse a lot of Strings.
        if tok_help.check_if_next_char_matches_any_of(&[
            '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
            'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
            'Y', 'Z', '!',
        ]) {
            match_token_reference(tok_help, regex_list)?
        } else {
            (TokenMatchStatus::NoMatch, 0)
        }
    } else {
        (TokenMatchStatus::NoMatch, 0)
    };
    if ref_len == exact_match_len {
        exact_match_found = true;
    }

    // String (if not above already)
    let (token_string, string_len) = if !exact_match_found && allow_pipe_arguments {
        match_token_string(tok_help, regex_list)?
    } else {
        // use values from above
        (token_string, string_len)
    };

    // Get longest match and use that.
    let longest_match = *vec![
        int_len,
        char_len,
        arg_n_len,
        ref_len,
        string_len,
        pipe_args_len,
    ]
    .iter()
    .max()
    .unwrap();

    // Extra check
    if longest_match > exact_match_len {
        log::debug!("Something matches that is longer then we expected. This can be an indication of a problem.");
        log::error!(
            "Indication of problem: `longest_match`({}) > `exact_match_len`({}).",
            longest_match,
            exact_match_len
        );
        panic!("Matched character length is different from the expected value.");
    }

    // Order here is important
    if int_len == longest_match {
        match token_int {
            TokenMatchStatus::Ok(result) => {
                // Move cursor directly
                tok_help.direct_mode_index_and_point(result.end_byte, result.end_point);
                tok_help.add_node_to_tree(result, token_arguments_id);
                return Ok(());
            }
            TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
            _ => {}
        }
    } else if char_len == longest_match {
        match token_char {
            TokenMatchStatus::Ok(result) => {
                // Move cursor directly
                tok_help.direct_mode_index_and_point(result.end_byte, result.end_point);
                tok_help.add_node_to_tree(result, token_arguments_id);
                return Ok(());
            }
            TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
            _ => {}
        }
    } else if arg_n_len == longest_match {
        match token_arg_n {
            TokenMatchStatus::Ok(result) => {
                // Move cursor directly
                tok_help.direct_mode_index_and_point(result.end_byte, result.end_point);
                tok_help.add_node_to_tree(result, token_arguments_id);
                return Ok(());
            }
            TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
            _ => {}
        }
    } else if ref_len == longest_match {
        match token_ref {
            TokenMatchStatus::Ok(result) => {
                // Move cursor directly
                tok_help.direct_mode_index_and_point(result.end_byte, result.end_point);
                tok_help.add_node_to_tree(result, token_arguments_id);
                return Ok(());
            }
            TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
            _ => {}
        }
    } else if string_len == longest_match {
        match token_string {
            TokenMatchStatus::Ok(result) => {
                // Check if the String contains a BangArgN argument.
                // If it does the String is converted to a BangArgN token.
                if contains_bang_arg_n(tok_help, regex_list, &result) {
                    // Contains `!ARGn` argument, so create different token.
                    create_token_bang_arg_n_sequence(
                        tok_help,
                        regex_list,
                        result,
                        token_arguments_id,
                    )?;
                    return Ok(());
                } else {
                    // Move cursor directly
                    tok_help.direct_mode_index_and_point(result.end_byte, result.end_point);
                    tok_help.add_node_to_tree(result, token_arguments_id);
                    return Ok(());
                }
            }
            TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
            _ => {}
        }
    } else if pipe_args_len == longest_match {
        match token_pipe_arguments {
            TokenMatchStatus::Ok(result) => {
                if !allow_pipe_arguments {
                    panic!("Pipe argument was matched, but is not allowed.");
                }
                // We want to parse the pipe argument further.
                let pipe_args_result = token_pipe_arguments::tokenize_token_argument_pipe_arguments(
                    tok_help,
                    regex_list,
                    diagnostics,
                    token_arguments_id,
                );
                // Check if the cursor is in the expected position.
                if tok_help.get_point() != result.end_point {
                    log::error!(
                        "Points do not match {:#?} and {:#?}",
                        tok_help.get_point(),
                        result.end_point
                    );
                    panic!("Something went wrong in the parsing of the pipe arguments.");
                }
                return pipe_args_result;
            }
            TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
            _ => {}
        }
    } else {
        unreachable!("No token value was the longest");
    }

    // `OkWithPrefixFound` can never happen here because
    // all `get_next_match` here should have `optional` set to true.
    unreachable!(
        "No token value was found, but also no EOF, \
        so `[`, `]`, `:`, `\\r` or `\\n` was found. This should have been caught earlier."
    );
}

fn match_token_int(
    tok_help: &mut TokenizerHelper,
    regex_list: &RegexList,
) -> Result<(TokenMatchStatus, usize), TokenizerEnd> {
    // Test if `token_argument_integer` (Does not move cursor)
    let token_int = tok_help.get_next_match(
        &regex_list.token_argument_integer,
        "token_argument_integer",
        Some("token_argument_integer"),
        true,
        false,
    );
    let mut int_len = 0;
    match &token_int {
        TokenMatchStatus::Ok(result) => int_len = result.end_byte - result.start_byte,
        TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
        _ => {}
    }
    Ok((token_int, int_len))
}

fn match_token_char(
    tok_help: &mut TokenizerHelper,
    regex_list: &RegexList,
) -> Result<(TokenMatchStatus, usize), TokenizerEnd> {
    // Test if `token_argument_character` (Does not move cursor)
    let token_char = tok_help.get_next_match(
        &regex_list.token_argument_character,
        "token_argument_character",
        Some("token_argument_character"),
        true,
        false,
    );
    let mut char_len = 0;
    match &token_char {
        TokenMatchStatus::Ok(result) => char_len = result.end_byte - result.start_byte,
        TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
        _ => {}
    }
    Ok((token_char, char_len))
}

fn match_token_arg_n(
    tok_help: &mut TokenizerHelper,
    regex_list: &RegexList,
) -> Result<(TokenMatchStatus, usize), TokenizerEnd> {
    // Test if `token_argument_arg_n` (Does not move cursor)
    let token_arg_n = tok_help.get_next_match(
        &regex_list.token_argument_arg_n,
        "token_argument_arg_n",
        Some("token_argument_arg_n"),
        true,
        false,
    );
    let mut arg_n_len = 0;
    match &token_arg_n {
        TokenMatchStatus::Ok(result) => arg_n_len = result.end_byte - result.start_byte,
        TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
        _ => {}
    }
    Ok((token_arg_n, arg_n_len))
}

fn match_token_reference(
    tok_help: &mut TokenizerHelper,
    regex_list: &RegexList,
) -> Result<(TokenMatchStatus, usize), TokenizerEnd> {
    // Test if `token_argument_reference` (Does not move cursor)
    let token_ref = tok_help.get_next_match(
        &regex_list.token_argument_reference,
        "token_argument_reference",
        Some("token_argument_reference"),
        true,
        false,
    );
    let mut ref_len = 0;
    match &token_ref {
        TokenMatchStatus::Ok(result) => ref_len = result.end_byte - result.start_byte,
        TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
        _ => {}
    }
    Ok((token_ref, ref_len))
}

fn match_token_string(
    tok_help: &mut TokenizerHelper,
    regex_list: &RegexList,
) -> Result<(TokenMatchStatus, usize), TokenizerEnd> {
    // Test if `token_argument_string` (Does not move cursor)
    let token_string = tok_help.get_next_match(
        &regex_list.token_argument_string,
        "token_argument_string",
        Some("token_argument_string"),
        true,
        false,
    );
    let mut string_len = 0;
    match &token_string {
        TokenMatchStatus::Ok(result) => string_len = result.end_byte - result.start_byte,
        TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
        _ => {}
    }
    Ok((token_string, string_len))
}

fn match_token_pipe_arguments(
    tok_help: &mut TokenizerHelper,
    regex_list: &RegexList,
) -> Result<(TokenMatchStatus, usize), TokenizerEnd> {
    // Test if `token_argument_pipe_arguments` (Does not move cursor)
    let token_pipe_arguments = tok_help.get_next_match(
        &regex_list.token_argument_pipe_arguments,
        "token_argument_pipe_arguments",
        Some("token_argument_pipe_arguments"),
        true,
        false,
    );
    let mut pipe_args_len = 0;
    match &token_pipe_arguments {
        TokenMatchStatus::Ok(result) => pipe_args_len = result.end_byte - result.start_byte,
        TokenMatchStatus::EoF => return Err(TokenizerEnd::UnexpectedEoF),
        _ => {}
    }
    Ok((token_pipe_arguments, pipe_args_len))
}

/// This function is different from the others because it only checks if it contains a
/// BangArgN variable.
fn contains_bang_arg_n(
    tok_help: &mut TokenizerHelper,
    regex_list: &RegexList,
    argument_data_node: &DataNode,
) -> bool {
    // Get text of token
    let text = tok_help.data_node_text(argument_data_node);

    regex_list.token_argument_bang_arg_n.find(&text).is_some()
}

/// Create a special type of sequence for value that contains BangArgN variables.
fn create_token_bang_arg_n_sequence(
    tok_help: &mut TokenizerHelper,
    regex_list: &RegexList,
    mut argument_data_node: DataNode,
    token_arguments_id: u64,
) -> Result<bool, TokenizerEnd> {
    // Change current DataNode to BangArgN sequence
    argument_data_node.kind = "token_argument_bang_arg_n_sequence".to_owned();
    argument_data_node.name = Some("token_argument_bang_arg_n_sequence".to_owned());
    // Get text of token
    let text = tok_help.data_node_text(&argument_data_node);
    let argument_start_byte = argument_data_node.start_byte;
    let argument_end_byte = argument_data_node.end_byte;
    let argument_data_node_id = argument_data_node.id;
    // Add `token_argument_bang_arg_n_sequence` to tree.
    tok_help.add_node_to_tree(argument_data_node, token_arguments_id);

    // Separate text into parts.
    let mut last_parsed_byte = argument_start_byte;
    for bang_arg_n in regex_list.token_argument_bang_arg_n.find_iter(&text) {
        // Check if prefix before `!ARGn`
        if argument_start_byte + bang_arg_n.start() > last_parsed_byte {
            // Mark part before it as original type
            let new_last_parsed_byte = argument_start_byte + bang_arg_n.start();
            let start_point = tok_help.get_point();
            // Move cursor
            let end_point = tok_help.move_index(new_last_parsed_byte);
            let data_node = tok_help.create_tsnode(
                last_parsed_byte,
                start_point,
                new_last_parsed_byte,
                end_point,
                "token_argument_string",
                Some("token_argument_string"),
            );
            // Add node String to tree.
            tok_help.add_node_to_tree(data_node, argument_data_node_id);
            // Update `last_parsed_byte`
            last_parsed_byte = new_last_parsed_byte;
        }
        // Add `!ARGn`
        let new_last_parsed_byte = argument_start_byte + bang_arg_n.end();
        let start_point = tok_help.get_point();
        // Move cursor
        let end_point = tok_help.move_index(new_last_parsed_byte);
        let data_node = tok_help.create_tsnode(
            last_parsed_byte,
            start_point,
            new_last_parsed_byte,
            end_point,
            "token_argument_bang_arg_n",
            Some("token_argument_bang_arg_n"),
        );
        // Add `token_argument_bang_arg_n` to tree.
        tok_help.add_node_to_tree(data_node, argument_data_node_id);
        // Update `last_parsed_byte`
        last_parsed_byte = new_last_parsed_byte;
    }

    if last_parsed_byte < argument_end_byte {
        // Parse everything after last `!ARGn`
        let new_last_parsed_byte = argument_end_byte;
        let start_point = tok_help.get_point();
        // Move cursor
        let end_point = tok_help.move_index(new_last_parsed_byte);
        let data_node = tok_help.create_tsnode(
            last_parsed_byte,
            start_point,
            new_last_parsed_byte,
            end_point,
            "token_argument_string",
            Some("token_argument_string"),
        );
        // Add node String to tree.
        tok_help.add_node_to_tree(data_node, argument_data_node_id);
        // Update `last_parsed_byte`
        last_parsed_byte = new_last_parsed_byte;
    }

    if last_parsed_byte > argument_end_byte {
        panic!("More bytes parsed as part of BangArgNSequence then part of Argument");
    }

    Ok(true)
}