meta-language 1.0.0

A self-describing links-network core for lossless language representation
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

use std::collections::HashSet;

use crate::configuration::ParseConfiguration;
use crate::link_network::{LinkId, LinkMetadata, LinkNetwork, LinkType};
use crate::mixed_regions::detect_embedded_regions_in_tree;
use crate::{structured_text_parser, tree_sitter_adapter};

pub fn attach_embedded_regions(
    network: &mut LinkNetwork,
    document: LinkId,
    text: &str,
    language: &str,
    configuration: ParseConfiguration,
) {
    let policy = configuration.region_detection_policy();
    // `record_grammars` scans the network, so it runs once per region
    // language rather than once per region: a document with a fenced block
    // in every section would otherwise parse in quadratic time.
    let mut recorded = HashSet::new();
    for region in detect_embedded_regions_in_tree(network, document, text, language, policy) {
        let region_language = region.language().to_string();
        let language_link = network.insert_typed_point(&region_language, LinkType::Language, None);
        let region_link = network.insert_link(
            [document, language_link],
            LinkMetadata::new()
                .with_link_type(LinkType::Region)
                .with_named(true)
                .with_term(format!("{region_language} region"))
                .with_language(region_language)
                .with_span(region.span()),
        );
        let range = region.span().byte_range();
        // A region spanning the whole document in the document's own language
        // already has its grammar CST below the document; reparsing it would
        // duplicate every node.
        if region.language().eq_ignore_ascii_case(language)
            && range.start() == 0
            && range.end() == text.len()
        {
            continue;
        }
        let region_text = &text[range.start()..range.end()];
        if recorded.insert(language_link) {
            network.record_grammars(language_link, region.language());
        }
        if tree_sitter_adapter::parse_embedded_region_into(
            network,
            region_link,
            region_text,
            region.language(),
            region.span(),
            configuration,
        )
        .is_none()
        {
            let _ = structured_text_parser::parse_embedded_region_into(
                network,
                region_link,
                region_text,
                region.language(),
                region.span(),
                configuration,
            );
        }
    }
}