meta-language 1.0.0

A self-describing links-network core for lossless language representation
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::sync::Arc;

use crate::{
    ByteRange, Link, LinkId, LinkMetadata, LinkNetwork, LinkType, ParseConfiguration,
    tree_sitter_adapter,
};

impl LinkNetwork {
    /// Applies a byte-range source edit and reparses the network.
    ///
    /// Tree-sitter-backed languages use tree-sitter's incremental parse path;
    /// other languages fall back to the built-in lossless parser. Links whose
    /// source spans are outside the replaced byte range keep their identifiers.
    pub fn apply_edit(&mut self, range: ByteRange, replacement: &str) -> bool {
        self.apply_edit_with_configuration(range, replacement, ParseConfiguration::default())
    }

    /// Applies a byte-range source edit using an explicit parse configuration.
    ///
    /// Returns `false` when the range is outside the reconstructed source text,
    /// splits a UTF-8 code point, or the network has no document language.
    pub fn apply_edit_with_configuration(
        &mut self,
        range: ByteRange,
        replacement: &str,
        configuration: ParseConfiguration,
    ) -> bool {
        let old_text = self.reconstruct_text();
        let Some(edited_text) = apply_text_edit(&old_text, range, replacement) else {
            return false;
        };
        let Some(language) = document_language(self).map(ToOwned::to_owned) else {
            return false;
        };

        let reparsed = tree_sitter_adapter::parse_incremental(
            &old_text,
            range,
            replacement,
            &language,
            configuration,
        )
        .unwrap_or_else(|| Self::parse(&edited_text, &language, configuration));

        let edit = AppliedEdit::new(range, replacement.len());
        *self = remap_reparsed_network(self, &reparsed, edit);
        true
    }
}

fn document_language(network: &LinkNetwork) -> Option<&str> {
    network
        .links()
        .find(|link| link.metadata().link_type() == Some(LinkType::Document))
        .and_then(|link| link.metadata().language())
}

fn remap_reparsed_network(
    old: &LinkNetwork,
    reparsed: &LinkNetwork,
    edit: AppliedEdit,
) -> LinkNetwork {
    let mut id_map = stable_id_map(old, reparsed, edit);
    let mut used_targets = id_map.values().copied().collect::<BTreeSet<_>>();
    let mut next_id = old.next_id.max(reparsed.next_id);

    for link in reparsed.links() {
        if id_map.contains_key(&link.id()) {
            continue;
        }

        let target = if used_targets.contains(&link.id()) {
            let fresh = next_unused_id(&mut next_id, &used_targets);
            used_targets.insert(fresh);
            fresh
        } else {
            used_targets.insert(link.id());
            link.id()
        };
        id_map.insert(link.id(), target);
    }
    keep_tied_sibling_order(reparsed, &mut id_map, &mut used_targets, &mut next_id);

    let mut links = BTreeMap::new();
    for link in reparsed.links() {
        let id = id_map[&link.id()];
        let references = link
            .references()
            .iter()
            .map(|reference| id_map[reference])
            .collect::<Vec<_>>();
        let candidate = Link {
            id,
            references: Arc::from(references),
            metadata: link.metadata().clone(),
        };

        if let Some(shared) = old.links.get(&id)
            && shared.as_ref() == &candidate
        {
            links.insert(id, Arc::clone(shared));
            continue;
        }

        links.insert(id, Arc::new(candidate));
    }

    let terms = reparsed
        .terms
        .iter()
        .filter_map(|(term, id)| id_map.get(id).map(|mapped| (Arc::clone(term), *mapped)))
        .collect();
    let next_id = used_targets
        .iter()
        .map(|id| id.as_u64())
        .max()
        .map_or(1, |id| id + 1);

    LinkNetwork {
        next_id,
        links,
        terms,
        concept_syntax: reparsed.concept_syntax.clone(),
        strings: reparsed.strings.clone(),
    }
}

fn stable_id_map(
    old: &LinkNetwork,
    reparsed: &LinkNetwork,
    edit: AppliedEdit,
) -> BTreeMap<LinkId, LinkId> {
    let mut old_links = OldLinkIndex::new(old, edit);
    let mut mapped = BTreeMap::new();

    for new_link in reparsed.links() {
        let key = MatchKey::of(new_link.metadata());
        let matched = match new_link.metadata().span() {
            Some(span) => old_links.take_spanned(key, span.byte_range()),
            None => old_links.take_unspanned(key, new_link),
        };
        if let Some(old_id) = matched {
            mapped.insert(new_link.id(), old_id);
        }
    }

    // A link without a span is recognized by the identifiers it references, so
    // it can only be matched once everything it references has been. Repeat
    // until a round maps nothing new.
    let mut pending = reparsed
        .links()
        .filter(|link| link.metadata().span().is_none() && !mapped.contains_key(&link.id()))
        .collect::<Vec<_>>();
    while !pending.is_empty() {
        let remaining = pending.len();
        let mut unmatched = Vec::with_capacity(remaining);
        for new_link in pending {
            let matched = new_link
                .references()
                .iter()
                .map(|reference| mapped.get(reference).copied())
                .collect::<Option<Vec<_>>>()
                .and_then(|references| {
                    old_links.take_referencing(MatchKey::of(new_link.metadata()), references)
                });
            match matched {
                Some(old_id) => {
                    mapped.insert(new_link.id(), old_id);
                }
                None => unmatched.push(new_link),
            }
        }
        if unmatched.len() == remaining {
            break;
        }
        pending = unmatched;
    }

    mapped
}

/// Metadata two links must share to keep one identifier, shaped as a map key so
/// that candidates can be looked up instead of searched for.
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
struct MatchKey<'a> {
    link_type: Option<LinkType>,
    named: bool,
    term: Option<&'a str>,
    definition: Option<&'a str>,
    language: Option<&'a str>,
    /// Parse flags spelled out as booleans, because `LinkFlags` carries no
    /// ordering of its own.
    flags: (bool, bool, bool, bool),
}

impl<'a> MatchKey<'a> {
    fn of(metadata: &'a LinkMetadata) -> Self {
        let flags = metadata.flags();
        Self {
            link_type: metadata.link_type(),
            named: metadata.is_named(),
            term: metadata.term(),
            definition: metadata.definition(),
            language: metadata.language(),
            flags: (
                flags.is_error(),
                flags.has_error(),
                flags.is_missing(),
                flags.is_extra(),
            ),
        }
    }
}

/// Identifiers of the previous network's links, grouped by everything a match
/// requires.
///
/// Reparsing produces about as many links as the previous parse held, so
/// searching every old link for every new one costs `O(old * new)` and made a
/// 15 KB edit take twenty seconds (issue #193). Every group keeps its links in
/// ascending identifier order and hands them out from the front, which is the
/// link a search over `old.links()` would have stopped at first.
struct OldLinkIndex<'a> {
    /// Links carrying a span, keyed by the span the edit moves them to.
    spanned: BTreeMap<(MatchKey<'a>, usize, usize), VecDeque<LinkId>>,
    /// Links carrying no span, keyed by the identifiers they reference.
    referencing: BTreeMap<(MatchKey<'a>, Vec<LinkId>), VecDeque<LinkId>>,
    /// Links carrying no span that reference themselves and nothing else.
    self_referencing: BTreeMap<MatchKey<'a>, VecDeque<LinkId>>,
    /// Links already handed out, which every group skips.
    used: BTreeSet<LinkId>,
}

impl<'a> OldLinkIndex<'a> {
    fn new(old: &'a LinkNetwork, edit: AppliedEdit) -> Self {
        let mut index = Self {
            spanned: BTreeMap::new(),
            referencing: BTreeMap::new(),
            self_referencing: BTreeMap::new(),
            used: BTreeSet::new(),
        };

        for link in old.links() {
            let key = MatchKey::of(link.metadata());
            if let Some(span) = link.metadata().span() {
                // Links the edit overlaps cannot keep their identifier.
                if let Some(range) = edit.adjusted_range(span.byte_range()) {
                    index
                        .spanned
                        .entry((key, range.start(), range.end()))
                        .or_default()
                        .push_back(link.id());
                }
                continue;
            }

            index
                .referencing
                .entry((key, link.references().to_vec()))
                .or_default()
                .push_back(link.id());
            if is_self_reference(link) {
                index
                    .self_referencing
                    .entry(key)
                    .or_default()
                    .push_back(link.id());
            }
        }

        index
    }

    /// Identifier of an unused old link whose shifted span is `range`.
    fn take_spanned(&mut self, key: MatchKey<'a>, range: ByteRange) -> Option<LinkId> {
        let matched = Self::front(
            self.spanned.get_mut(&(key, range.start(), range.end())),
            &self.used,
        )?;
        self.used.insert(matched);
        Some(matched)
    }

    /// Identifier of an unused old link that carries no span and either
    /// references exactly what `new_link` references or, when `new_link`
    /// references only itself, references only itself as well.
    fn take_unspanned(&mut self, key: MatchKey<'a>, new_link: &Link) -> Option<LinkId> {
        let by_references = Self::front(
            self.referencing
                .get_mut(&(key, new_link.references().to_vec())),
            &self.used,
        );
        let by_self_reference = if is_self_reference(new_link) {
            Self::front(self.self_referencing.get_mut(&key), &self.used)
        } else {
            None
        };

        // A search over `old.links()` would have stopped at whichever of the
        // two candidates was inserted first, which is the smaller identifier.
        let matched = by_references.into_iter().chain(by_self_reference).min()?;
        self.used.insert(matched);
        Some(matched)
    }

    /// Identifier of an unused old link that carries no span and references
    /// exactly `references`.
    fn take_referencing(&mut self, key: MatchKey<'a>, references: Vec<LinkId>) -> Option<LinkId> {
        let matched = Self::front(self.referencing.get_mut(&(key, references)), &self.used)?;
        self.used.insert(matched);
        Some(matched)
    }

    /// First unused identifier in `group`, dropping the used ones it skips.
    fn front(group: Option<&mut VecDeque<LinkId>>, used: &BTreeSet<LinkId>) -> Option<LinkId> {
        let group = group?;
        while group.front().is_some_and(|id| used.contains(id)) {
            group.pop_front();
        }
        group.front().copied()
    }
}

fn apply_text_edit(old_text: &str, range: ByteRange, replacement: &str) -> Option<String> {
    if range.end() > old_text.len()
        || !old_text.is_char_boundary(range.start())
        || !old_text.is_char_boundary(range.end())
    {
        return None;
    }

    let mut edited =
        String::with_capacity(old_text.len() - (range.end() - range.start()) + replacement.len());
    edited.push_str(&old_text[..range.start()]);
    edited.push_str(replacement);
    edited.push_str(&old_text[range.end()..]);
    Some(edited)
}

/// Siblings that share a span keep their order by identifier (a zero-width
/// MISSING leaf before another at the same point), as the network orders
/// siblings by span, then identifier. Where the kept identifiers would reverse
/// two of them, the later one gets a fresh identifier.
fn keep_tied_sibling_order(
    reparsed: &LinkNetwork,
    id_map: &mut BTreeMap<LinkId, LinkId>,
    used_targets: &mut BTreeSet<LinkId>,
    next_id: &mut u64,
) {
    let mut ties: BTreeMap<(Vec<LinkId>, usize, usize), Vec<LinkId>> = BTreeMap::new();
    for link in reparsed.links() {
        if let Some(span) = link.metadata().span() {
            let range = span.byte_range();
            ties.entry((link.references().to_vec(), range.start(), range.end()))
                .or_default()
                .push(link.id());
        }
    }
    for group in ties.values().filter(|group| group.len() > 1) {
        let mut last: Option<LinkId> = None;
        for id in group {
            let target = id_map[id];
            if last.is_some_and(|last| target <= last) {
                let fresh = next_unused_id(next_id, used_targets);
                used_targets.insert(fresh);
                id_map.insert(*id, fresh);
                last = Some(fresh);
            } else {
                last = Some(target);
            }
        }
    }
}

fn next_unused_id(next_id: &mut u64, used_targets: &BTreeSet<LinkId>) -> LinkId {
    loop {
        let candidate = LinkId(*next_id);
        *next_id += 1;
        if !used_targets.contains(&candidate) {
            return candidate;
        }
    }
}

fn is_self_reference(link: &Link) -> bool {
    link.references() == [link.id()]
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct AppliedEdit {
    old_start: usize,
    old_end: usize,
    new_end: usize,
}

impl AppliedEdit {
    const fn new(range: ByteRange, replacement_len: usize) -> Self {
        Self {
            old_start: range.start(),
            old_end: range.end(),
            new_end: range.start() + replacement_len,
        }
    }

    const fn adjusted_range(self, range: ByteRange) -> Option<ByteRange> {
        if range.end() <= self.old_start {
            return Some(range);
        }
        if range.start() < self.old_end {
            return None;
        }

        let start = shift_byte(range.start(), self.old_end, self.new_end);
        let end = shift_byte(range.end(), self.old_end, self.new_end);
        Some(ByteRange::new(start, end))
    }
}

const fn shift_byte(byte: usize, old_end: usize, new_end: usize) -> usize {
    if new_end >= old_end {
        byte + (new_end - old_end)
    } else {
        byte - (old_end - new_end)
    }
}