1use std::borrow::Cow;
4use std::collections::{BTreeMap, BTreeSet};
5use std::fmt;
6use std::ops::Range;
7use std::str::FromStr as _;
8use std::sync::{
9 Arc, OnceLock, RwLock,
10 atomic::{AtomicU64, Ordering},
11};
12use std::time::{Duration, Instant};
13
14use rhai::{CustomType, Dynamic, Engine, FuncRegistration, ImmutableString, TypeBuilder};
15use similar::{Algorithm, DiffOp, capture_diff_slices_deadline};
16use syntect::easy::ScopeRangeIterator;
17use syntect::highlighting::ScopeSelectors;
18use syntect::parsing::{ParseState, ScopeStack, SyntaxDefinition, SyntaxSet};
19use thiserror::Error;
20use unicode_segmentation::UnicodeSegmentation as _;
21
22use crate::ComponentInstancePath;
23
24pub const DEFAULT_DOCUMENT_MAX_BYTES: usize = 10 * 1024 * 1024;
25pub const DEFAULT_DOCUMENT_MAX_LINES: usize = 500_000;
26pub const DEFAULT_DIFF_MAX_HUNKS: usize = 100_000;
27pub const DEFAULT_DIFF_TIMEOUT: Duration = Duration::from_secs(2);
28const MAX_INLINE_DIFF_BYTES: usize = 256 * 1024;
29
30#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
31pub struct DocumentLimits {
32 pub max_document_bytes: usize,
33 pub max_document_lines: usize,
34 pub max_diff_total_bytes: usize,
35 pub max_diff_hunks: usize,
36 pub diff_timeout: Duration,
37 pub max_markdown_table_columns: usize,
39}
40
41impl Default for DocumentLimits {
42 fn default() -> Self {
43 Self {
44 max_document_bytes: DEFAULT_DOCUMENT_MAX_BYTES,
45 max_document_lines: DEFAULT_DOCUMENT_MAX_LINES,
46 max_diff_total_bytes: DEFAULT_DOCUMENT_MAX_BYTES * 2,
47 max_diff_hunks: DEFAULT_DIFF_MAX_HUNKS,
48 diff_timeout: DEFAULT_DIFF_TIMEOUT,
49 max_markdown_table_columns: 128,
50 }
51 }
52}
53
54impl DocumentLimits {
55 pub fn validate(self) -> Result<(), DocumentError> {
61 if self.max_document_bytes == 0
62 || self.max_document_lines == 0
63 || self.max_diff_total_bytes == 0
64 || self.max_diff_hunks == 0
65 || self.diff_timeout.is_zero()
66 || self.max_markdown_table_columns == 0
67 {
68 Err(DocumentError::InvalidLimits)
69 } else {
70 Ok(())
71 }
72 }
73}
74
75#[derive(Clone)]
76pub struct DocumentRuntimeConfig {
77 limits: Arc<RwLock<DocumentLimits>>,
78 markdown_cache: Arc<RwLock<Option<Arc<dyn crate::MarkdownPreparationCache>>>>,
79 markdown_preparations: Arc<MarkdownPreparationCounters>,
80}
81
82#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
86pub struct MarkdownPreparationStats {
87 pub jobs_started: u64,
88 pub jobs_succeeded: u64,
89 pub jobs_failed: u64,
90 pub cache_hits: u64,
91}
92
93#[derive(Default)]
94struct MarkdownPreparationCounters {
95 started: AtomicU64,
96 succeeded: AtomicU64,
97 failed: AtomicU64,
98 hits: AtomicU64,
99}
100
101impl fmt::Debug for DocumentRuntimeConfig {
102 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
103 formatter
104 .debug_struct("DocumentRuntimeConfig")
105 .field("limits", &self.limits())
106 .field("markdown_cache_enabled", &self.markdown_cache().is_some())
107 .finish()
108 }
109}
110
111impl Default for DocumentRuntimeConfig {
112 fn default() -> Self {
113 Self::new()
114 }
115}
116
117impl DocumentRuntimeConfig {
118 #[must_use]
119 pub fn new() -> Self {
120 Self {
121 limits: Arc::new(RwLock::new(DocumentLimits::default())),
122 markdown_cache: Arc::new(RwLock::new(None)),
123 markdown_preparations: Arc::default(),
124 }
125 }
126
127 #[must_use]
128 pub fn limits(&self) -> DocumentLimits {
129 self.limits
130 .read()
131 .map_or_else(|poisoned| *poisoned.into_inner(), |limits| *limits)
132 }
133
134 pub fn set_limits(&self, limits: DocumentLimits) -> Result<(), DocumentError> {
140 limits.validate()?;
141 *self
142 .limits
143 .write()
144 .map_err(|_| DocumentError::DocumentConfigPoisoned)? = limits;
145 Ok(())
146 }
147
148 #[must_use]
150 pub fn markdown_cache(&self) -> Option<Arc<dyn crate::MarkdownPreparationCache>> {
151 self.markdown_cache
152 .read()
153 .unwrap_or_else(std::sync::PoisonError::into_inner)
154 .clone()
155 }
156
157 pub fn set_markdown_cache(
166 &self,
167 cache: Option<Arc<dyn crate::MarkdownPreparationCache>>,
168 ) -> Result<(), DocumentError> {
169 *self
170 .markdown_cache
171 .write()
172 .map_err(|_| DocumentError::DocumentConfigPoisoned)? = cache;
173 Ok(())
174 }
175
176 #[must_use]
179 pub fn markdown_preparation_stats(&self) -> MarkdownPreparationStats {
180 let counters = &self.markdown_preparations;
181 MarkdownPreparationStats {
182 jobs_started: counters.started.load(Ordering::Relaxed),
183 jobs_succeeded: counters.succeeded.load(Ordering::Relaxed),
184 jobs_failed: counters.failed.load(Ordering::Relaxed),
185 cache_hits: counters.hits.load(Ordering::Relaxed),
186 }
187 }
188
189 pub(crate) fn markdown_cache_hit(&self) {
190 self.markdown_preparations
191 .hits
192 .fetch_add(1, Ordering::Relaxed);
193 }
194
195 pub(crate) fn prepare_markdown(
196 &self,
197 request: &crate::MarkdownPreparation,
198 previous: Option<&crate::PreparedMarkdown>,
199 canceled: &std::sync::atomic::AtomicBool,
200 alive: &std::sync::atomic::AtomicBool,
201 ) -> Result<crate::PreparedMarkdown, String> {
202 let counters = &self.markdown_preparations;
203 counters.started.fetch_add(1, Ordering::Relaxed);
204 let result = request.prepare_with_liveness(previous, canceled, alive);
205 if result.is_ok() {
206 &counters.succeeded
207 } else {
208 &counters.failed
209 }
210 .fetch_add(1, Ordering::Relaxed);
211 result
212 }
213}
214
215const RHAI_SYNTAX: &str = r#"%YAML 1.2
216---
217name: Rhai
218file_extensions: [rhai]
219scope: source.rhai
220contexts:
221 main:
222 - include: comments
223 - match: '\b(fn|let|const|if|else|switch|for|in|while|loop|do|until|break|continue|return|throw|try|catch|import|export|as|private)\b'
224 scope: keyword.control.rhai
225 - match: '\b(true|false)\b'
226 scope: constant.language.rhai
227 - match: '\b[0-9]+(?:\.[0-9]+)?\b'
228 scope: constant.numeric.rhai
229 - match: '"'
230 push: double-quoted-string
231 - match: "'"
232 push: single-quoted-string
233 - match: '`'
234 push: template-string
235 - match: '\b[A-Za-z_][A-Za-z0-9_]*(?=\s*\()'
236 scope: entity.name.function.rhai
237 - match: '[+\-*/%!=<>|&^~?:]+'
238 scope: keyword.operator.rhai
239 comments:
240 - match: '//.*$'
241 scope: comment.line.double-slash.rhai
242 - match: '/\*'
243 scope: punctuation.definition.comment.begin.rhai
244 push:
245 - meta_scope: comment.block.rhai
246 - match: '\*/'
247 scope: punctuation.definition.comment.end.rhai
248 pop: true
249 double-quoted-string:
250 - meta_scope: string.quoted.double.rhai
251 - match: '\\.'
252 scope: constant.character.escape.rhai
253 - match: '"'
254 pop: true
255 single-quoted-string:
256 - meta_scope: string.quoted.single.rhai
257 - match: '\\.'
258 scope: constant.character.escape.rhai
259 - match: "'"
260 pop: true
261 template-string:
262 - meta_scope: string.quoted.other.rhai
263 - match: '\\.'
264 scope: constant.character.escape.rhai
265 - match: '`'
266 pop: true
267"#;
268
269const TYPESCRIPT_SYNTAX: &str = r#"%YAML 1.2
270---
271name: TypeScript
272file_extensions: [ts, tsx]
273scope: source.ts
274contexts:
275 main:
276 - match: '//.*$'
277 scope: comment.line.double-slash.ts
278 - match: '/\*'
279 push:
280 - meta_scope: comment.block.ts
281 - match: '\*/'
282 pop: true
283 - match: '\b(async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|keyof|let|namespace|new|of|private|protected|public|readonly|return|set|static|super|switch|throw|try|type|typeof|var|void|while|with|yield)\b'
284 scope: keyword.control.ts
285 - match: '\b(any|bigint|boolean|never|number|object|string|symbol|unknown|undefined|null|true|false)\b'
286 scope: storage.type.ts
287 - match: '\b[0-9]+(?:\.[0-9]+)?\b'
288 scope: constant.numeric.ts
289 - match: '"'
290 push: double-string
291 - match: "'"
292 push: single-string
293 - match: '`'
294 push: template-string
295 - match: '\b[A-Za-z_$][A-Za-z0-9_$]*(?=\s*\()'
296 scope: entity.name.function.ts
297 - match: '</?[A-Za-z][A-Za-z0-9:.-]*'
298 scope: entity.name.tag.tsx
299 - match: '[+\-*/%!=<>|&^~?:]+'
300 scope: keyword.operator.ts
301 double-string:
302 - meta_scope: string.quoted.double.ts
303 - match: '\\.'
304 scope: constant.character.escape.ts
305 - match: '"'
306 pop: true
307 single-string:
308 - meta_scope: string.quoted.single.ts
309 - match: '\\.'
310 scope: constant.character.escape.ts
311 - match: "'"
312 pop: true
313 template-string:
314 - meta_scope: string.quoted.other.ts
315 - match: '\\.'
316 scope: constant.character.escape.ts
317 - match: '`'
318 pop: true
319"#;
320
321const JSONC_SYNTAX: &str = r#"%YAML 1.2
322---
323name: JSON with Comments
324file_extensions: [jsonc]
325scope: source.json.comments
326contexts:
327 main:
328 - match: '//.*$'
329 scope: comment.line.double-slash.json
330 - match: '/\*'
331 push:
332 - meta_scope: comment.block.json
333 - match: '\*/'
334 pop: true
335 - match: '"'
336 push: string
337 - match: '-?\b[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?\b'
338 scope: constant.numeric.json
339 - match: '\b(true|false|null)\b'
340 scope: constant.language.json
341 - match: '[{}\[\],:]'
342 scope: punctuation.separator.json
343 string:
344 - meta_scope: string.quoted.double.json
345 - match: '\\(?:["\\/bfnrt]|u[0-9A-Fa-f]{4})'
346 scope: constant.character.escape.json
347 - match: '"'
348 pop: true
349"#;
350
351const TOML_SYNTAX: &str = r#"%YAML 1.2
352---
353name: TOML
354file_extensions: [toml]
355scope: source.toml
356contexts:
357 main:
358 - match: '#.*$'
359 scope: comment.line.number-sign.toml
360 - match: '^\s*\[\[?[^\]]+\]\]?'
361 scope: entity.name.section.toml
362 - match: '^[A-Za-z0-9_.-]+(?=\s*=)'
363 scope: variable.other.key.toml
364 - match: '"""'
365 push: multiline-string
366 - match: "'''"
367 push: multiline-literal
368 - match: '"'
369 push: string
370 - match: "'"
371 push: literal
372 - match: '\b(true|false)\b'
373 scope: constant.language.toml
374 - match: '[+-]?\b(?:0x[0-9A-Fa-f_]+|0o[0-7_]+|0b[01_]+|[0-9][0-9_]*(?:\.[0-9_]+)?)\b'
375 scope: constant.numeric.toml
376 - match: '[=,.{}\[\]]'
377 scope: punctuation.separator.toml
378 string:
379 - meta_scope: string.quoted.double.toml
380 - match: '\\.'
381 scope: constant.character.escape.toml
382 - match: '"'
383 pop: true
384 literal:
385 - meta_scope: string.quoted.single.toml
386 - match: "'"
387 pop: true
388 multiline-string:
389 - meta_scope: string.quoted.triple.toml
390 - match: '"""'
391 pop: true
392 multiline-literal:
393 - meta_scope: string.quoted.triple.toml
394 - match: "'''"
395 pop: true
396"#;
397
398const DOCKERFILE_SYNTAX: &str = r#"%YAML 1.2
399---
400name: Dockerfile
401file_extensions: [Dockerfile, dockerfile]
402scope: source.dockerfile
403contexts:
404 main:
405 - match: '#.*$'
406 scope: comment.line.number-sign.dockerfile
407 - match: '(?i)^\s*(ADD|ARG|CMD|COPY|ENTRYPOINT|ENV|EXPOSE|FROM|HEALTHCHECK|LABEL|MAINTAINER|ONBUILD|RUN|SHELL|STOPSIGNAL|USER|VOLUME|WORKDIR)(?=\s)'
408 scope: keyword.control.dockerfile
409 - match: '\$\{?[A-Za-z_][A-Za-z0-9_]*\}?'
410 scope: variable.other.dockerfile
411 - match: '"'
412 push: double-string
413 - match: "'"
414 push: single-string
415 double-string:
416 - meta_scope: string.quoted.double.dockerfile
417 - match: '\\.'
418 scope: constant.character.escape.dockerfile
419 - match: '"'
420 pop: true
421 single-string:
422 - meta_scope: string.quoted.single.dockerfile
423 - match: "'"
424 pop: true
425"#;
426
427const EXTRA_SYNTAXES: &[&str] = &[
428 RHAI_SYNTAX,
429 TYPESCRIPT_SYNTAX,
430 JSONC_SYNTAX,
431 TOML_SYNTAX,
432 DOCKERFILE_SYNTAX,
433];
434
435#[derive(Clone)]
436pub struct NativeTextDocument {
437 snapshot: Arc<NativeTextSnapshot>,
438}
439
440#[derive(Debug, Eq, PartialEq)]
441struct NativeTextSnapshot {
442 identity: String,
443 revision: u64,
444 text: Arc<str>,
445}
446
447impl NativeTextDocument {
448 pub fn new(
454 identity: impl Into<String>,
455 revision: u64,
456 text: impl Into<Arc<str>>,
457 ) -> Result<Self, DocumentError> {
458 let identity = identity.into();
459 validate_identity(&identity)?;
460 Ok(Self {
461 snapshot: Arc::new(NativeTextSnapshot {
462 identity,
463 revision,
464 text: text.into(),
465 }),
466 })
467 }
468
469 #[must_use]
470 pub fn identity(&self) -> &str {
471 &self.snapshot.identity
472 }
473
474 #[must_use]
475 pub fn revision(&self) -> u64 {
476 self.snapshot.revision
477 }
478
479 #[must_use]
480 pub fn text(&self) -> &str {
481 &self.snapshot.text
482 }
483
484 #[must_use]
485 pub fn text_arc(&self) -> Arc<str> {
486 Arc::clone(&self.snapshot.text)
487 }
488}
489
490impl PartialEq for NativeTextDocument {
491 fn eq(&self, other: &Self) -> bool {
492 Arc::ptr_eq(&self.snapshot, &other.snapshot) || self.snapshot == other.snapshot
493 }
494}
495
496impl Eq for NativeTextDocument {}
497
498impl fmt::Debug for NativeTextDocument {
499 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
500 formatter
501 .debug_struct("NativeTextDocument")
502 .field("identity", &self.identity())
503 .field("revision", &self.revision())
504 .field("bytes", &self.text().len())
505 .finish()
506 }
507}
508
509impl CustomType for NativeTextDocument {
510 fn build(mut builder: TypeBuilder<Self>) {
511 builder
512 .with_name("NativeTextDocument")
513 .with_get("identity", |document: &mut Self| {
514 ImmutableString::from(document.identity().to_owned())
515 })
516 .with_get("revision", |document: &mut Self| {
517 i64::try_from(document.revision()).unwrap_or(i64::MAX)
518 })
519 .with_get("len", |document: &mut Self| {
520 i64::try_from(document.text().len()).unwrap_or(i64::MAX)
521 })
522 .with_fn("to_string", |document: &mut Self| {
523 format!(
524 "NativeTextDocument({}, revision={})",
525 document.identity(),
526 document.revision()
527 )
528 });
529 }
530}
531
532pub(crate) fn register_document_api(engine: &mut Engine) {
533 engine.build_type::<NativeTextDocument>();
534 FuncRegistration::new("is_native_text_document")
535 .in_global_namespace()
536 .register_into_engine(engine, |value: Dynamic| value.is::<NativeTextDocument>());
537}
538
539#[derive(Clone, Debug, Default)]
540pub struct NativeTextDocumentRegistry {
541 documents: BTreeMap<String, NativeTextDocument>,
542 readers: BTreeMap<String, BTreeSet<crate::read_dependency::ReadDependency>>,
543 missing_readers: BTreeMap<String, BTreeSet<crate::read_dependency::ReadDependency>>,
544}
545
546impl NativeTextDocumentRegistry {
547 #[must_use]
548 pub fn new() -> Self {
549 Self::default()
550 }
551
552 pub fn register(
559 &mut self,
560 name: impl Into<String>,
561 document: NativeTextDocument,
562 ) -> Result<(), DocumentError> {
563 self.register_with_invalidated(name.into(), document)?;
564 Ok(())
565 }
566
567 pub(crate) fn register_with_invalidated(
568 &mut self,
569 name: String,
570 document: NativeTextDocument,
571 ) -> Result<BTreeSet<ComponentInstancePath>, DocumentError> {
572 validate_identity(&name)?;
573 if self.documents.contains_key(&name) {
574 return Err(DocumentError::DuplicateDocument(name));
575 }
576 self.documents.insert(name.clone(), document);
577 Ok(crate::read_dependency::owners(
578 self.missing_readers.remove(&name).unwrap_or_default(),
579 ))
580 }
581
582 pub fn release(
589 &mut self,
590 name: &str,
591 ) -> Result<BTreeSet<ComponentInstancePath>, DocumentError> {
592 self.documents
593 .get(name)
594 .ok_or_else(|| DocumentError::UnknownDocument(name.to_owned()))?;
595 let readers = self.readers.get(name).cloned().unwrap_or_default();
596 let mut missing = self.missing_readers.clone();
597 for reader in &readers {
598 crate::read_dependency::track_missing_reader(&mut missing, reader, name).map_err(
599 |budget| DocumentError::MissingDependencyBudget {
600 kind: budget.kind,
601 limit: budget.limit,
602 },
603 )?;
604 }
605 self.documents.remove(name);
606 self.readers.remove(name);
607 self.missing_readers = missing;
608 Ok(crate::read_dependency::owners(readers))
609 }
610
611 pub fn get(&self, name: &str) -> Result<NativeTextDocument, DocumentError> {
620 self.documents
621 .get(name)
622 .cloned()
623 .ok_or_else(|| DocumentError::UnknownDocument(name.to_owned()))
624 }
625
626 pub fn replace(
632 &mut self,
633 name: &str,
634 document: NativeTextDocument,
635 ) -> Result<BTreeSet<ComponentInstancePath>, DocumentError> {
636 let current = self
637 .documents
638 .get_mut(name)
639 .ok_or_else(|| DocumentError::UnknownDocument(name.to_owned()))?;
640 if current == &document {
641 return Ok(BTreeSet::new());
642 }
643 if current.identity() == document.identity() && document.revision() <= current.revision() {
644 return Err(DocumentError::NonMonotonicRevision {
645 identity: document.identity().to_owned(),
646 current: current.revision(),
647 next: document.revision(),
648 });
649 }
650 *current = document;
651 Ok(crate::read_dependency::owners(
652 self.readers.get(name).cloned().unwrap_or_default(),
653 ))
654 }
655
656 pub fn append(
663 &mut self,
664 name: &str,
665 text: &str,
666 ) -> Result<BTreeSet<ComponentInstancePath>, DocumentError> {
667 let current = self
668 .documents
669 .get(name)
670 .ok_or_else(|| DocumentError::UnknownDocument(name.to_owned()))?;
671 if text.is_empty() {
672 return Ok(BTreeSet::new());
673 }
674 let next = current.revision().checked_add(1).ok_or_else(|| {
675 DocumentError::NonMonotonicRevision {
676 identity: current.identity().to_owned(),
677 current: current.revision(),
678 next: current.revision(),
679 }
680 })?;
681 let mut grown = String::with_capacity(current.text().len() + text.len());
682 grown.push_str(current.text());
683 grown.push_str(text);
684 let document = NativeTextDocument::new(current.identity(), next, grown)?;
685 self.replace(name, document)
686 }
687
688 #[cfg(test)]
689 pub(crate) fn read_tracked(
690 &mut self,
691 reader: &ComponentInstancePath,
692 name: &str,
693 ) -> Result<NativeTextDocument, DocumentError> {
694 self.read_dependency(
695 Some(&crate::read_dependency::ReadDependency::component(reader)),
696 name,
697 )
698 }
699
700 pub(crate) fn read_dependency(
702 &mut self,
703 reader: Option<&crate::read_dependency::ReadDependency>,
704 name: &str,
705 ) -> Result<NativeTextDocument, DocumentError> {
706 validate_identity(name)?;
707 let Some(document) = self.documents.get(name).cloned() else {
708 if let Some(reader) = reader {
709 crate::read_dependency::track_missing_reader(
710 &mut self.missing_readers,
711 reader,
712 name,
713 )
714 .map_err(|budget| DocumentError::MissingDependencyBudget {
715 kind: budget.kind,
716 limit: budget.limit,
717 })?;
718 }
719 return Err(DocumentError::UnknownDocument(name.to_owned()));
720 };
721 if let Some(reader) = reader {
722 self.readers
723 .entry(name.to_owned())
724 .or_default()
725 .insert(reader.clone());
726 }
727 Ok(document)
728 }
729
730 pub(crate) fn reset_reader(&mut self, reader: &ComponentInstancePath) {
731 self.reset_contribution(&crate::read_dependency::ReadDependency::component(reader));
732 }
733 pub(crate) fn reset_contribution(&mut self, reader: &crate::read_dependency::ReadDependency) {
734 for readers in self.readers.values_mut() {
735 readers.remove(reader);
736 }
737 self.readers.retain(|_, readers| !readers.is_empty());
738 crate::read_dependency::retain_readers(&mut self.missing_readers, |candidate| {
739 candidate != reader
740 });
741 }
742
743 pub(crate) fn retain_reader_scope(
744 &mut self,
745 root: &ComponentInstancePath,
746 active: &BTreeSet<ComponentInstancePath>,
747 ) {
748 for readers in self.readers.values_mut() {
749 readers.retain(|reader| reader.retained_in_owner_scope(root, active));
750 }
751 self.readers.retain(|_, readers| !readers.is_empty());
752 crate::read_dependency::retain_readers(&mut self.missing_readers, |reader| {
753 reader.retained_in_owner_scope(root, active)
754 });
755 }
756
757 pub(crate) fn remove_reader_scope(&mut self, root: &ComponentInstancePath) {
758 for readers in self.readers.values_mut() {
759 readers.retain(|reader| !reader.owner.is_within(root));
760 }
761 self.readers.retain(|_, readers| !readers.is_empty());
762 crate::read_dependency::retain_readers(&mut self.missing_readers, |reader| {
763 !reader.owner.is_within(root)
764 });
765 }
766
767 pub(crate) fn retain_contributions(
768 &mut self,
769 scope: &ComponentInstancePath,
770 active: &BTreeSet<crate::read_dependency::ReadContribution>,
771 ) {
772 crate::read_dependency::retain_readers(&mut self.readers, |reader| {
773 reader.retained_in_contribution_scope(scope, active)
774 });
775 crate::read_dependency::retain_readers(&mut self.missing_readers, |reader| {
776 reader.retained_in_contribution_scope(scope, active)
777 });
778 }
779}
780
781#[derive(Clone)]
782pub struct SyntaxRegistry {
783 syntax_set: Arc<RwLock<Arc<SyntaxSet>>>,
784}
785
786impl fmt::Debug for SyntaxRegistry {
787 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
788 formatter
789 .debug_struct("SyntaxRegistry")
790 .field("syntaxes", &self.snapshot().syntaxes().len())
791 .finish()
792 }
793}
794
795impl Default for SyntaxRegistry {
796 fn default() -> Self {
797 Self::new()
798 }
799}
800
801impl SyntaxRegistry {
802 pub(crate) fn from_snapshot(snapshot: Arc<SyntaxSet>) -> Self {
803 Self {
804 syntax_set: Arc::new(RwLock::new(snapshot)),
805 }
806 }
807
808 #[must_use]
809 pub fn new() -> Self {
810 Self {
811 syntax_set: Arc::new(RwLock::new(Arc::clone(default_syntax_set()))),
812 }
813 }
814
815 pub fn register_sublime_syntax(&self, source: &str) -> Result<(), DocumentError> {
821 let syntax = SyntaxDefinition::load_from_str(source, true, None)
822 .map_err(|error| DocumentError::Syntax(error.to_string()))?;
823 let current = self
824 .syntax_set
825 .read()
826 .map_err(|_| DocumentError::SyntaxRegistryPoisoned)?
827 .clone();
828 let mut builder = current.as_ref().clone().into_builder();
829 builder.add(syntax);
830 let next = Arc::new(builder.build());
831 *self
832 .syntax_set
833 .write()
834 .map_err(|_| DocumentError::SyntaxRegistryPoisoned)? = next;
835 Ok(())
836 }
837
838 #[must_use]
839 pub fn snapshot(&self) -> Arc<SyntaxSet> {
840 self.syntax_set
841 .read()
842 .map_or_else(|poisoned| poisoned.into_inner().clone(), |set| set.clone())
843 }
844}
845
846fn default_syntax_set() -> &'static Arc<SyntaxSet> {
847 static SYNTAXES: OnceLock<Arc<SyntaxSet>> = OnceLock::new();
848 SYNTAXES.get_or_init(|| {
849 let mut builder = SyntaxSet::load_defaults_newlines().into_builder();
850 for source in EXTRA_SYNTAXES {
851 builder.add(
852 SyntaxDefinition::load_from_str(source, true, None)
853 .expect("built-in syntax is valid"),
854 );
855 }
856 Arc::new(builder.build())
857 })
858}
859
860#[derive(Clone, Debug, Eq, PartialEq)]
861pub enum DocumentSource {
862 Inline(Arc<str>),
863 Native(NativeTextDocument),
864}
865
866impl DocumentSource {
867 #[must_use]
868 pub fn identity(&self) -> Cow<'_, str> {
869 match self {
870 Self::Inline(_) => Cow::Borrowed("inline"),
871 Self::Native(document) => Cow::Borrowed(document.identity()),
872 }
873 }
874
875 #[must_use]
876 pub fn revision(&self) -> u64 {
877 match self {
878 Self::Inline(text) => stable_text_hash(text),
879 Self::Native(document) => document.revision(),
880 }
881 }
882
883 #[must_use]
884 pub fn text(&self) -> Arc<str> {
885 match self {
886 Self::Inline(text) => Arc::clone(text),
887 Self::Native(document) => document.text_arc(),
888 }
889 }
890}
891
892impl From<String> for DocumentSource {
893 fn from(value: String) -> Self {
894 Self::Inline(Arc::from(value))
895 }
896}
897
898impl From<NativeTextDocument> for DocumentSource {
899 fn from(value: NativeTextDocument) -> Self {
900 Self::Native(value)
901 }
902}
903
904#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
905pub enum DocumentWrap {
906 #[default]
907 None,
908 Viewport,
909 Column(usize),
910}
911
912#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
913pub enum DiffViewMode {
914 #[default]
915 Unified,
916 Split,
917}
918
919#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
920pub enum DiffWhitespace {
921 #[default]
922 Exact,
923 IgnoreChanges,
924 IgnoreAll,
925}
926
927#[derive(Clone, Debug, Eq, PartialEq)]
928pub struct DocumentDescriptor {
929 pub source: DocumentSource,
930 pub label: String,
931 pub file_name: Option<String>,
932 pub language: Option<String>,
933}
934
935#[derive(Clone, Copy, Debug, Eq, PartialEq)]
936pub enum SyntaxTokenKind {
937 Comment,
938 String,
939 Number,
940 Keyword,
941 Function,
942 Type,
943 Variable,
944 Constant,
945 Operator,
946 Punctuation,
947 Tag,
948 Attribute,
949}
950
951impl SyntaxTokenKind {
952 #[must_use]
953 pub const fn theme_token(self) -> &'static str {
954 match self {
955 Self::Comment => "syntax.comment",
956 Self::String => "syntax.string",
957 Self::Number => "syntax.number",
958 Self::Keyword => "syntax.keyword",
959 Self::Function => "syntax.function",
960 Self::Type => "syntax.type",
961 Self::Variable => "syntax.variable",
962 Self::Constant => "syntax.constant",
963 Self::Operator => "syntax.operator",
964 Self::Punctuation => "syntax.punctuation",
965 Self::Tag => "syntax.tag",
966 Self::Attribute => "syntax.attribute",
967 }
968 }
969}
970
971#[derive(Clone, Debug, Eq, PartialEq)]
972pub struct SyntaxSpan {
973 pub range: Range<usize>,
974 pub kind: SyntaxTokenKind,
975}
976
977#[derive(Clone, Debug, Eq, PartialEq)]
978pub struct DocumentLine {
979 pub range: Range<usize>,
980 pub syntax: Vec<SyntaxSpan>,
981}
982
983#[derive(Clone, Debug)]
984pub struct PreparedDocument {
985 identity: String,
986 revision: u64,
987 text: Arc<str>,
988 lines: Arc<[DocumentLine]>,
989 language: String,
990 max_line_chars: usize,
991 ends_with_newline: bool,
992 highlight_error: Option<String>,
993}
994
995impl PreparedDocument {
996 #[must_use]
997 pub fn identity(&self) -> &str {
998 &self.identity
999 }
1000
1001 #[must_use]
1002 pub const fn revision(&self) -> u64 {
1003 self.revision
1004 }
1005
1006 #[must_use]
1007 pub fn text(&self) -> &str {
1008 &self.text
1009 }
1010
1011 #[must_use]
1012 pub fn text_arc(&self) -> Arc<str> {
1013 Arc::clone(&self.text)
1014 }
1015
1016 #[must_use]
1017 pub fn lines(&self) -> &[DocumentLine] {
1018 &self.lines
1019 }
1020
1021 #[must_use]
1022 pub fn line_text(&self, index: usize) -> Option<&str> {
1023 let line = self.lines.get(index)?;
1024 self.text.get(line.range.clone())
1025 }
1026
1027 #[must_use]
1028 pub fn language(&self) -> &str {
1029 &self.language
1030 }
1031
1032 #[must_use]
1033 pub const fn max_line_chars(&self) -> usize {
1034 self.max_line_chars
1035 }
1036
1037 #[must_use]
1038 pub const fn ends_with_newline(&self) -> bool {
1039 self.ends_with_newline
1040 }
1041
1042 #[must_use]
1043 pub fn highlight_error(&self) -> Option<&str> {
1044 self.highlight_error.as_deref()
1045 }
1046}
1047
1048pub fn prepare_document(
1055 descriptor: &DocumentDescriptor,
1056 syntaxes: &SyntaxRegistry,
1057) -> Result<PreparedDocument, DocumentError> {
1058 prepare_document_with_limits(descriptor, syntaxes, DocumentLimits::default())
1059}
1060
1061pub fn prepare_document_with_limits(
1067 descriptor: &DocumentDescriptor,
1068 syntaxes: &SyntaxRegistry,
1069 limits: DocumentLimits,
1070) -> Result<PreparedDocument, DocumentError> {
1071 let text = descriptor.source.text();
1072 if text.len() > limits.max_document_bytes {
1073 return Err(DocumentError::TooManyBytes {
1074 actual: text.len(),
1075 limit: limits.max_document_bytes,
1076 });
1077 }
1078 let ranges = document_line_ranges(&text);
1079 if ranges.len() > limits.max_document_lines {
1080 return Err(DocumentError::TooManyLines {
1081 actual: ranges.len(),
1082 limit: limits.max_document_lines,
1083 });
1084 }
1085 let syntax_set = syntaxes.snapshot();
1086 let syntax = resolve_syntax(
1087 &syntax_set,
1088 descriptor.language.as_deref(),
1089 descriptor.file_name.as_deref(),
1090 );
1091 let language = syntax.name.clone();
1092 let mut parser = ParseState::new(syntax);
1093 let mut stack = ScopeStack::new();
1094 let selectors = syntax_selectors();
1095 let mut lines = Vec::with_capacity(ranges.len());
1096 let mut max_line_chars = 0usize;
1097 let mut highlight_error = None;
1098 let mut highlighting = true;
1099 for range in ranges {
1100 let line = &text[range.clone()];
1101 max_line_chars = max_line_chars.max(line.chars().count());
1102 let parse_end = text[range.end..]
1103 .find('\n')
1104 .map_or(range.end, |offset| range.end + offset + 1);
1105 let parse_start = range.start;
1106 let parse_line = &text[parse_start..parse_end];
1107 let mut spans: Vec<SyntaxSpan> = Vec::new();
1108 if highlighting {
1109 match parser.parse_line(parse_line, &syntax_set) {
1110 Ok(operations) => {
1111 for (token_range, operation) in ScopeRangeIterator::new(&operations, parse_line)
1112 {
1113 if let Err(error) = stack.apply(operation) {
1114 highlight_error.get_or_insert_with(|| error.to_string());
1115 highlighting = false;
1116 spans.clear();
1117 break;
1118 }
1119 let start = token_range.start.min(line.len());
1120 let end = token_range.end.min(line.len());
1121 if start >= end {
1122 continue;
1123 }
1124 if let Some(kind) = selectors.kind(stack.as_slice()) {
1125 if let Some(previous) = spans.last_mut()
1126 && previous.kind == kind
1127 && previous.range.end == start
1128 {
1129 previous.range.end = end;
1130 } else {
1131 spans.push(SyntaxSpan {
1132 range: start..end,
1133 kind,
1134 });
1135 }
1136 }
1137 }
1138 }
1139 Err(error) => {
1140 highlight_error.get_or_insert_with(|| error.to_string());
1141 highlighting = false;
1142 }
1143 }
1144 }
1145 lines.push(DocumentLine {
1146 range,
1147 syntax: spans,
1148 });
1149 }
1150 Ok(PreparedDocument {
1151 identity: descriptor.source.identity().into_owned(),
1152 revision: descriptor.source.revision(),
1153 text,
1154 lines: lines.into(),
1155 language,
1156 max_line_chars,
1157 ends_with_newline: descriptor.source.text().ends_with('\n'),
1158 highlight_error,
1159 })
1160}
1161
1162fn document_line_ranges(text: &str) -> Vec<Range<usize>> {
1163 let mut ranges = Vec::new();
1164 let mut start = 0usize;
1165 for (index, byte) in text.bytes().enumerate() {
1166 if byte == b'\n' {
1167 let mut end = index;
1168 if text.as_bytes().get(index.wrapping_sub(1)) == Some(&b'\r') && index > start {
1169 end -= 1;
1170 }
1171 ranges.push(start..end);
1172 start = index + 1;
1173 }
1174 }
1175 if start < text.len() || text.is_empty() || text.ends_with('\n') {
1176 ranges.push(start..text.len());
1177 }
1178 ranges
1179}
1180
1181fn resolve_syntax<'a>(
1182 syntaxes: &'a SyntaxSet,
1183 language: Option<&str>,
1184 file_name: Option<&str>,
1185) -> &'a syntect::parsing::SyntaxReference {
1186 language
1187 .and_then(|language| {
1188 let token = language_alias(language);
1189 syntaxes.find_syntax_by_token(token.as_ref())
1190 })
1191 .or_else(|| {
1192 file_name.and_then(|file_name| {
1193 syntaxes.find_syntax_by_extension(file_name).or_else(|| {
1194 file_name
1195 .rsplit_once('.')
1196 .and_then(|(_, ext)| syntaxes.find_syntax_by_extension(ext))
1197 })
1198 })
1199 })
1200 .unwrap_or_else(|| syntaxes.find_syntax_plain_text())
1201}
1202
1203fn language_alias(language: &str) -> Cow<'_, str> {
1204 let normalized = language.trim().to_ascii_lowercase();
1205 match normalized.as_str() {
1206 "plaintext" | "plain_text" | "text" => Cow::Borrowed("txt"),
1207 "javascript" => Cow::Borrowed("js"),
1208 "typescript" => Cow::Borrowed("ts"),
1209 "shell" | "bash" => Cow::Borrowed("sh"),
1210 "python" => Cow::Borrowed("py"),
1211 "markdown" => Cow::Borrowed("md"),
1212 "rust" => Cow::Borrowed("rs"),
1213 "golang" => Cow::Borrowed("go"),
1214 _ => Cow::Owned(normalized),
1215 }
1216}
1217
1218struct SyntaxSelectors {
1219 comment: ScopeSelectors,
1220 string: ScopeSelectors,
1221 number: ScopeSelectors,
1222 keyword: ScopeSelectors,
1223 function: ScopeSelectors,
1224 type_name: ScopeSelectors,
1225 variable: ScopeSelectors,
1226 constant: ScopeSelectors,
1227 operator: ScopeSelectors,
1228 punctuation: ScopeSelectors,
1229 tag: ScopeSelectors,
1230 attribute: ScopeSelectors,
1231}
1232
1233impl SyntaxSelectors {
1234 fn new() -> Self {
1235 let parse = |value: &str| ScopeSelectors::from_str(value).expect("static scope selector");
1236 Self {
1237 comment: parse("comment"),
1238 string: parse("string"),
1239 number: parse("constant.numeric"),
1240 keyword: parse("keyword, storage.modifier"),
1241 function: parse("entity.name.function, support.function"),
1242 type_name: parse("entity.name.type, support.type, storage.type"),
1243 variable: parse("variable"),
1244 constant: parse("constant"),
1245 operator: parse("keyword.operator"),
1246 punctuation: parse("punctuation"),
1247 tag: parse("entity.name.tag"),
1248 attribute: parse("entity.other.attribute-name"),
1249 }
1250 }
1251
1252 fn kind(&self, stack: &[syntect::parsing::Scope]) -> Option<SyntaxTokenKind> {
1253 [
1254 (&self.comment, SyntaxTokenKind::Comment),
1255 (&self.string, SyntaxTokenKind::String),
1256 (&self.number, SyntaxTokenKind::Number),
1257 (&self.function, SyntaxTokenKind::Function),
1258 (&self.type_name, SyntaxTokenKind::Type),
1259 (&self.tag, SyntaxTokenKind::Tag),
1260 (&self.attribute, SyntaxTokenKind::Attribute),
1261 (&self.operator, SyntaxTokenKind::Operator),
1262 (&self.keyword, SyntaxTokenKind::Keyword),
1263 (&self.variable, SyntaxTokenKind::Variable),
1264 (&self.constant, SyntaxTokenKind::Constant),
1265 (&self.punctuation, SyntaxTokenKind::Punctuation),
1266 ]
1267 .into_iter()
1268 .find_map(|(selector, kind)| selector.does_match(stack).map(|_| kind))
1269 }
1270}
1271
1272fn syntax_selectors() -> &'static SyntaxSelectors {
1273 static SELECTORS: OnceLock<SyntaxSelectors> = OnceLock::new();
1274 SELECTORS.get_or_init(SyntaxSelectors::new)
1275}
1276
1277#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1278pub enum DiffRowKind {
1279 Equal,
1280 LeftOnly,
1281 RightOnly,
1282 Modified,
1283}
1284
1285#[derive(Clone, Debug, Eq, PartialEq)]
1286pub struct DiffAlignedRow {
1287 pub left: Option<usize>,
1288 pub right: Option<usize>,
1289 pub kind: DiffRowKind,
1290 pub hunk: Option<usize>,
1291 pub left_inline: Vec<Range<usize>>,
1292 pub right_inline: Vec<Range<usize>>,
1293}
1294
1295#[derive(Clone, Debug, Eq, PartialEq)]
1296pub enum DiffDisplayRow {
1297 Content(usize),
1298 Fold {
1299 id: usize,
1300 full_range: Range<usize>,
1301 hidden_rows: usize,
1302 },
1303}
1304
1305#[derive(Clone, Debug)]
1306pub struct PreparedDiff {
1307 pub left: PreparedDocument,
1308 pub right: PreparedDocument,
1309 pub rows: Arc<[DiffAlignedRow]>,
1310 pub collapsed: Arc<[DiffDisplayRow]>,
1311 pub hunk_count: usize,
1312}
1313
1314pub fn prepare_diff(
1320 left: &DocumentDescriptor,
1321 right: &DocumentDescriptor,
1322 whitespace: DiffWhitespace,
1323 context_lines: Option<usize>,
1324 syntaxes: &SyntaxRegistry,
1325) -> Result<PreparedDiff, DocumentError> {
1326 prepare_diff_with_limits(
1327 left,
1328 right,
1329 whitespace,
1330 context_lines,
1331 syntaxes,
1332 DocumentLimits::default(),
1333 )
1334}
1335
1336#[allow(clippy::too_many_lines)]
1342pub fn prepare_diff_with_limits(
1343 left: &DocumentDescriptor,
1344 right: &DocumentDescriptor,
1345 whitespace: DiffWhitespace,
1346 context_lines: Option<usize>,
1347 syntaxes: &SyntaxRegistry,
1348 limits: DocumentLimits,
1349) -> Result<PreparedDiff, DocumentError> {
1350 let total_bytes = left
1351 .source
1352 .text()
1353 .len()
1354 .saturating_add(right.source.text().len());
1355 if total_bytes > limits.max_diff_total_bytes {
1356 return Err(DocumentError::TooManyDiffBytes {
1357 actual: total_bytes,
1358 limit: limits.max_diff_total_bytes,
1359 });
1360 }
1361 let left = prepare_document_with_limits(left, syntaxes, limits)?;
1362 let right = prepare_document_with_limits(right, syntaxes, limits)?;
1363 let left_keys = diff_keys(&left, whitespace);
1364 let right_keys = diff_keys(&right, whitespace);
1365 let deadline = Instant::now().checked_add(limits.diff_timeout);
1366 let ops = capture_diff_slices_deadline(Algorithm::Patience, &left_keys, &right_keys, deadline);
1367 let mut rows = Vec::with_capacity(left.lines().len().max(right.lines().len()));
1368 let mut hunk = 0usize;
1369 let mut in_change = false;
1370 for operation in ops {
1371 match operation {
1372 DiffOp::Equal {
1373 old_index,
1374 new_index,
1375 len,
1376 } => {
1377 in_change = false;
1378 rows.extend((0..len).map(|offset| DiffAlignedRow {
1379 left: Some(old_index + offset),
1380 right: Some(new_index + offset),
1381 kind: DiffRowKind::Equal,
1382 hunk: None,
1383 left_inline: Vec::new(),
1384 right_inline: Vec::new(),
1385 }));
1386 }
1387 DiffOp::Delete {
1388 old_index, old_len, ..
1389 } => {
1390 let id = next_hunk(&mut hunk, &mut in_change, limits.max_diff_hunks)?;
1391 rows.extend((0..old_len).map(|offset| DiffAlignedRow {
1392 left: Some(old_index + offset),
1393 right: None,
1394 kind: DiffRowKind::LeftOnly,
1395 hunk: Some(id),
1396 left_inline: Vec::new(),
1397 right_inline: Vec::new(),
1398 }));
1399 }
1400 DiffOp::Insert {
1401 new_index, new_len, ..
1402 } => {
1403 let id = next_hunk(&mut hunk, &mut in_change, limits.max_diff_hunks)?;
1404 rows.extend((0..new_len).map(|offset| DiffAlignedRow {
1405 left: None,
1406 right: Some(new_index + offset),
1407 kind: DiffRowKind::RightOnly,
1408 hunk: Some(id),
1409 left_inline: Vec::new(),
1410 right_inline: Vec::new(),
1411 }));
1412 }
1413 DiffOp::Replace {
1414 old_index,
1415 old_len,
1416 new_index,
1417 new_len,
1418 } => {
1419 let id = next_hunk(&mut hunk, &mut in_change, limits.max_diff_hunks)?;
1420 let aligned = old_len.max(new_len);
1421 for offset in 0..aligned {
1422 let left_index = (offset < old_len).then_some(old_index + offset);
1423 let right_index = (offset < new_len).then_some(new_index + offset);
1424 let (left_inline, right_inline) = match (left_index, right_index) {
1425 (Some(left_index), Some(right_index)) => inline_ranges(
1426 left.line_text(left_index).unwrap_or_default(),
1427 right.line_text(right_index).unwrap_or_default(),
1428 deadline,
1429 ),
1430 _ => (Vec::new(), Vec::new()),
1431 };
1432 rows.push(DiffAlignedRow {
1433 left: left_index,
1434 right: right_index,
1435 kind: match (left_index, right_index) {
1436 (Some(_), Some(_)) => DiffRowKind::Modified,
1437 (Some(_), None) => DiffRowKind::LeftOnly,
1438 (None, Some(_)) => DiffRowKind::RightOnly,
1439 (None, None) => unreachable!(),
1440 },
1441 hunk: Some(id),
1442 left_inline,
1443 right_inline,
1444 });
1445 }
1446 }
1447 }
1448 }
1449 let collapsed = collapse_rows(&rows, context_lines);
1450 Ok(PreparedDiff {
1451 left,
1452 right,
1453 rows: rows.into(),
1454 collapsed: collapsed.into(),
1455 hunk_count: hunk,
1456 })
1457}
1458
1459fn next_hunk(
1460 hunks: &mut usize,
1461 in_change: &mut bool,
1462 limit: usize,
1463) -> Result<usize, DocumentError> {
1464 if !*in_change {
1465 *hunks = hunks.checked_add(1).ok_or(DocumentError::TooManyHunks {
1466 actual: usize::MAX,
1467 limit,
1468 })?;
1469 *in_change = true;
1470 }
1471 if *hunks > limit {
1472 return Err(DocumentError::TooManyHunks {
1473 actual: *hunks,
1474 limit,
1475 });
1476 }
1477 Ok(*hunks - 1)
1478}
1479
1480fn diff_keys(document: &PreparedDocument, whitespace: DiffWhitespace) -> Vec<Cow<'_, str>> {
1481 document
1482 .lines()
1483 .iter()
1484 .enumerate()
1485 .map(|(index, _)| {
1486 let line = document.line_text(index).unwrap_or_default();
1487 match whitespace {
1488 DiffWhitespace::Exact => Cow::Borrowed(line),
1489 DiffWhitespace::IgnoreChanges => {
1490 Cow::Owned(line.split_whitespace().collect::<Vec<_>>().join(" "))
1491 }
1492 DiffWhitespace::IgnoreAll => Cow::Owned(
1493 line.chars()
1494 .filter(|character| !character.is_whitespace())
1495 .collect(),
1496 ),
1497 }
1498 })
1499 .collect()
1500}
1501
1502fn inline_ranges(
1503 left: &str,
1504 right: &str,
1505 deadline: Option<Instant>,
1506) -> (Vec<Range<usize>>, Vec<Range<usize>>) {
1507 if left.len().saturating_add(right.len()) > MAX_INLINE_DIFF_BYTES {
1508 return (
1509 (!left.is_empty())
1510 .then_some(0..left.len())
1511 .into_iter()
1512 .collect(),
1513 (!right.is_empty())
1514 .then_some(0..right.len())
1515 .into_iter()
1516 .collect(),
1517 );
1518 }
1519 let left_graphemes = left.graphemes(true).collect::<Vec<_>>();
1520 let right_graphemes = right.graphemes(true).collect::<Vec<_>>();
1521 let operations = capture_diff_slices_deadline(
1522 Algorithm::Patience,
1523 &left_graphemes,
1524 &right_graphemes,
1525 deadline,
1526 );
1527 let mut left_offset = 0usize;
1528 let mut right_offset = 0usize;
1529 let mut left_ranges = Vec::new();
1530 let mut right_ranges = Vec::new();
1531 for operation in operations {
1532 for change in operation.iter_changes(&left_graphemes, &right_graphemes) {
1533 let len = change.value().len();
1534 match change.tag() {
1535 similar::ChangeTag::Equal => {
1536 left_offset += len;
1537 right_offset += len;
1538 }
1539 similar::ChangeTag::Delete => {
1540 push_range(&mut left_ranges, left_offset..left_offset + len);
1541 left_offset += len;
1542 }
1543 similar::ChangeTag::Insert => {
1544 push_range(&mut right_ranges, right_offset..right_offset + len);
1545 right_offset += len;
1546 }
1547 }
1548 }
1549 }
1550 (left_ranges, right_ranges)
1551}
1552
1553fn push_range(ranges: &mut Vec<Range<usize>>, range: Range<usize>) {
1554 if range.is_empty() {
1555 return;
1556 }
1557 if let Some(previous) = ranges.last_mut()
1558 && previous.end == range.start
1559 {
1560 previous.end = range.end;
1561 } else {
1562 ranges.push(range);
1563 }
1564}
1565
1566fn collapse_rows(rows: &[DiffAlignedRow], context: Option<usize>) -> Vec<DiffDisplayRow> {
1567 let Some(context) = context else {
1568 return (0..rows.len()).map(DiffDisplayRow::Content).collect();
1569 };
1570 if rows.iter().all(|row| row.kind == DiffRowKind::Equal) {
1571 return if rows.is_empty() {
1572 Vec::new()
1573 } else {
1574 vec![DiffDisplayRow::Fold {
1575 id: 0,
1576 full_range: 0..rows.len(),
1577 hidden_rows: rows.len(),
1578 }]
1579 };
1580 }
1581 let mut keep = vec![false; rows.len()];
1582 for (index, row) in rows.iter().enumerate() {
1583 if row.kind != DiffRowKind::Equal {
1584 let start = index.saturating_sub(context);
1585 let end = index
1586 .saturating_add(context)
1587 .saturating_add(1)
1588 .min(rows.len());
1589 keep[start..end].fill(true);
1590 }
1591 }
1592 let mut display = Vec::new();
1593 let mut index = 0usize;
1594 let mut fold = 0usize;
1595 while index < rows.len() {
1596 if keep[index] {
1597 display.push(DiffDisplayRow::Content(index));
1598 index += 1;
1599 continue;
1600 }
1601 let start = index;
1602 while index < rows.len() && !keep[index] {
1603 index += 1;
1604 }
1605 display.push(DiffDisplayRow::Fold {
1606 id: fold,
1607 full_range: start..index,
1608 hidden_rows: index - start,
1609 });
1610 fold += 1;
1611 }
1612 display
1613}
1614
1615fn stable_text_hash(text: &str) -> u64 {
1616 const OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
1617 const PRIME: u64 = 0x0000_0100_0000_01b3;
1618 text.bytes().fold(OFFSET, |hash, byte| {
1619 (hash ^ u64::from(byte)).wrapping_mul(PRIME)
1620 })
1621}
1622
1623fn validate_identity(value: &str) -> Result<(), DocumentError> {
1624 if value.is_empty()
1625 || value.len() > 256
1626 || !value.chars().all(|character| {
1627 character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '.' | '/' | ':')
1628 })
1629 {
1630 Err(DocumentError::InvalidIdentity(value.to_owned()))
1631 } else {
1632 Ok(())
1633 }
1634}
1635
1636#[derive(Clone, Debug, Error, Eq, PartialEq)]
1637pub enum DocumentError {
1638 #[error("native document missing-dependency {kind} exceeds limit {limit}")]
1639 MissingDependencyBudget { kind: &'static str, limit: usize },
1640 #[error("document identity `{0}` must contain 1-256 safe ASCII characters")]
1641 InvalidIdentity(String),
1642 #[error("native text document `{0}` is already registered")]
1643 DuplicateDocument(String),
1644 #[error("native text document `{0}` is not registered")]
1645 UnknownDocument(String),
1646 #[error(
1647 "native text document `{identity}` revision must increase: current {current}, next {next}"
1648 )]
1649 NonMonotonicRevision {
1650 identity: String,
1651 current: u64,
1652 next: u64,
1653 },
1654 #[error("document contains {actual} bytes, exceeding the {limit}-byte budget")]
1655 TooManyBytes { actual: usize, limit: usize },
1656 #[error("diff inputs contain {actual} bytes, exceeding the {limit}-byte budget")]
1657 TooManyDiffBytes { actual: usize, limit: usize },
1658 #[error("document contains {actual} lines, exceeding the {limit}-line budget")]
1659 TooManyLines { actual: usize, limit: usize },
1660 #[error("diff contains {actual} hunks, exceeding the {limit}-hunk budget")]
1661 TooManyHunks { actual: usize, limit: usize },
1662 #[error("syntax definition or parse failed: {0}")]
1663 Syntax(String),
1664 #[error("syntax registry lock was poisoned")]
1665 SyntaxRegistryPoisoned,
1666 #[error("document resource limits must all be greater than zero")]
1667 InvalidLimits,
1668 #[error("document runtime configuration lock was poisoned")]
1669 DocumentConfigPoisoned,
1670}
1671
1672#[cfg(test)]
1673mod tests {
1674 use super::*;
1675
1676 fn descriptor(text: &str, language: &str) -> DocumentDescriptor {
1677 DocumentDescriptor {
1678 source: DocumentSource::from(text.to_owned()),
1679 label: language.to_owned(),
1680 file_name: None,
1681 language: Some(language.to_owned()),
1682 }
1683 }
1684
1685 #[test]
1686 fn native_documents_are_revisioned_and_registry_reads_are_exact() {
1687 let first = NativeTextDocument::new("server/a", 1, Arc::<str>::from("port=80\n")).unwrap();
1688 let second =
1689 NativeTextDocument::new("server/a", 2, Arc::<str>::from("port=443\n")).unwrap();
1690 let reader = ComponentInstancePath::root("View", "main");
1691 let mut registry = NativeTextDocumentRegistry::new();
1692 registry.register("config", first).unwrap();
1693 assert_eq!(
1694 registry.read_tracked(&reader, "config").unwrap().revision(),
1695 1
1696 );
1697 assert_eq!(
1698 registry.replace("config", second).unwrap(),
1699 BTreeSet::from([reader])
1700 );
1701 let stale = NativeTextDocument::new("server/a", 2, Arc::<str>::from("stale")).unwrap();
1702 assert!(matches!(
1703 registry.replace("config", stale),
1704 Err(DocumentError::NonMonotonicRevision { .. })
1705 ));
1706 }
1707
1708 #[test]
1709 fn an_append_is_the_next_revision_of_the_same_document() {
1710 let reader = ComponentInstancePath::root("View", "main");
1711 let mut registry = NativeTextDocumentRegistry::new();
1712 registry
1713 .register(
1714 "reply",
1715 NativeTextDocument::new("reply/7", 3, Arc::<str>::from("# Reply\n")).unwrap(),
1716 )
1717 .unwrap();
1718 registry.read_tracked(&reader, "reply").unwrap();
1719 assert_eq!(
1720 registry.append("reply", "\nfirst chunk").unwrap(),
1721 BTreeSet::from([reader.clone()])
1722 );
1723 let grown = registry.read_tracked(&reader, "reply").unwrap();
1724 assert_eq!(
1725 (grown.identity(), grown.revision(), grown.text()),
1726 ("reply/7", 4, "# Reply\n\nfirst chunk")
1727 );
1728 assert!(
1729 registry.append("reply", "").unwrap().is_empty(),
1730 "empty text changes nothing"
1731 );
1732 assert_eq!(
1733 registry.read_tracked(&reader, "reply").unwrap().revision(),
1734 4
1735 );
1736 assert!(matches!(
1737 registry.append("missing", "text"),
1738 Err(DocumentError::UnknownDocument(_))
1739 ));
1740 }
1741
1742 #[test]
1743 fn mounted_document_registration_release_and_pending_reads_are_exact() {
1744 let root = ComponentInstancePath::root("View", "documents");
1745 let reader = root.child("Reader", "a");
1746 let sibling = root.child("Reader", "b");
1747 let mut registry = NativeTextDocumentRegistry::new();
1748 assert!(registry.read_tracked(&reader, "late").is_err());
1749 assert!(registry.read_tracked(&sibling, "other").is_err());
1750 let document = NativeTextDocument::new("reply/42", 1, "first").unwrap();
1751 assert_eq!(
1752 registry
1753 .register_with_invalidated("late".to_owned(), document.clone())
1754 .unwrap(),
1755 BTreeSet::from([reader.clone()])
1756 );
1757 assert!(registry.register("late", document.clone()).is_err());
1758 let held = registry.read_tracked(&reader, "late").unwrap();
1759 assert_eq!(
1760 registry.release("late").unwrap(),
1761 BTreeSet::from([reader.clone()])
1762 );
1763 assert_eq!(
1764 held.text(),
1765 "first",
1766 "release must not invalidate immutable reader snapshots"
1767 );
1768 assert!(registry.get("late").is_err());
1769 assert!(registry.replace("late", document.clone()).is_err());
1770 assert_eq!(
1771 registry
1772 .register_with_invalidated("late".to_owned(), document.clone())
1773 .unwrap(),
1774 BTreeSet::from([reader.clone()])
1775 );
1776 assert!(
1777 registry
1778 .replace(
1779 "late",
1780 NativeTextDocument::new("reply/42", 0, "stale").unwrap()
1781 )
1782 .is_err()
1783 );
1784 assert_eq!(registry.get("late").unwrap(), document);
1785 registry.remove_reader_scope(&root);
1786 assert!(
1787 registry
1788 .register_with_invalidated("other".to_owned(), document)
1789 .unwrap()
1790 .is_empty()
1791 );
1792 }
1793
1794 #[test]
1795 fn failed_document_candidate_restores_registry_and_pending_lookups() {
1796 let root = ComponentInstancePath::root("View", "rollback");
1797 let mut runtime = crate::UiRuntimeState::default();
1798 assert!(
1799 runtime
1800 .native_documents
1801 .read_tracked(&root, "late")
1802 .is_err()
1803 );
1804 let snapshot = runtime.begin_transaction().unwrap();
1805 runtime
1806 .register_native_text_document_from_host(
1807 "late",
1808 NativeTextDocument::new("late", 1, "candidate").unwrap(),
1809 )
1810 .unwrap();
1811 runtime.rollback_transaction(snapshot).unwrap();
1812 assert!(runtime.native_documents.get("late").is_err());
1813 assert_eq!(
1814 runtime
1815 .native_documents
1816 .register_with_invalidated(
1817 "late".to_owned(),
1818 NativeTextDocument::new("late", 2, "accepted").unwrap()
1819 )
1820 .unwrap(),
1821 BTreeSet::from([root])
1822 );
1823 }
1824
1825 #[test]
1826 fn caught_missing_document_lookups_are_bounded_and_reset_with_the_owner() {
1827 use std::{cell::RefCell, rc::Rc};
1828 let reader = ComponentInstancePath::root("View", "missing-documents");
1829 let registry = Rc::new(RefCell::new(NativeTextDocumentRegistry::new()));
1830 let captured = registry.clone();
1831 let owner = reader.clone();
1832 let mut engine = Engine::new();
1833 register_document_api(&mut engine);
1834 engine.register_fn(
1835 "missing_document",
1836 move |name: ImmutableString| -> Result<NativeTextDocument, Box<rhai::EvalAltResult>> {
1837 captured
1838 .borrow_mut()
1839 .read_tracked(&owner, &name)
1840 .map_err(|error| {
1841 Box::new(rhai::EvalAltResult::ErrorRuntime(
1842 error.to_string().into(),
1843 rhai::Position::NONE,
1844 ))
1845 })
1846 },
1847 );
1848 engine
1849 .eval::<()>(
1850 "for i in 0..1000 { try { missing_document(`late-${i}`); } catch (error) { () } }",
1851 )
1852 .unwrap();
1853 let mut registry = registry.borrow_mut();
1854 assert_eq!(
1855 registry.missing_readers.len(),
1856 crate::read_dependency::MAX_MISSING_NAMES_PER_READER
1857 );
1858 assert!(matches!(
1859 registry.read_tracked(&reader, "overflow"),
1860 Err(DocumentError::MissingDependencyBudget { .. })
1861 ));
1862 registry.reset_reader(&reader);
1863 assert!(registry.missing_readers.is_empty());
1864 assert!(matches!(
1865 registry.read_tracked(&reader, "after-rerender"),
1866 Err(DocumentError::UnknownDocument(_))
1867 ));
1868 registry.remove_reader_scope(&reader);
1869 assert!(registry.missing_readers.is_empty());
1870 assert!(
1871 registry
1872 .register_with_invalidated(
1873 "after-rerender".to_owned(),
1874 NativeTextDocument::new("fresh", 1, "fresh").unwrap()
1875 )
1876 .unwrap()
1877 .is_empty()
1878 );
1879 }
1880
1881 #[test]
1882 fn rhai_and_go_are_built_in_and_unknown_languages_fall_back_to_plain_text() {
1883 let registry = SyntaxRegistry::new();
1884 let rhai = prepare_document(&descriptor("fn view() { 42 }\n", "rhai"), ®istry).unwrap();
1885 assert_eq!(rhai.language(), "Rhai");
1886 assert!(
1887 rhai.lines()[0]
1888 .syntax
1889 .iter()
1890 .any(|span| span.kind == SyntaxTokenKind::Keyword),
1891 "{:?}",
1892 rhai.lines()[0].syntax
1893 );
1894
1895 let go = prepare_document(
1896 &descriptor("package main\nfunc main() {}\n", "go"),
1897 ®istry,
1898 )
1899 .unwrap();
1900 assert_eq!(go.language(), "Go");
1901 let plain = prepare_document(&descriptor("hello\n", "not-a-language"), ®istry).unwrap();
1902 assert_eq!(plain.language(), "Plain Text");
1903 }
1904
1905 #[test]
1906 fn public_launch_language_pack_resolves_every_declared_language() {
1907 let registry = SyntaxRegistry::new();
1908 let mut missing = Vec::new();
1909 for language in [
1910 "rhai",
1911 "rust",
1912 "go",
1913 "javascript",
1914 "typescript",
1915 "tsx",
1916 "json",
1917 "jsonc",
1918 "toml",
1919 "yaml",
1920 "markdown",
1921 "bash",
1922 "python",
1923 "html",
1924 "css",
1925 "sql",
1926 "dockerfile",
1927 "RUST",
1928 ] {
1929 let document = prepare_document(&descriptor("value = 1\n", language), ®istry)
1930 .unwrap_or_else(|error| panic!("{language}: {error}"));
1931 if document.language() == "Plain Text" {
1932 missing.push(language);
1933 }
1934 }
1935 assert!(missing.is_empty(), "missing {missing:?}");
1936 }
1937
1938 #[test]
1939 fn line_ranges_normalize_crlf_but_retain_terminal_empty_line() {
1940 let registry = SyntaxRegistry::new();
1941 let document = prepare_document(&descriptor("one\r\ntwo\n", "text"), ®istry).unwrap();
1942 assert_eq!(document.lines().len(), 3);
1943 assert_eq!(document.line_text(0), Some("one"));
1944 assert_eq!(document.line_text(1), Some("two"));
1945 assert_eq!(document.line_text(2), Some(""));
1946 assert!(document.ends_with_newline());
1947 }
1948
1949 #[test]
1950 fn diff_is_direction_neutral_refines_unicode_and_collapses_context() {
1951 let registry = SyntaxRegistry::new();
1952 let left = descriptor(
1953 "same\n城市 = 东京\nunchanged 1\nunchanged 2\nunchanged 3\nunchanged 4\n",
1954 "rhai",
1955 );
1956 let right = descriptor(
1957 "same\n城市 = 上海\nunchanged 1\nunchanged 2\nunchanged 3\nunchanged 4\n",
1958 "rhai",
1959 );
1960 let diff = prepare_diff(&left, &right, DiffWhitespace::Exact, Some(1), ®istry).unwrap();
1961 assert_eq!(diff.hunk_count, 1);
1962 let modified = diff
1963 .rows
1964 .iter()
1965 .find(|row| row.kind == DiffRowKind::Modified)
1966 .unwrap();
1967 assert_ne!(modified.left_inline, [] as [std::ops::Range<usize>; 0]);
1968 assert_ne!(modified.right_inline, [] as [std::ops::Range<usize>; 0]);
1969 assert!(
1970 diff.collapsed
1971 .iter()
1972 .any(|row| matches!(row, DiffDisplayRow::Fold { .. }))
1973 );
1974 }
1975
1976 #[test]
1977 fn whitespace_policy_is_explicit() {
1978 let registry = SyntaxRegistry::new();
1979 let left = descriptor("value = 1\n", "rhai");
1980 let right = descriptor("value = 1\n", "rhai");
1981 assert_eq!(
1982 prepare_diff(&left, &right, DiffWhitespace::Exact, None, ®istry)
1983 .unwrap()
1984 .hunk_count,
1985 1
1986 );
1987 assert_eq!(
1988 prepare_diff(
1989 &left,
1990 &right,
1991 DiffWhitespace::IgnoreChanges,
1992 None,
1993 ®istry,
1994 )
1995 .unwrap()
1996 .hunk_count,
1997 0
1998 );
1999 }
2000
2001 #[test]
2002 fn host_limits_fail_explicitly_before_unbounded_work() {
2003 let registry = SyntaxRegistry::new();
2004 let descriptor = descriptor("12345", "text");
2005 let limits = DocumentLimits {
2006 max_document_bytes: 4,
2007 ..DocumentLimits::default()
2008 };
2009 assert!(matches!(
2010 prepare_document_with_limits(&descriptor, ®istry, limits),
2011 Err(DocumentError::TooManyBytes {
2012 actual: 5,
2013 limit: 4
2014 })
2015 ));
2016 let runtime = DocumentRuntimeConfig::new();
2017 assert_eq!(runtime.limits(), DocumentLimits::default());
2018 assert_eq!(
2019 runtime.set_limits(DocumentLimits {
2020 max_diff_hunks: 0,
2021 ..DocumentLimits::default()
2022 }),
2023 Err(DocumentError::InvalidLimits)
2024 );
2025 }
2026
2027 #[test]
2028 fn host_registered_syntax_participates_in_background_safe_snapshots() {
2029 let registry = SyntaxRegistry::new();
2030 registry
2031 .register_sublime_syntax(
2032 r"%YAML 1.2
2033---
2034name: Probe
2035file_extensions: [probe]
2036scope: source.probe
2037contexts:
2038 main:
2039 - match: '\bprobe\b'
2040 scope: keyword.control.probe
2041",
2042 )
2043 .unwrap();
2044 let document = prepare_document(&descriptor("probe value\n", "probe"), ®istry).unwrap();
2045 assert_eq!(document.language(), "Probe");
2046 assert!(
2047 document.lines()[0]
2048 .syntax
2049 .iter()
2050 .any(|span| span.kind == SyntaxTokenKind::Keyword)
2051 );
2052 }
2053}