code_moniker_core/lang/

mod.rs

pub mod build_manifest;
pub mod c;
pub mod callable;
pub mod cs;
mod document;
pub mod extractor;
pub mod go;
pub mod java;
pub mod kinds;
pub mod python;
pub mod rs;
pub mod sdk;
pub mod sql;
pub mod tree_util;
pub mod ts;

pub use document::{ParsedDocument, SyntaxInjection};
#[doc(hidden)]
pub use extractor::assert_conformance;
pub use extractor::{ExtractionContext, KindSpec, LangExtractor};

/// Adding a row registers the language for `Lang::from_tag` / `tag` /
/// `allowed_kinds` / `allowed_visibilities` and the conformance tests.
macro_rules! define_languages {
	($($(#[$attr:meta])* $variant:ident => $module:ty),* $(,)?) => {
		#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
			pub enum Lang {
			$(
				$(#[$attr])*
				$variant,
				)*
			}
			impl Lang {
			pub const ALL: &'static [Lang] = &[
				$(
					$(#[$attr])*
					Self::$variant,
				)*
				];
				pub fn from_tag(s: &str) -> Option<Self> {
				$(
					$(#[$attr])*
					if s == <$module as $crate::lang::LangExtractor>::LANG_TAG {
						return Some(Self::$variant);
					}
				)*
				None
			}

			pub fn tag(self) -> &'static str {
				match self {
					$(
						$(#[$attr])*
						Self::$variant => <$module as $crate::lang::LangExtractor>::LANG_TAG,
					)*
				}
			}

			pub fn allowed_kinds(self) -> &'static [&'static str] {
				match self {
					$(
						$(#[$attr])*
						Self::$variant => <$module as $crate::lang::LangExtractor>::ALLOWED_KINDS,
					)*
				}
			}

			pub fn kind_specs(self) -> &'static [$crate::lang::KindSpec] {
				match self {
					$(
						$(#[$attr])*
						Self::$variant => <$module as $crate::lang::LangExtractor>::KIND_SPECS,
					)*
				}
			}

			pub fn kind_spec(self, id: &str) -> Option<&'static $crate::lang::KindSpec> {
				self.kind_specs().iter().find(|spec| spec.id == id)
			}

			pub fn allowed_visibilities(self) -> &'static [&'static str] {
				match self {
					$(
						$(#[$attr])*
						Self::$variant => <$module as $crate::lang::LangExtractor>::ALLOWED_VISIBILITIES,
					)*
				}
			}

			pub fn ignores_visibility(self) -> bool {
				self.allowed_visibilities().is_empty()
			}

			pub fn file_root(
				self,
				uri: &str,
				anchor: &$crate::core::moniker::Moniker,
			) -> Option<$crate::core::moniker::Moniker> {
				match self {
					$(
						$(#[$attr])*
						Self::$variant => <$module as $crate::lang::LangExtractor>::file_root(uri, anchor),
					)*
				}
			}

			pub fn parse(self, uri: &str, source: &str) -> $crate::lang::ParsedDocument {
				match self {
					$(
						$(#[$attr])*
						Self::$variant => <$module as $crate::lang::LangExtractor>::parse(uri, source),
					)*
				}
			}
		}

		#[cfg(test)]
		mod _conformance_dispatch {
			use $crate::lang::LangExtractor;

			/// Dispatches a closure that takes `(lang_tag, allowed_kinds, allowed_visibilities)`
			/// over every registered language. Used by language conformance tests.
			pub(crate) fn for_each_language(
				mut f: impl FnMut(
					&'static str,
					&'static [&'static str],
					&'static [&'static str],
					&'static [$crate::lang::KindSpec],
				),
			) {
				$(
					$(#[$attr])*
					f(
						<$module as LangExtractor>::LANG_TAG,
						<$module as LangExtractor>::ALLOWED_KINDS,
						<$module as LangExtractor>::ALLOWED_VISIBILITIES,
						<$module as LangExtractor>::KIND_SPECS,
					);
				)*
			}
		}
	};
}

define_languages! {
	Ts     => crate::lang::ts::Lang,
	Rs     => crate::lang::rs::Lang,
	Java   => crate::lang::java::Lang,
	Python => crate::lang::python::Lang,
	Go     => crate::lang::go::Lang,
	C      => crate::lang::c::Lang,
	Cs     => crate::lang::cs::Lang,
	Sql    => crate::lang::sql::Lang,
}

/// Parse source text without indexing it.
///
/// Canonical language tags are the tags returned by [`Lang::tag`]. PL/pgSQL
/// is also accepted as a standalone embedded language.
pub fn parse_source(language: &str, uri: &str, source: &str) -> Option<ParsedDocument> {
	if language.eq_ignore_ascii_case("plpgsql") {
		return Some(sql::parse_plpgsql(source));
	}
	Lang::from_tag(language).map(|lang| lang.parse(uri, source))
}

#[cfg(test)]
pub(crate) use _conformance_dispatch::for_each_language;

#[cfg(test)]
mod language_registry_tests {
	use super::for_each_language;

	#[test]
	fn language_registry_matches_dispatch_table() {
		let mut visited = 0usize;
		for_each_language(|tag, kinds, visibilities, _| {
			visited += 1;
			let lang = super::Lang::from_tag(tag).expect("dispatch tag must resolve through Lang");
			assert_eq!(lang.tag(), tag);
			assert_eq!(lang.allowed_kinds(), kinds);
			assert_eq!(lang.allowed_visibilities(), visibilities);
		});

		assert_eq!(
			visited,
			super::Lang::ALL.len(),
			"for_each_language visited {visited} languages but Lang::ALL contains {}; the cfg gates of the dispatch table and the macro variants are out of sync",
			super::Lang::ALL.len()
		);
	}

	#[test]
	fn every_registered_language_parses_on_demand() {
		for lang in super::Lang::ALL {
			let document = lang.parse("empty", "");
			assert!(
				!document.primary().root_node().kind().is_empty(),
				"{} returned an empty root kind",
				lang.tag()
			);
		}
	}

	#[test]
	fn stateless_parser_accepts_standalone_plpgsql() {
		let document = super::parse_source(
			"plpgsql",
			"snippet.plpgsql",
			"DECLARE total numeric; BEGIN total := 1; RETURN total; END;",
		)
		.expect("PL/pgSQL parser");
		let root = document.primary().root_node();
		assert_eq!(root.kind(), "source_file");
		assert!(!root.has_error());
	}
}

#[cfg(test)]
mod shape_coverage_tests {
	use super::for_each_language;
	use crate::core::shape::shape_of;

	#[test]
	fn every_allowed_kind_has_a_shape() {
		let mut missing: Vec<(String, String)> = Vec::new();
		for_each_language(|tag, kinds, _, _| {
			for k in kinds {
				if shape_of(k.as_bytes()).is_none() {
					missing.push((tag.to_string(), (*k).to_string()));
				}
			}
		});
		assert!(
			missing.is_empty(),
			"kinds in ALLOWED_KINDS without an entry in core::shape::SHAPE_TABLE: {missing:?}"
		);
	}

	#[test]
	fn internal_kinds_have_a_shape() {
		for k in [b"module".as_slice(), b"comment", b"local", b"param"] {
			assert!(
				shape_of(k).is_some(),
				"internal kind {:?} must have a shape entry",
				std::str::from_utf8(k).unwrap()
			);
		}
	}
}

#[cfg(test)]
mod kind_contract_tests {
	use super::for_each_language;
	use crate::core::shape::shape_of;

	#[test]
	fn every_language_kind_spec_matches_allowed_kinds() {
		for_each_language(|tag, kinds, _, specs| {
			let spec_ids: Vec<_> = specs.iter().map(|spec| spec.id).collect();
			assert_eq!(
				sort(&spec_ids),
				sort(kinds),
				"{tag} KIND_SPECS must describe exactly ALLOWED_KINDS"
			);
		});
	}

	#[test]
	fn every_language_kind_spec_has_stable_semantics() {
		for_each_language(|tag, _, _, specs| {
			assert!(!specs.is_empty(), "{tag} must declare kind specs");
			let mut seen_ids = std::collections::HashSet::new();
			for spec in specs {
				assert!(
					seen_ids.insert(spec.id),
					"{tag} duplicates kind spec `{}`",
					spec.id
				);
				assert!(
					!spec.label.is_empty(),
					"{tag} kind `{}` has no label",
					spec.id
				);
				assert_ne!(spec.order, 0, "{tag} kind `{}` has no order", spec.id);
				assert_eq!(
					shape_of(spec.id.as_bytes()),
					Some(spec.shape),
					"{tag} kind `{}` shape must stay aligned with core shape taxonomy",
					spec.id
				);
			}
		});
	}

	fn sort<'a>(xs: &[&'a str]) -> Vec<&'a str> {
		let mut v: Vec<&str> = xs.to_vec();
		v.sort_unstable();
		v
	}
}

#[cfg(test)]
mod comment_collapse_tests {
	use crate::core::moniker::MonikerBuilder;

	struct Case {
		tag: &'static str,
		uri: &'static str,
		run: fn(&'static str) -> crate::core::code_graph::CodeGraph,
	}

	fn anchor() -> crate::core::moniker::Moniker {
		MonikerBuilder::new().project(b"app").build()
	}

	fn cases() -> Vec<Case> {
		vec![
			Case {
				tag: "rs",
				uri: "test.rs",
				run: |src| {
					super::rs::extract(
						"test.rs",
						src,
						&anchor(),
						false,
						&super::rs::Presets::default(),
					)
				},
			},
			Case {
				tag: "ts",
				uri: "test.ts",
				run: |src| {
					super::ts::extract(
						"test.ts",
						src,
						&anchor(),
						false,
						&super::ts::Presets::default(),
					)
				},
			},
			Case {
				tag: "python",
				uri: "test.py",
				run: |src| {
					super::python::extract(
						"test.py",
						src,
						&anchor(),
						false,
						&super::python::Presets::default(),
					)
				},
			},
			Case {
				tag: "go",
				uri: "test.go",
				run: |src| {
					super::go::extract(
						"test.go",
						src,
						&anchor(),
						false,
						&super::go::Presets::default(),
					)
				},
			},
			Case {
				tag: "java",
				uri: "test.java",
				run: |src| {
					super::java::extract(
						"test.java",
						src,
						&anchor(),
						false,
						&super::java::Presets::default(),
					)
				},
			},
			Case {
				tag: "c",
				uri: "test.c",
				run: |src| {
					super::c::extract(
						"test.c",
						src,
						&anchor(),
						false,
						&super::c::Presets::default(),
					)
				},
			},
			Case {
				tag: "cs",
				uri: "test.cs",
				run: |src| {
					super::cs::extract(
						"test.cs",
						src,
						&anchor(),
						false,
						&super::cs::Presets::default(),
					)
				},
			},
			Case {
				tag: "sql",
				uri: "test.sql",
				run: |src| {
					super::sql::extract(
						"test.sql",
						src,
						&anchor(),
						false,
						&super::sql::Presets::default(),
					)
				},
			},
		]
	}

	const ADJACENT: &[(&str, &str)] = &[
		("rs", "// a\n// b\n// c\nstruct Foo;\n"),
		("ts", "// a\n// b\n// c\nclass Foo {}"),
		("python", "# a\n# b\n# c\nclass Foo: pass\n"),
		("go", "package x\n// a\n// b\n// c\nfunc Foo() {}\n"),
		("java", "// a\n// b\n// c\nclass Foo {}\n"),
		("c", "// a\n// b\n// c\nint foo(void) { return 0; }\n"),
		("cs", "// a\n// b\n// c\nclass Foo {}\n"),
		(
			"sql",
			"-- a\n-- b\n-- c\nCREATE FUNCTION f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$;\n",
		),
	];

	const SPLIT_BY_BLANK: &[(&str, &str)] = &[
		("rs", "// a\n// b\n\n// c\nstruct Foo;\n"),
		("ts", "// a\n// b\n\n// c\nclass Foo {}"),
		("python", "# a\n# b\n\n# c\nclass Foo: pass\n"),
		("go", "package x\n// a\n// b\n\n// c\nfunc Foo() {}\n"),
		("java", "// a\n// b\n\n// c\nclass Foo {}\n"),
		("c", "// a\n// b\n\n// c\nint foo(void) { return 0; }\n"),
		("cs", "// a\n// b\n\n// c\nclass Foo {}\n"),
		(
			"sql",
			"-- a\n-- b\n\n-- c\nCREATE FUNCTION f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$;\n",
		),
	];

	fn count_comments(g: &crate::core::code_graph::CodeGraph) -> usize {
		g.defs().filter(|d| d.kind == b"comment").count()
	}

	#[test]
	fn each_language_collapses_three_adjacent_line_comments_into_one_def() {
		for case in cases() {
			let src = ADJACENT
				.iter()
				.find(|(tag, _)| *tag == case.tag)
				.expect("adjacent fixture")
				.1;
			let g = (case.run)(src);
			assert_eq!(
				count_comments(&g),
				1,
				"lang={} ({}): three adjacent line comments must collapse to one def",
				case.tag,
				case.uri
			);
		}
	}

	#[test]
	fn each_language_splits_runs_on_blank_line() {
		for case in cases() {
			let src = SPLIT_BY_BLANK
				.iter()
				.find(|(tag, _)| *tag == case.tag)
				.expect("blank-line fixture")
				.1;
			let g = (case.run)(src);
			assert_eq!(
				count_comments(&g),
				2,
				"lang={} ({}): blank line must break the run into two defs",
				case.tag,
				case.uri
			);
		}
	}
}