use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use crate::environment::installed_executable_path;
use crate::grammar::{
DeclarationRules, DirectoryNamespaceMode, ExternalDriverPayload, GrammarCapabilities,
GrammarSpec,
};
use crate::layout::gritpack_state_root;
use crate::models::{
InstalledPackage, PackageManifestFile, PackagesState, ReleaseResponse, ToolState,
};
use crate::support::{
existing_package_preferred, installed_package_identity_key,
installed_package_matches_selected_release, installed_tool_dirs,
normalized_manifest_target, package_manifest_identity_key, package_source_root,
version_satisfies,
};
use crate::{sanitize_relative_package_path, sha256_hex, CliError};
use glob::Pattern;
use walkdir::WalkDir;
const PROJECT_RESOLVER_CACHE_DIR: &str = "cache";
const PROJECT_RESOLVER_STATE_FILE: &str = "resolver-state.v1.bin";
const PROJECT_RESOLVER_STATE_SCHEMA_VERSION: u32 = 6;
#[derive(serde::Serialize, serde::Deserialize)]
struct PersistedPackagesStateV1 {
schema_version: u32,
packages_state: PackagesState,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct PersistedProjectResolverCache {
schema_version: u32,
cache: ProjectResolverCache,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct PersistedProjectResolverCacheV2 {
schema_version: u32,
cache: ProjectResolverCacheV2,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct PersistedProjectResolverCacheV3 {
schema_version: u32,
cache: ProjectResolverCacheV3,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct PersistedProjectResolverCacheV4 {
schema_version: u32,
cache: ProjectResolverCacheV4,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
pub(crate) struct ProjectResolverCache {
packages_state: PackagesState,
#[serde(default)]
snapshot_fingerprint: Option<String>,
#[serde(default)]
command_index: BTreeMap<String, Vec<ProjectExecutableRecord>>,
#[serde(default)]
package_index: BTreeMap<String, ProjectPackageRecord>,
#[serde(default)]
identity_index: BTreeMap<(String, String, String, String), ProjectInstalledPackageRecord>,
#[serde(default)]
install_path_index: BTreeMap<String, ProjectInstalledPackageRecord>,
#[serde(default)]
local_path_dirs: Vec<String>,
#[serde(default)]
participating_files: Vec<ProjectParticipatingFileRecord>,
#[serde(default)]
grammar_segments: Vec<ProjectGrammarSegmentRecord>,
#[serde(skip, default)]
runtime_stale: bool,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct ProjectResolverCacheV2 {
packages_state: PackagesState,
#[serde(default)]
snapshot_fingerprint: Option<String>,
#[serde(default)]
command_index: BTreeMap<String, Vec<ProjectExecutableRecord>>,
#[serde(default)]
package_index: BTreeMap<String, ProjectPackageRecord>,
#[serde(default)]
identity_index: BTreeMap<(String, String, String, String), ProjectInstalledPackageRecord>,
#[serde(default)]
install_path_index: BTreeMap<String, ProjectInstalledPackageRecord>,
#[serde(default)]
local_path_dirs: Vec<String>,
#[serde(default)]
participating_files: Vec<ProjectParticipatingFileRecord>,
#[serde(default)]
grammar_segments: Vec<ProjectGrammarSegmentRecordV2>,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct ProjectResolverCacheV3 {
packages_state: PackagesState,
#[serde(default)]
snapshot_fingerprint: Option<String>,
#[serde(default)]
command_index: BTreeMap<String, Vec<ProjectExecutableRecord>>,
#[serde(default)]
package_index: BTreeMap<String, ProjectPackageRecord>,
#[serde(default)]
identity_index: BTreeMap<(String, String, String, String), ProjectInstalledPackageRecord>,
#[serde(default)]
install_path_index: BTreeMap<String, ProjectInstalledPackageRecord>,
#[serde(default)]
local_path_dirs: Vec<String>,
#[serde(default)]
participating_files: Vec<ProjectParticipatingFileRecord>,
#[serde(default)]
grammar_segments: Vec<ProjectGrammarSegmentRecordV3>,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct ProjectResolverCacheV4 {
packages_state: PackagesState,
#[serde(default)]
snapshot_fingerprint: Option<String>,
#[serde(default)]
command_index: BTreeMap<String, Vec<ProjectExecutableRecord>>,
#[serde(default)]
package_index: BTreeMap<String, ProjectPackageRecord>,
#[serde(default)]
identity_index: BTreeMap<(String, String, String, String), ProjectInstalledPackageRecord>,
#[serde(default)]
install_path_index: BTreeMap<String, ProjectInstalledPackageRecord>,
#[serde(default)]
local_path_dirs: Vec<String>,
#[serde(default)]
participating_files: Vec<ProjectParticipatingFileRecord>,
#[serde(default)]
grammar_segments: Vec<ProjectGrammarSegmentRecordV4>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub(crate) struct ProjectParticipatingFileRecord {
pub(crate) package_name: String,
#[serde(default)]
pub(crate) package_version: String,
#[serde(default)]
pub(crate) package_dialect: String,
#[serde(default = "default_package_target_identity")]
pub(crate) package_target: String,
pub(crate) direct: bool,
#[serde(default)]
pub(crate) source_root_path: String,
#[serde(default)]
pub(crate) package_relative_path: String,
#[serde(default)]
pub(crate) content_fingerprint: Option<String>,
pub(crate) path: String,
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
pub(crate) struct ProjectGrammarCapabilitiesRecord {
pub(crate) authoritative_negative_lookup: bool,
pub(crate) supports_prefix_queries: bool,
pub(crate) supports_child_enumeration: bool,
pub(crate) supports_multi_file_modules: bool,
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
pub(crate) struct ProjectGrammarCompletenessRecord {
pub(crate) negative_lookup_authoritative: bool,
pub(crate) prefix_authoritative: bool,
pub(crate) child_enumeration_authoritative: bool,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub(crate) struct ProjectGrammarDescriptorRecord {
pub(crate) dialect: String,
pub(crate) name: String,
pub(crate) version: u32,
#[serde(default)]
pub(crate) capabilities: ProjectGrammarCapabilitiesRecord,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub(crate) enum ProjectGrammarSegmentState {
Unavailable,
Loaded,
Stale,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub(crate) struct ProjectGrammarSegmentRecord {
pub(crate) descriptor: ProjectGrammarDescriptorRecord,
pub(crate) state: ProjectGrammarSegmentState,
#[serde(default = "default_namespace_separator")]
pub(crate) namespace_separator: String,
#[serde(default = "default_case_sensitive")]
pub(crate) case_sensitive: bool,
#[serde(default)]
pub(crate) completeness: ProjectGrammarCompletenessRecord,
#[serde(default)]
pub(crate) module_entries: Vec<ProjectGrammarModuleEntryRecord>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct ProjectGrammarSegmentRecordV2 {
descriptor: ProjectGrammarDescriptorRecord,
state: ProjectGrammarSegmentState,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct ProjectGrammarSegmentRecordV3 {
descriptor: ProjectGrammarDescriptorRecord,
state: ProjectGrammarSegmentState,
#[serde(default = "default_namespace_separator")]
namespace_separator: String,
#[serde(default = "default_case_sensitive")]
case_sensitive: bool,
#[serde(default)]
module_entries: Vec<ProjectGrammarModuleEntryRecord>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct ProjectGrammarSegmentRecordV4 {
descriptor: ProjectGrammarDescriptorRecord,
state: ProjectGrammarSegmentState,
#[serde(default = "default_namespace_separator")]
namespace_separator: String,
#[serde(default = "default_case_sensitive")]
case_sensitive: bool,
#[serde(default)]
module_entries: Vec<ProjectGrammarModuleEntryRecord>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub(crate) struct ProjectGrammarModuleEntryRecord {
pub(crate) module_name: String,
pub(crate) package_name: String,
#[serde(default)]
pub(crate) direct: bool,
pub(crate) file_path: String,
}
fn default_namespace_separator() -> String {
"::".to_string()
}
fn default_case_sensitive() -> bool {
true
}
fn default_package_target_identity() -> String {
"generic".to_string()
}
type ExternalDriverPayloadIndex = BTreeMap<String, BTreeMap<String, Vec<String>>>;
impl Default for ProjectResolverCache {
fn default() -> Self {
Self {
packages_state: PackagesState::default(),
snapshot_fingerprint: None,
command_index: BTreeMap::new(),
package_index: BTreeMap::new(),
identity_index: BTreeMap::new(),
install_path_index: BTreeMap::new(),
local_path_dirs: Vec::new(),
participating_files: Vec::new(),
grammar_segments: Vec::new(),
runtime_stale: false,
}
}
}
#[derive(Clone, Default, serde::Serialize, serde::Deserialize)]
struct ProjectPackageRecord {
#[serde(default)]
executables: Vec<ProjectExecutableRecord>,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct ProjectExecutableRecord {
name: String,
path: String,
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct ProjectInstalledPackageRecord {
package: InstalledPackage,
}
impl ProjectResolverCache {
pub(crate) fn from_packages_state(packages_state: &PackagesState) -> Result<Self, CliError> {
Self::from_packages_state_with_grammar_specs(packages_state, &[])
}
pub(crate) fn from_packages_state_with_grammar_specs(
packages_state: &PackagesState,
grammar_specs: &[GrammarSpec],
) -> Result<Self, CliError> {
Self::from_packages_state_with_grammar_specs_and_external_driver_payloads(
packages_state,
grammar_specs,
&[],
)
}
pub(crate) fn from_packages_state_with_grammar_specs_and_external_driver_payloads(
packages_state: &PackagesState,
grammar_specs: &[GrammarSpec],
external_driver_payloads: &[ExternalDriverPayload],
) -> Result<Self, CliError> {
let mut command_index = BTreeMap::<String, Vec<ProjectExecutableRecord>>::new();
let mut package_index = BTreeMap::<String, ProjectPackageRecord>::new();
let mut identity_index =
BTreeMap::<(String, String, String, String), ProjectInstalledPackageRecord>::new();
let mut install_path_index = BTreeMap::<String, ProjectInstalledPackageRecord>::new();
let mut local_path_dirs = Vec::<String>::new();
for package in &packages_state.installs {
let executable_records = package
.executables
.iter()
.map(|executable| {
Ok(ProjectExecutableRecord {
name: executable.name.clone(),
path: installed_executable_path(package, executable)?
.to_string_lossy()
.to_string(),
})
})
.collect::<Result<Vec<_>, CliError>>()?;
for executable in &executable_records {
command_index
.entry(executable.name.clone())
.or_default()
.push(executable.clone());
}
package_index.insert(
package.name.clone(),
ProjectPackageRecord {
executables: executable_records,
},
);
install_path_index.insert(
package.install_path.clone(),
ProjectInstalledPackageRecord {
package: package.clone(),
},
);
let identity_key = installed_package_identity_key(package);
match identity_index.get(&identity_key) {
Some(current) if !existing_package_preferred(package, ¤t.package) => {}
_ => {
identity_index.insert(
identity_key,
ProjectInstalledPackageRecord {
package: package.clone(),
},
);
}
}
for dir in installed_tool_dirs(package)? {
let dir = dir.to_string_lossy().to_string();
if local_path_dirs.last() != Some(&dir) && !local_path_dirs.contains(&dir) {
local_path_dirs.push(dir);
}
}
}
let participating_files = participating_files_from_packages(packages_state)?;
let grammar_segments = grammar_segments_from_specs(
grammar_specs,
&participating_files,
external_driver_payloads,
)?;
Ok(Self {
packages_state: packages_state.clone(),
snapshot_fingerprint: Some(project_snapshot_fingerprint(
packages_state,
&participating_files,
&grammar_segments,
)),
command_index,
package_index,
identity_index,
install_path_index,
local_path_dirs,
participating_files,
grammar_segments,
runtime_stale: false,
})
}
pub(crate) fn packages_state(&self) -> &PackagesState {
&self.packages_state
}
pub(crate) fn snapshot_fingerprint(&self) -> Option<&str> {
if self.runtime_stale {
None
} else {
self.snapshot_fingerprint.as_deref()
}
}
pub(crate) fn local_path_dirs(&self) -> Vec<PathBuf> {
self.local_path_dirs.iter().map(PathBuf::from).collect()
}
pub(crate) fn participating_files(&self) -> &[ProjectParticipatingFileRecord] {
&self.participating_files
}
pub(crate) fn grammar_segments(&self) -> &[ProjectGrammarSegmentRecord] {
&self.grammar_segments
}
pub(crate) fn is_runtime_stale(&self) -> bool {
self.runtime_stale
}
#[cfg(test)]
pub(crate) fn with_grammar_specs(
mut self,
grammar_specs: Vec<GrammarSpec>,
) -> Result<Self, CliError> {
self.grammar_segments = grammar_segments_from_specs(
&grammar_specs,
&self.participating_files,
&[],
)?;
Ok(self)
}
pub(crate) fn resolve_local_command(&self, name: &str) -> Result<Option<PathBuf>, CliError> {
if let Some(by_command) = self.command_index.get(name) {
if by_command.len() > 1 {
return Err(CliError::Message(format!(
"multiple local commands named {} are installed",
name
)));
}
if let Some(command) = by_command.first() {
return Ok(Some(PathBuf::from(&command.path)));
}
}
let Some(package) = self.package_index.get(name) else {
return Ok(None);
};
if package.executables.is_empty() {
return Err(CliError::Message(format!(
"local package {} does not expose a runnable command",
name
)));
}
if package.executables.len() > 1 {
return Err(CliError::Message(format!(
"local package {} exposes multiple commands; run one by command name",
name
)));
}
Ok(Some(PathBuf::from(&package.executables[0].path)))
}
pub(crate) fn find_matching_installed_package(
&self,
selected: &ReleaseResponse,
fallback_dialect: &str,
constraint: &str,
require_existing_path: bool,
) -> Option<InstalledPackage> {
let key = (
selected.name.clone(),
selected.version.clone(),
selected
.dialect
.clone()
.unwrap_or_else(|| fallback_dialect.to_string()),
normalized_manifest_target(selected.target.as_deref()).to_string(),
);
let existing = self.identity_index.get(&key)?.package.clone();
if !installed_package_matches_selected_release(&existing, selected, fallback_dialect)
|| !version_satisfies(&existing.version, constraint)
{
return None;
}
if require_existing_path && !Path::new(&existing.install_path).exists() {
return None;
}
Some(existing)
}
pub(crate) fn preserved_package_for_manifest(
&self,
package_root: &Path,
manifest: &PackageManifestFile,
) -> Option<InstalledPackage> {
let package_root = package_root.to_string_lossy().to_string();
self.install_path_index
.get(&package_root)
.map(|record| record.package.clone())
.or_else(|| {
self.identity_index
.get(&package_manifest_identity_key(manifest))
.map(|record| record.package.clone())
})
}
}
pub(crate) fn packages_state_path(root: &Path) -> PathBuf {
project_cache_root(root).join(PROJECT_RESOLVER_STATE_FILE)
}
fn project_cache_root(install_root: &Path) -> PathBuf {
install_root
.parent()
.map(|gritpack_root| gritpack_root.join(PROJECT_RESOLVER_CACHE_DIR))
.unwrap_or_else(|| install_root.join(PROJECT_RESOLVER_CACHE_DIR))
}
pub(crate) async fn load_project_resolver_cache(
install_root: &Path,
) -> Result<ProjectResolverCache, CliError> {
let packages_path = packages_state_path(install_root);
if packages_path.exists() {
let bytes = tokio::fs::read(&packages_path).await?;
if let Ok(persisted) = bincode::deserialize::<PersistedProjectResolverCache>(&bytes) {
if persisted.schema_version == PROJECT_RESOLVER_STATE_SCHEMA_VERSION {
return Ok(apply_runtime_staleness(persisted.cache));
}
if persisted.schema_version == 5 {
return Ok(apply_runtime_staleness(ProjectResolverCache::from_v5(
persisted.cache,
)));
}
}
if let Ok(persisted) = bincode::deserialize::<PersistedProjectResolverCacheV2>(&bytes) {
if persisted.schema_version == 2 {
return Ok(apply_runtime_staleness(ProjectResolverCache::from_v2(
persisted.cache,
)));
}
}
if let Ok(persisted) = bincode::deserialize::<PersistedProjectResolverCacheV3>(&bytes) {
if persisted.schema_version == 3 {
return Ok(apply_runtime_staleness(ProjectResolverCache::from_v3(
persisted.cache,
)));
}
}
if let Ok(persisted) = bincode::deserialize::<PersistedProjectResolverCacheV4>(&bytes) {
if persisted.schema_version == 4 {
return Ok(apply_runtime_staleness(ProjectResolverCache::from_v4(
persisted.cache,
)));
}
}
let persisted = bincode::deserialize::<PersistedPackagesStateV1>(&bytes)?;
if persisted.schema_version != 1 {
return Err(CliError::Message(format!(
"unsupported project resolver cache schema version {}",
persisted.schema_version
)));
}
return Ok(apply_runtime_staleness(ProjectResolverCache::from_packages_state(
&persisted.packages_state,
)?));
}
Ok(ProjectResolverCache::default())
}
#[cfg(test)]
pub(crate) async fn load_packages_state(install_root: &Path) -> Result<PackagesState, CliError> {
Ok(load_project_resolver_cache(install_root)
.await?
.packages_state()
.clone())
}
pub(crate) async fn write_state_files(
install_root: &Path,
packages_state: &PackagesState,
) -> Result<(), CliError> {
write_state_files_with_grammar_specs(install_root, packages_state, &[]).await
}
pub(crate) async fn write_state_files_with_grammar_specs(
install_root: &Path,
packages_state: &PackagesState,
grammar_specs: &[GrammarSpec],
) -> Result<(), CliError> {
write_state_files_with_grammar_specs_and_external_driver_payloads(
install_root,
packages_state,
grammar_specs,
&[],
)
.await
}
pub(crate) async fn write_state_files_with_grammar_specs_and_external_driver_payloads(
install_root: &Path,
packages_state: &PackagesState,
grammar_specs: &[GrammarSpec],
external_driver_payloads: &[ExternalDriverPayload],
) -> Result<(), CliError> {
let packages_path = packages_state_path(install_root);
if let Some(parent) = packages_path.parent() {
tokio::fs::create_dir_all(parent).await?;
}
let persisted = PersistedProjectResolverCache {
schema_version: PROJECT_RESOLVER_STATE_SCHEMA_VERSION,
cache: ProjectResolverCache::from_packages_state_with_grammar_specs_and_external_driver_payloads(
packages_state,
grammar_specs,
external_driver_payloads,
)?,
};
tokio::fs::write(&packages_path, bincode::serialize(&persisted)?).await?;
Ok(())
}
fn grammar_segments_from_specs(
grammar_specs: &[GrammarSpec],
participating_files: &[ProjectParticipatingFileRecord],
external_driver_payloads: &[ExternalDriverPayload],
) -> Result<Vec<ProjectGrammarSegmentRecord>, CliError> {
let mut seen = BTreeSet::new();
let mut segments = Vec::with_capacity(grammar_specs.len());
let external_driver_payload_index =
build_external_driver_payload_index(external_driver_payloads)?;
for grammar_spec in grammar_specs {
validate_grammar_spec(grammar_spec)?;
let grammar_key = (
grammar_spec.id.dialect.clone(),
grammar_spec.id.name.clone(),
grammar_spec.id.version,
);
if !seen.insert(grammar_key.clone()) {
return Err(CliError::Message(format!(
"duplicate grammar spec {}:{}@{}",
grammar_key.0, grammar_key.1, grammar_key.2
)));
}
let module_entries = compile_path_module_entries(
grammar_spec,
participating_files,
&external_driver_payload_index,
)?;
let state = if participating_files.is_empty() {
ProjectGrammarSegmentState::Unavailable
} else {
ProjectGrammarSegmentState::Loaded
};
segments.push(ProjectGrammarSegmentRecord {
descriptor: ProjectGrammarDescriptorRecord {
dialect: grammar_spec.id.dialect.clone(),
name: grammar_spec.id.name.clone(),
version: grammar_spec.id.version,
capabilities: grammar_capabilities_record(&grammar_spec.capabilities),
},
state,
namespace_separator: grammar_spec.namespace.separator.clone(),
case_sensitive: grammar_spec.namespace.case_sensitive,
completeness: grammar_completeness_record(&grammar_spec.completeness),
module_entries,
});
}
Ok(segments)
}
fn compile_path_module_entries(
grammar_spec: &GrammarSpec,
participating_files: &[ProjectParticipatingFileRecord],
external_driver_payload_index: &ExternalDriverPayloadIndex,
) -> Result<Vec<ProjectGrammarModuleEntryRecord>, CliError> {
let include_patterns = compile_patterns(&grammar_spec.source_files.include)?;
let exclude_patterns = compile_patterns(&grammar_spec.source_files.exclude)?;
let allowed_extensions = grammar_spec
.source_files
.extensions
.iter()
.map(|extension| extension.trim_start_matches('.').to_string())
.collect::<Vec<_>>();
let mut seen = BTreeSet::new();
let mut entries = Vec::new();
for record in participating_files {
let module_names = module_names_for_file(
record,
grammar_spec,
&include_patterns,
&exclude_patterns,
&allowed_extensions,
external_driver_payload_index,
)?;
for module_name in module_names {
let entry_key = (
module_name.clone(),
record.package_name.clone(),
record.path.clone(),
);
if seen.insert(entry_key) {
entries.push(ProjectGrammarModuleEntryRecord {
module_name,
package_name: record.package_name.clone(),
direct: record.direct,
file_path: record.path.clone(),
});
}
}
}
Ok(entries)
}
fn compile_patterns(patterns: &[crate::grammar::PathPattern]) -> Result<Vec<Pattern>, CliError> {
patterns
.iter()
.map(|pattern| Pattern::new(&pattern.value).map_err(|error| CliError::Message(error.to_string())))
.collect()
}
fn module_names_for_file(
record: &ProjectParticipatingFileRecord,
grammar_spec: &GrammarSpec,
include_patterns: &[Pattern],
exclude_patterns: &[Pattern],
allowed_extensions: &[String],
external_driver_payload_index: &ExternalDriverPayloadIndex,
) -> Result<Vec<String>, CliError> {
let relative_path = Path::new(&record.path)
.strip_prefix(&record.source_root_path)
.map_err(|error| CliError::Message(error.to_string()))?;
let relative_string = relative_path.to_string_lossy().replace('\\', "/");
let source_root_name = Path::new(&record.source_root_path)
.file_name()
.and_then(|value| value.to_str())
.unwrap_or_default();
let package_relative_string = if source_root_name.is_empty() {
relative_string.clone()
} else if relative_string.is_empty() {
source_root_name.to_string()
} else {
format!("{source_root_name}/{relative_string}")
};
if !allowed_extensions.is_empty() {
let Some(extension) = relative_path.extension().and_then(|value| value.to_str()) else {
return Ok(Vec::new());
};
if !allowed_extensions.iter().any(|allowed| allowed == extension) {
return Ok(Vec::new());
}
}
if !include_patterns.is_empty()
&& !include_patterns.iter().any(|pattern| {
pattern.matches(&relative_string) || pattern.matches(&package_relative_string)
})
{
return Ok(Vec::new());
}
if exclude_patterns.iter().any(|pattern| {
pattern.matches(&relative_string) || pattern.matches(&package_relative_string)
}) {
return Ok(Vec::new());
}
let declaration_names = declaration_module_names(
record,
grammar_spec,
external_driver_payload_index,
)?;
if !declaration_names.is_empty() {
return Ok(declaration_names);
}
let DeclarationRules::ExternalDriver(rule) = &grammar_spec.declarations else {
return Ok(Vec::new());
};
if external_driver_payload_index.contains_key(&rule.name) {
return Ok(Vec::new());
}
Ok(path_fallback_module_name(relative_path, grammar_spec)?.into_iter().collect())
}
fn declaration_module_names(
record: &ProjectParticipatingFileRecord,
grammar_spec: &GrammarSpec,
external_driver_payload_index: &ExternalDriverPayloadIndex,
) -> Result<Vec<String>, CliError> {
let source = std::fs::read_to_string(&record.path)?;
let names = match &grammar_spec.declarations {
DeclarationRules::TokenSequence(rule) => {
extract_token_sequence_names(&source, rule.tokens.as_slice(), grammar_spec)
}
DeclarationRules::PatternSet(patterns) => patterns
.iter()
.flat_map(|pattern| extract_pattern_names(&source, &pattern.pattern, grammar_spec))
.collect(),
DeclarationRules::ExternalDriver(rule) => external_driver_declared_names(
record,
grammar_spec,
&rule.name,
external_driver_payload_index,
),
};
Ok(dedup_strings(names))
}
fn build_external_driver_payload_index(
payloads: &[ExternalDriverPayload],
) -> Result<ExternalDriverPayloadIndex, CliError> {
let mut index = BTreeMap::<String, BTreeMap<String, Vec<String>>>::new();
for payload in payloads {
let driver_name = payload.driver_name.trim();
if driver_name.is_empty() {
return Err(CliError::Message(
"external driver payload name cannot be empty".to_string(),
));
}
let driver_files = index.entry(driver_name.to_string()).or_default();
for file in &payload.files {
let normalized_path = normalize_driver_payload_path(&file.file_path);
if normalized_path.is_empty() {
return Err(CliError::Message(format!(
"external driver payload {} contains an empty file path",
driver_name
)));
}
if driver_files.contains_key(&normalized_path) {
return Err(CliError::Message(format!(
"external driver payload {} contains duplicate file path {}",
driver_name, normalized_path
)));
}
driver_files.insert(normalized_path, file.declared_names.clone());
}
}
Ok(index)
}
fn external_driver_declared_names(
record: &ProjectParticipatingFileRecord,
grammar_spec: &GrammarSpec,
driver_name: &str,
external_driver_payload_index: &ExternalDriverPayloadIndex,
) -> Vec<String> {
let Some(driver_files) = external_driver_payload_index.get(driver_name) else {
return Vec::new();
};
let Some(declared_names) = driver_files.get(&normalize_driver_payload_path(&record.path)) else {
return Vec::new();
};
declared_names
.iter()
.filter_map(|name| normalize_external_driver_declared_name(name, grammar_spec))
.collect()
}
fn normalize_driver_payload_path(path: &str) -> String {
path.trim().replace('\\', "/")
}
fn normalize_external_driver_declared_name(
name: &str,
grammar_spec: &GrammarSpec,
) -> Option<String> {
let trimmed = name.trim();
if trimmed.is_empty() {
return None;
}
let mut normalized = trimmed.to_string();
if !grammar_spec.namespace.case_sensitive {
normalized = normalized.to_lowercase();
}
Some(normalized)
}
fn extract_token_sequence_names(
source: &str,
tokens: &[String],
grammar_spec: &GrammarSpec,
) -> Vec<String> {
if tokens.is_empty() {
return Vec::new();
}
let prefix = tokens.join(" ");
source
.lines()
.filter_map(|line| {
let trimmed = line.trim_start();
trimmed.strip_prefix(&prefix)
})
.filter_map(|remainder| extract_declared_name(remainder, grammar_spec))
.collect()
}
fn extract_pattern_names(source: &str, pattern: &str, grammar_spec: &GrammarSpec) -> Vec<String> {
source
.lines()
.filter_map(|line| line.find(pattern).map(|index| &line[index + pattern.len()..]))
.filter_map(|remainder| extract_declared_name(remainder, grammar_spec))
.collect()
}
fn extract_declared_name(remainder: &str, grammar_spec: &GrammarSpec) -> Option<String> {
let trimmed = remainder.trim_start();
if trimmed.is_empty() {
return None;
}
let mut end = 0usize;
while end < trimmed.len() {
let rest = &trimmed[end..];
if rest.starts_with(&grammar_spec.namespace.separator) {
end += grammar_spec.namespace.separator.len();
continue;
}
let Some(ch) = rest.chars().next() else {
break;
};
if ch.is_alphanumeric() || ch == '_' {
end += ch.len_utf8();
continue;
}
break;
}
if end == 0 {
return None;
}
let mut name = trimmed[..end].to_string();
if !grammar_spec.namespace.case_sensitive {
name = name.to_lowercase();
}
Some(name)
}
fn dedup_strings(strings: Vec<String>) -> Vec<String> {
let mut seen = BTreeSet::new();
let mut result = Vec::new();
for value in strings {
if seen.insert(value.clone()) {
result.push(value);
}
}
result
}
fn path_fallback_module_name(
relative_path: &Path,
grammar_spec: &GrammarSpec,
) -> Result<Option<String>, CliError> {
let stem = relative_path
.file_stem()
.and_then(|value| value.to_str())
.ok_or_else(|| CliError::Message(format!("unable to derive module name from {}", relative_path.display())))?;
let mut segments = Vec::new();
if matches!(grammar_spec.namespace.directory_layout, DirectoryNamespaceMode::AppendDirectories)
{
if let Some(parent) = relative_path.parent() {
for component in parent.components() {
let std::path::Component::Normal(value) = component else {
continue;
};
let segment = value.to_string_lossy().to_string();
if grammar_spec.namespace.allow_empty_segments || !segment.is_empty() {
segments.push(segment);
}
}
}
}
if grammar_spec.namespace.allow_empty_segments || !stem.is_empty() {
segments.push(stem.to_string());
}
if segments.is_empty() {
return Ok(None);
}
if !grammar_spec.namespace.case_sensitive {
for segment in &mut segments {
*segment = segment.to_lowercase();
}
}
Ok(Some(segments.join(&grammar_spec.namespace.separator)))
}
fn participating_files_from_packages(
packages_state: &PackagesState,
) -> Result<Vec<ProjectParticipatingFileRecord>, CliError> {
let mut seen = BTreeSet::new();
let mut records = Vec::new();
for package in &packages_state.installs {
let source_root = package_source_root(
Path::new(&package.install_path),
package.source_root.as_deref(),
)?;
if !source_root.exists() {
continue;
}
if source_root.is_file() {
let path = source_root.to_string_lossy().to_string();
if seen.insert((package.name.clone(), path.clone())) {
records.push(ProjectParticipatingFileRecord {
package_name: package.name.clone(),
package_version: package.version.clone(),
package_dialect: package.dialect.clone(),
package_target: normalized_manifest_target(package.target.as_deref()).to_string(),
direct: package.direct,
source_root_path: source_root.to_string_lossy().to_string(),
package_relative_path: canonical_package_relative_path(&source_root, &source_root)?,
content_fingerprint: Some(file_content_fingerprint(&source_root)?),
path,
});
}
continue;
}
for entry in WalkDir::new(&source_root)
.sort_by_file_name()
.into_iter()
.filter_map(Result::ok)
{
if !entry.file_type().is_file() {
continue;
}
let path = entry.path().to_string_lossy().to_string();
if seen.insert((package.name.clone(), path.clone())) {
records.push(ProjectParticipatingFileRecord {
package_name: package.name.clone(),
package_version: package.version.clone(),
package_dialect: package.dialect.clone(),
package_target: normalized_manifest_target(package.target.as_deref()).to_string(),
direct: package.direct,
source_root_path: source_root.to_string_lossy().to_string(),
package_relative_path: canonical_package_relative_path(&source_root, entry.path())?,
content_fingerprint: Some(file_content_fingerprint(entry.path())?),
path,
});
}
}
}
records.sort_by(|left, right| {
(
left.package_name.as_str(),
left.package_version.as_str(),
left.package_dialect.as_str(),
left.package_target.as_str(),
left.package_relative_path.as_str(),
)
.cmp(&(
right.package_name.as_str(),
right.package_version.as_str(),
right.package_dialect.as_str(),
right.package_target.as_str(),
right.package_relative_path.as_str(),
))
});
Ok(records)
}
fn canonical_package_relative_path(source_root: &Path, file_path: &Path) -> Result<String, CliError> {
let relative_base = if source_root.is_file() {
source_root.parent().unwrap_or(source_root)
} else {
source_root
};
let relative = file_path.strip_prefix(relative_base).unwrap_or(file_path);
let sanitized = sanitize_relative_package_path(relative)?;
Ok(sanitized.to_string_lossy().replace('\\', "/"))
}
fn file_content_fingerprint(path: &Path) -> Result<String, CliError> {
let bytes = std::fs::read(path)?;
Ok(format!("sha256:{}", sha256_hex(&bytes)))
}
fn project_snapshot_fingerprint(
packages_state: &PackagesState,
participating_files: &[ProjectParticipatingFileRecord],
grammar_segments: &[ProjectGrammarSegmentRecord],
) -> String {
let mut lines = Vec::new();
for package in &packages_state.installs {
lines.push(format!(
"pkg\0{}\0{}\0{}\0{}\0{}\0{}",
package.name,
package.version,
package.dialect,
package.target.as_deref().unwrap_or("generic"),
package.package_uuid,
package.install_path,
));
}
for file in participating_files {
lines.push(format!(
"file\0{}\0{}\0{}\0{}\0{}\0{}\0{}\0{}\0{}",
file.package_name,
file.package_version,
file.package_dialect,
file.package_target,
if file.direct { "direct" } else { "transitive" },
file.package_relative_path,
file.source_root_path,
file.path,
file.content_fingerprint.as_deref().unwrap_or(""),
));
}
for segment in grammar_segments {
lines.push(format!(
"grammar\0{}\0{}\0{}\0{}\0{}\0{}\0{}\0{}\0{}\0{}\0{}\0{}",
segment.descriptor.dialect,
segment.descriptor.name,
segment.descriptor.version,
segment.namespace_separator,
segment.case_sensitive,
segment.completeness.negative_lookup_authoritative,
segment.completeness.prefix_authoritative,
segment.completeness.child_enumeration_authoritative,
segment
.descriptor
.capabilities
.authoritative_negative_lookup,
segment.descriptor.capabilities.supports_prefix_queries,
segment
.descriptor
.capabilities
.supports_child_enumeration,
segment.descriptor.capabilities.supports_multi_file_modules,
));
}
format!("sha256:{}", sha256_hex(lines.join("\n").as_bytes()))
}
impl ProjectResolverCache {
fn from_v5(mut cache: ProjectResolverCache) -> Self {
cache.participating_files = participating_files_from_packages(&cache.packages_state)
.unwrap_or(cache.participating_files);
mark_cache_stale(&mut cache);
cache
}
fn from_v2(cache: ProjectResolverCacheV2) -> Self {
Self {
packages_state: cache.packages_state,
snapshot_fingerprint: cache.snapshot_fingerprint,
command_index: cache.command_index,
package_index: cache.package_index,
identity_index: cache.identity_index,
install_path_index: cache.install_path_index,
local_path_dirs: cache.local_path_dirs,
participating_files: cache.participating_files,
grammar_segments: cache
.grammar_segments
.into_iter()
.map(|segment| ProjectGrammarSegmentRecord {
descriptor: segment.descriptor,
state: segment.state,
namespace_separator: default_namespace_separator(),
case_sensitive: default_case_sensitive(),
completeness: default_migrated_completeness(),
module_entries: Vec::new(),
})
.collect(),
runtime_stale: false,
}
}
fn from_v3(cache: ProjectResolverCacheV3) -> Self {
Self {
packages_state: cache.packages_state,
snapshot_fingerprint: cache.snapshot_fingerprint,
command_index: cache.command_index,
package_index: cache.package_index,
identity_index: cache.identity_index,
install_path_index: cache.install_path_index,
local_path_dirs: cache.local_path_dirs,
participating_files: cache.participating_files,
grammar_segments: cache
.grammar_segments
.into_iter()
.map(|segment| ProjectGrammarSegmentRecord {
descriptor: segment.descriptor,
state: segment.state,
namespace_separator: segment.namespace_separator,
case_sensitive: segment.case_sensitive,
completeness: default_migrated_completeness(),
module_entries: segment.module_entries,
})
.collect(),
runtime_stale: false,
}
}
fn from_v4(cache: ProjectResolverCacheV4) -> Self {
Self {
packages_state: cache.packages_state,
snapshot_fingerprint: cache.snapshot_fingerprint,
command_index: cache.command_index,
package_index: cache.package_index,
identity_index: cache.identity_index,
install_path_index: cache.install_path_index,
local_path_dirs: cache.local_path_dirs,
participating_files: cache.participating_files,
grammar_segments: cache
.grammar_segments
.into_iter()
.map(|segment| ProjectGrammarSegmentRecord {
descriptor: segment.descriptor.clone(),
state: segment.state,
namespace_separator: segment.namespace_separator,
case_sensitive: segment.case_sensitive,
completeness: migrated_completeness_from_descriptor(&segment.descriptor),
module_entries: segment.module_entries,
})
.collect(),
runtime_stale: false,
}
}
}
fn apply_runtime_staleness(mut cache: ProjectResolverCache) -> ProjectResolverCache {
let current_participating_files = match participating_files_from_packages(&cache.packages_state) {
Ok(records) => records,
Err(_) => {
mark_cache_stale(&mut cache);
return cache;
}
};
if current_participating_files != cache.participating_files {
mark_cache_stale(&mut cache);
}
cache
}
fn mark_cache_stale(cache: &mut ProjectResolverCache) {
cache.runtime_stale = true;
for segment in &mut cache.grammar_segments {
segment.state = ProjectGrammarSegmentState::Stale;
}
}
fn validate_grammar_spec(grammar_spec: &GrammarSpec) -> Result<(), CliError> {
if grammar_spec.id.dialect.trim().is_empty() {
return Err(CliError::Message("grammar spec dialect cannot be empty".to_string()));
}
if grammar_spec.id.name.trim().is_empty() {
return Err(CliError::Message("grammar spec name cannot be empty".to_string()));
}
if grammar_spec.completeness.prefix_authoritative
&& !grammar_spec.capabilities.supports_prefix_queries
{
return Err(CliError::Message(format!(
"grammar spec {}:{}@{} declares authoritative prefix lookups without prefix query support",
grammar_spec.id.dialect, grammar_spec.id.name, grammar_spec.id.version
)));
}
if grammar_spec.completeness.child_enumeration_authoritative
&& !grammar_spec.capabilities.supports_child_enumeration
{
return Err(CliError::Message(format!(
"grammar spec {}:{}@{} declares authoritative child enumeration without child enumeration support",
grammar_spec.id.dialect, grammar_spec.id.name, grammar_spec.id.version
)));
}
if grammar_spec.completeness.negative_lookup_authoritative
&& !grammar_spec.capabilities.authoritative_negative_lookup
{
return Err(CliError::Message(format!(
"grammar spec {}:{}@{} declares authoritative negative lookup without negative lookup support",
grammar_spec.id.dialect, grammar_spec.id.name, grammar_spec.id.version
)));
}
Ok(())
}
fn grammar_capabilities_record(
capabilities: &GrammarCapabilities,
) -> ProjectGrammarCapabilitiesRecord {
ProjectGrammarCapabilitiesRecord {
authoritative_negative_lookup: capabilities.authoritative_negative_lookup,
supports_prefix_queries: capabilities.supports_prefix_queries,
supports_child_enumeration: capabilities.supports_child_enumeration,
supports_multi_file_modules: capabilities.supports_multi_file_modules,
}
}
fn grammar_completeness_record(
completeness: &crate::grammar::CompletenessRules,
) -> ProjectGrammarCompletenessRecord {
ProjectGrammarCompletenessRecord {
negative_lookup_authoritative: completeness.negative_lookup_authoritative,
prefix_authoritative: completeness.prefix_authoritative,
child_enumeration_authoritative: completeness.child_enumeration_authoritative,
}
}
fn default_migrated_completeness() -> ProjectGrammarCompletenessRecord {
ProjectGrammarCompletenessRecord::default()
}
fn migrated_completeness_from_descriptor(
descriptor: &ProjectGrammarDescriptorRecord,
) -> ProjectGrammarCompletenessRecord {
ProjectGrammarCompletenessRecord {
negative_lookup_authoritative: descriptor.capabilities.authoritative_negative_lookup,
prefix_authoritative: false,
child_enumeration_authoritative: false,
}
}
pub(crate) async fn load_tool_state() -> Result<ToolState, CliError> {
let path = tool_state_path()?;
if path.exists() {
let bytes = tokio::fs::read(&path).await?;
return Ok(serde_json::from_slice(&bytes)?);
}
Ok(ToolState::default())
}
pub(crate) async fn write_tool_state(state: &ToolState) -> Result<(), CliError> {
let path = tool_state_path()?;
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent).await?;
}
tokio::fs::write(path, serde_json::to_vec_pretty(state)?).await?;
Ok(())
}
pub(crate) fn tool_state_path() -> Result<PathBuf, CliError> {
Ok(gritpack_state_root()?.join("tools.json"))
}