pub mod go_stdlib;
pub mod prelude;
pub mod types;
use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
use std::fs;
use std::hash::{Hash, Hasher};
use std::io;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use serde::{Deserialize, Serialize};
use syntax::program::File;
use crate::store::{ENTRY_MODULE_ID, Store};
use types::CachedDefinition;
pub const CACHE_FORMAT_VERSION: u32 = 1;
pub const COMPILER_VERSION_HASH: u64 = const_fnv1a_hash(env!("CARGO_PKG_VERSION").as_bytes());
pub const STDLIB_HASH: u64 = stdlib::STDLIB_CONTENT_HASH;
pub const PRELUDE_HASH: u64 = stdlib::PRELUDE_CONTENT_HASH;
pub const GO_STDLIB_HASH: u64 = stdlib::GO_STD_CONTENT_HASH;
const FNV_OFFSET: u64 = 0xcbf29ce484222325;
const FNV_PRIME: u64 = 0x100000001b3;
const fn const_fnv1a_hash(bytes: &[u8]) -> u64 {
let mut hash = FNV_OFFSET;
let mut i = 0;
while i < bytes.len() {
hash ^= bytes[i] as u64;
hash = hash.wrapping_mul(FNV_PRIME);
i += 1;
}
hash
}
struct FnvHasher(u64);
impl FnvHasher {
fn new() -> Self {
Self(FNV_OFFSET)
}
}
impl Hasher for FnvHasher {
fn write(&mut self, bytes: &[u8]) {
for &byte in bytes {
self.0 ^= byte as u64;
self.0 = self.0.wrapping_mul(FNV_PRIME);
}
}
fn finish(&self) -> u64 {
self.0
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModuleInterface {
pub version: u32,
pub compiler_version: u64,
pub stdlib_hash: u64,
pub module_hash: u64,
pub source_hash: u64,
pub dependency_hashes: HashMap<String, u64>,
pub files: Vec<CachedFile>,
pub definitions: HashMap<String, CachedDefinition>,
pub ufcs_methods: Vec<(String, String)>,
pub emit_stamp: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CachedFile {
pub name: String,
pub source: String,
}
#[derive(Debug, Clone)]
pub struct CompiledModule {
pub module_id: String,
pub source_hash: u64,
pub dep_hashes: HashMap<String, u64>,
}
#[derive(Debug, Clone)]
pub struct EmitStamp {
pub module_id: String,
pub artifact_hash: u64,
}
pub fn compute_emit_artifact_hash(source_hash: u64, go_module: &str) -> u64 {
let mut hasher = FnvHasher::new();
source_hash.hash(&mut hasher);
go_module.hash(&mut hasher);
hasher.finish()
}
pub fn hash_module_sources(files: &[File]) -> u64 {
let mut hasher = FnvHasher::new();
let mut sorted: Vec<_> = files.iter().collect();
sorted.sort_by_key(|f| &f.name);
for file in sorted {
file.name.hash(&mut hasher);
file.source.hash(&mut hasher);
}
hasher.finish()
}
pub fn compute_module_hash(source_hash: u64, dep_hashes: &HashMap<String, u64>) -> u64 {
let mut hasher = FnvHasher::new();
source_hash.hash(&mut hasher);
let mut deps: Vec<_> = dep_hashes.iter().collect();
deps.sort_by_key(|(k, _)| *k);
for (name, hash) in deps {
name.hash(&mut hasher);
hash.hash(&mut hasher);
}
hasher.finish()
}
pub fn get_dependency_module_hashes(
module_id: &str,
edges: &HashMap<String, HashSet<String>>,
module_hashes: &HashMap<String, u64>,
) -> HashMap<String, u64> {
let Some(deps) = edges.get(module_id) else {
return HashMap::default();
};
deps.iter()
.map(|dep_id| {
let hash = if dep_id.starts_with("go:") || dep_id == "prelude" {
STDLIB_HASH
} else {
*module_hashes.get(dep_id).unwrap_or(&0)
};
(dep_id.clone(), hash)
})
.collect()
}
pub fn is_cache_valid(
cache: &ModuleInterface,
current_source_hash: u64,
current_dep_hashes: &HashMap<String, u64>,
) -> bool {
cache.version == CACHE_FORMAT_VERSION
&& cache.compiler_version == COMPILER_VERSION_HASH
&& cache.stdlib_hash == STDLIB_HASH
&& cache.source_hash == current_source_hash
&& cache.dependency_hashes == *current_dep_hashes
}
pub fn cache_path(project_root: &Path, module_id: &str) -> PathBuf {
project_root
.join("target")
.join("cache")
.join(cache_file_name(module_id))
}
pub fn cache_file_name(module_id: &str) -> String {
format!("{}.cache", module_id.replace('/', "_"))
}
pub fn try_load_cache(
module_id: &str,
expected_source_hash: u64,
expected_dep_hashes: &HashMap<String, u64>,
expected_artifact_hash: Option<u64>,
project_root: &Path,
check_go_files: bool,
) -> Option<ModuleInterface> {
let path = cache_path(project_root, module_id);
let bytes = fs::read(&path).ok()?;
let interface: ModuleInterface = match bincode::deserialize(&bytes) {
Ok(i) => i,
Err(_) => {
let _ = fs::remove_file(&path);
return None;
}
};
if !is_cache_valid(&interface, expected_source_hash, expected_dep_hashes) {
let _ = fs::remove_file(&path);
return None;
}
if check_go_files {
if interface.emit_stamp != expected_artifact_hash {
return None;
}
if !all_go_outputs_exist(module_id, &interface.files, project_root) {
return None;
}
}
Some(interface)
}
fn all_go_outputs_exist(module_id: &str, cached_files: &[CachedFile], project_root: &Path) -> bool {
let target_dir = if module_id == ENTRY_MODULE_ID {
project_root.join("target")
} else {
project_root.join("target").join(module_id)
};
for cached_file in cached_files {
if cached_file.name.ends_with(".lis") && !cached_file.name.ends_with(".d.lis") {
let go_name = cached_file.name.replace(".lis", ".go");
if !target_dir.join(&go_name).exists() {
return false;
}
}
}
true
}
pub fn save_module_cache(
compiled: &CompiledModule,
store: &Store,
project_root: &Path,
ufcs_methods: &HashSet<(String, String)>,
) -> io::Result<()> {
let module_hash = compute_module_hash(compiled.source_hash, &compiled.dep_hashes);
let Some(module) = store.get_module(&compiled.module_id) else {
return Err(io::Error::other("module not found in store"));
};
let mut all_files: Vec<_> = module
.files
.values()
.chain(module.typedefs.values())
.collect();
all_files.sort_by_key(|f| &f.name);
let file_id_to_index: HashMap<u32, u32> = all_files
.iter()
.enumerate()
.map(|(idx, f)| (f.id, idx as u32))
.collect();
let interface = ModuleInterface {
version: CACHE_FORMAT_VERSION,
compiler_version: COMPILER_VERSION_HASH,
stdlib_hash: STDLIB_HASH,
module_hash,
source_hash: compiled.source_hash,
dependency_hashes: compiled.dep_hashes.clone(),
files: all_files
.iter()
.map(|f| CachedFile {
name: f.name.clone(),
source: f.source.clone(),
})
.collect(),
definitions: extract_public_definitions(store, &compiled.module_id, &file_id_to_index),
ufcs_methods: {
let prefix = format!("{}.", compiled.module_id);
ufcs_methods
.iter()
.filter(|(type_id, _)| type_id.starts_with(&prefix))
.cloned()
.collect()
},
emit_stamp: None,
};
let path = cache_path(project_root, &compiled.module_id);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
let temp_path = path.with_extension("cache.tmp");
let bytes = bincode::serialize(&interface).map_err(io::Error::other)?;
fs::write(&temp_path, bytes)?;
fs::rename(&temp_path, &path)?;
Ok(())
}
fn extract_public_definitions(
store: &Store,
module_id: &str,
file_id_to_index: &HashMap<u32, u32>,
) -> HashMap<String, CachedDefinition> {
let Some(module) = store.get_module(module_id) else {
return HashMap::default();
};
module
.definitions
.iter()
.filter(|(_, definition)| definition.visibility().is_public())
.map(|(name, definition)| {
(
name.to_string(),
CachedDefinition::from_definition(definition, file_id_to_index),
)
})
.collect()
}
pub fn register_cached_module(
store: &mut Store,
module_id: &str,
cached: ModuleInterface,
project_root: &Path,
) {
store.add_module(module_id);
let mut file_ids: Vec<u32> = vec![];
for cached_file in &cached.files {
let file_id = store.new_file_id();
file_ids.push(file_id);
let display_path = cached_file_display_path(project_root, module_id, &cached_file.name);
let file = File::new_cached(
module_id,
&cached_file.name,
&display_path,
&cached_file.source,
file_id,
);
store.store_file(module_id, file);
}
let module = store.get_module_mut(module_id).unwrap();
for (qualified_name, cached_definition) in cached.definitions {
let definition = cached_definition.to_definition(&file_ids);
module.definitions.insert(qualified_name.into(), definition);
}
store.mark_visited(module_id);
}
fn cached_file_display_path(project_root: &Path, module_id: &str, bare_name: &str) -> String {
let on_disk = if module_id == ENTRY_MODULE_ID {
project_root.join("src").join(bare_name)
} else {
project_root.join("src").join(module_id).join(bare_name)
};
crate::path::relative_to_cwd(&on_disk).unwrap_or_else(|| bare_name.to_string())
}
pub fn apply_emit_stamps(
project_root: &Path,
updates: &[(EmitStamp, Option<u64>)],
) -> io::Result<()> {
for (stamp, value) in updates {
let path = cache_path(project_root, &stamp.module_id);
let bytes = match fs::read(&path) {
Ok(b) => b,
Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
Err(e) => return Err(e),
};
let mut interface: ModuleInterface = match bincode::deserialize(&bytes) {
Ok(i) => i,
Err(_) => {
let _ = fs::remove_file(&path);
continue;
}
};
interface.emit_stamp = *value;
let temp_path = path.with_extension("cache.tmp");
let new_bytes = bincode::serialize(&interface).map_err(io::Error::other)?;
fs::write(&temp_path, new_bytes)?;
fs::rename(&temp_path, &path)?;
}
Ok(())
}
pub fn is_cache_disabled() -> bool {
std::env::var("LISETTE_NO_CACHE")
.map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
.unwrap_or(false)
}
static GLOBAL_CACHE_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
pub(crate) fn global_cache_temp_path(final_path: &Path) -> PathBuf {
let counter = GLOBAL_CACHE_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed);
final_path.with_extension(format!("tmp.{}.{}", std::process::id(), counter))
}
pub(crate) fn prune_legacy_global_caches(dir: &Path, prefix: &str) {
let Ok(entries) = fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let name = entry.file_name();
let name = name.to_string_lossy();
if name.starts_with(prefix) && name.contains("_compiler_") {
let _ = fs::remove_file(entry.path());
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use syntax::types::{Symbol, Type};
#[test]
fn test_hash_module_sources_deterministic() {
let file1 = File::new_cached("mod", "a.lis", "a.lis", "fn foo() {}", 1);
let file2 = File::new_cached("mod", "b.lis", "b.lis", "fn bar() {}", 2);
let hash1 = hash_module_sources(&[file1.clone(), file2.clone()]);
let hash2 = hash_module_sources(&[file2.clone(), file1.clone()]);
assert_eq!(hash1, hash2);
}
#[test]
fn test_hash_module_sources_content_sensitive() {
let file1 = File::new_cached("mod", "a.lis", "a.lis", "fn foo() {}", 1);
let file2 = File::new_cached("mod", "a.lis", "a.lis", "fn bar() {}", 1);
let hash1 = hash_module_sources(&[file1]);
let hash2 = hash_module_sources(&[file2]);
assert_ne!(hash1, hash2);
}
#[test]
fn test_compute_module_hash_includes_deps() {
let source_hash = 12345u64;
let mut deps1 = HashMap::default();
deps1.insert("dep_a".to_string(), 111u64);
let mut deps2 = HashMap::default();
deps2.insert("dep_a".to_string(), 222u64);
let hash1 = compute_module_hash(source_hash, &deps1);
let hash2 = compute_module_hash(source_hash, &deps2);
assert_ne!(hash1, hash2);
}
#[test]
fn test_compute_module_hash_deterministic() {
let source_hash = 12345u64;
let mut deps = HashMap::default();
deps.insert("dep_b".to_string(), 222u64);
deps.insert("dep_a".to_string(), 111u64);
let hash1 = compute_module_hash(source_hash, &deps);
let hash2 = compute_module_hash(source_hash, &deps);
assert_eq!(hash1, hash2);
}
#[test]
fn test_cache_validity_checks_version() {
let cache = ModuleInterface {
version: CACHE_FORMAT_VERSION + 1, compiler_version: COMPILER_VERSION_HASH,
stdlib_hash: STDLIB_HASH,
module_hash: 0,
source_hash: 100,
dependency_hashes: HashMap::default(),
files: vec![],
definitions: HashMap::default(),
ufcs_methods: vec![],
emit_stamp: None,
};
assert!(!is_cache_valid(&cache, 100, &HashMap::default()));
}
#[test]
fn test_cache_validity_checks_compiler_version() {
let cache = ModuleInterface {
version: CACHE_FORMAT_VERSION,
compiler_version: COMPILER_VERSION_HASH + 1, stdlib_hash: STDLIB_HASH,
module_hash: 0,
source_hash: 100,
dependency_hashes: HashMap::default(),
files: vec![],
definitions: HashMap::default(),
ufcs_methods: vec![],
emit_stamp: None,
};
assert!(!is_cache_valid(&cache, 100, &HashMap::default()));
}
#[test]
fn test_cache_validity_checks_source_hash() {
let cache = ModuleInterface {
version: CACHE_FORMAT_VERSION,
compiler_version: COMPILER_VERSION_HASH,
stdlib_hash: STDLIB_HASH,
module_hash: 0,
source_hash: 100,
dependency_hashes: HashMap::default(),
files: vec![],
definitions: HashMap::default(),
ufcs_methods: vec![],
emit_stamp: None,
};
assert!(!is_cache_valid(&cache, 200, &HashMap::default()));
assert!(is_cache_valid(&cache, 100, &HashMap::default()));
}
#[test]
fn test_cache_validity_checks_dep_hashes() {
let mut cached_deps = HashMap::default();
cached_deps.insert("dep".to_string(), 111u64);
let cache = ModuleInterface {
version: CACHE_FORMAT_VERSION,
compiler_version: COMPILER_VERSION_HASH,
stdlib_hash: STDLIB_HASH,
module_hash: 0,
source_hash: 100,
dependency_hashes: cached_deps.clone(),
files: vec![],
definitions: HashMap::default(),
ufcs_methods: vec![],
emit_stamp: None,
};
let mut different_deps = HashMap::default();
different_deps.insert("dep".to_string(), 222u64);
assert!(!is_cache_valid(&cache, 100, &different_deps));
assert!(is_cache_valid(&cache, 100, &cached_deps));
}
#[test]
fn test_type_roundtrip_bincode() {
let ty = Type::Function {
params: vec![Type::Nominal {
id: Symbol::from_raw("int"),
params: vec![],
underlying_ty: None,
}],
param_mutability: vec![false],
bounds: vec![],
return_type: Box::new(Type::Nominal {
id: Symbol::from_raw("main.MyType"),
params: vec![Type::Tuple(vec![Type::Never])],
underlying_ty: None,
}),
};
let bytes = bincode::serialize(&ty).unwrap();
let restored: Type = bincode::deserialize(&bytes).unwrap();
assert_eq!(ty, restored);
}
#[test]
fn test_cache_path_format() {
let path = cache_path(Path::new("/project"), "utils");
assert_eq!(path, PathBuf::from("/project/target/cache/utils.cache"));
let path = cache_path(Path::new("/project"), "deep/nested/mod");
assert_eq!(
path,
PathBuf::from("/project/target/cache/deep_nested_mod.cache")
);
}
#[test]
fn test_get_dependency_module_hashes_uses_stdlib_hash() {
let mut edges = HashMap::default();
let mut deps = HashSet::default();
deps.insert("go:fmt".to_string());
deps.insert("prelude".to_string());
deps.insert("user_mod".to_string());
edges.insert("my_mod".to_string(), deps);
let mut module_hashes = HashMap::default();
module_hashes.insert("user_mod".to_string(), 12345u64);
let result = get_dependency_module_hashes("my_mod", &edges, &module_hashes);
assert_eq!(result.get("go:fmt"), Some(&STDLIB_HASH));
assert_eq!(result.get("prelude"), Some(&STDLIB_HASH));
assert_eq!(result.get("user_mod"), Some(&12345u64));
}
#[test]
fn hash_module_sources_independent_of_display_path() {
let cli_file = File::new(
"greet",
"greet.lis",
"src/greet/greet.lis",
"pub fn x() -> int { 1 }",
vec![],
1,
);
let lsp_file = File::new(
"greet",
"greet.lis",
"greet.lis",
"pub fn x() -> int { 1 }",
vec![],
1,
);
assert_eq!(
hash_module_sources(&[cli_file]),
hash_module_sources(&[lsp_file]),
);
}
#[test]
fn cache_file_purity_no_src_prefix() {
let cached = CachedFile {
name: "greet.lis".to_string(),
source: "pub fn x() -> int { 1 }".to_string(),
};
let bytes = bincode::serialize(&cached).unwrap();
let serialized = String::from_utf8_lossy(&bytes);
assert!(
!serialized.contains("src/"),
"CachedFile must not contain `src/` prefix; got: {serialized:?}"
);
}
#[test]
fn artifact_hash_depends_on_go_module() {
let h1 = compute_emit_artifact_hash(100, "github.com/old/proj");
let h2 = compute_emit_artifact_hash(100, "github.com/new/proj");
assert_ne!(h1, h2);
}
#[test]
fn apply_emit_stamps_round_trip() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
std::fs::create_dir_all(root.join("target").join("cache")).unwrap();
let interface = ModuleInterface {
version: CACHE_FORMAT_VERSION,
compiler_version: COMPILER_VERSION_HASH,
stdlib_hash: STDLIB_HASH,
module_hash: 0,
source_hash: 100,
dependency_hashes: HashMap::default(),
files: vec![],
definitions: HashMap::default(),
ufcs_methods: vec![],
emit_stamp: None,
};
let path = cache_path(root, "greet");
std::fs::write(&path, bincode::serialize(&interface).unwrap()).unwrap();
let stamp = EmitStamp {
module_id: "greet".to_string(),
artifact_hash: 999,
};
apply_emit_stamps(root, &[(stamp.clone(), Some(999))]).unwrap();
let reread: ModuleInterface = bincode::deserialize(&std::fs::read(&path).unwrap()).unwrap();
assert_eq!(reread.emit_stamp, Some(999));
assert_eq!(reread.source_hash, 100);
apply_emit_stamps(root, &[(stamp, None)]).unwrap();
let reread: ModuleInterface = bincode::deserialize(&std::fs::read(&path).unwrap()).unwrap();
assert_eq!(reread.emit_stamp, None);
}
#[test]
fn apply_emit_stamps_missing_cache_is_no_op() {
let tmp = tempfile::tempdir().unwrap();
let stamp = EmitStamp {
module_id: "absent".to_string(),
artifact_hash: 0,
};
let result = apply_emit_stamps(tmp.path(), &[(stamp, None)]);
assert!(result.is_ok());
}
#[test]
fn try_load_cache_rejects_unstamped_for_emit() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
std::fs::create_dir_all(root.join("target").join("cache")).unwrap();
std::fs::create_dir_all(root.join("target").join("greet")).unwrap();
std::fs::write(root.join("target").join("greet").join("greet.go"), "").unwrap();
let interface = ModuleInterface {
version: CACHE_FORMAT_VERSION,
compiler_version: COMPILER_VERSION_HASH,
stdlib_hash: STDLIB_HASH,
module_hash: 0,
source_hash: 100,
dependency_hashes: HashMap::default(),
files: vec![CachedFile {
name: "greet.lis".to_string(),
source: String::new(),
}],
definitions: HashMap::default(),
ufcs_methods: vec![],
emit_stamp: None,
};
let path = cache_path(root, "greet");
std::fs::write(&path, bincode::serialize(&interface).unwrap()).unwrap();
let loaded = try_load_cache("greet", 100, &HashMap::default(), None, root, false);
assert!(loaded.is_some(), "Check phase must accept unstamped cache");
let loaded = try_load_cache(
"greet",
100,
&HashMap::default(),
Some(compute_emit_artifact_hash(100, "github.com/test/x")),
root,
true,
);
assert!(
loaded.is_none(),
"Emit phase must reject cache with emit_stamp = None"
);
}
#[test]
fn try_load_cache_rejects_after_debug_invalidation() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
std::fs::create_dir_all(root.join("target").join("cache")).unwrap();
std::fs::create_dir_all(root.join("target").join("greet")).unwrap();
std::fs::write(root.join("target").join("greet").join("greet.go"), "").unwrap();
let artifact_hash = compute_emit_artifact_hash(100, "github.com/test/x");
let interface = ModuleInterface {
version: CACHE_FORMAT_VERSION,
compiler_version: COMPILER_VERSION_HASH,
stdlib_hash: STDLIB_HASH,
module_hash: 0,
source_hash: 100,
dependency_hashes: HashMap::default(),
files: vec![CachedFile {
name: "greet.lis".to_string(),
source: String::new(),
}],
definitions: HashMap::default(),
ufcs_methods: vec![],
emit_stamp: Some(artifact_hash),
};
let path = cache_path(root, "greet");
std::fs::write(&path, bincode::serialize(&interface).unwrap()).unwrap();
assert!(
try_load_cache(
"greet",
100,
&HashMap::default(),
Some(artifact_hash),
root,
true,
)
.is_some()
);
let stamp = EmitStamp {
module_id: "greet".to_string(),
artifact_hash,
};
apply_emit_stamps(root, &[(stamp, None)]).unwrap();
assert!(
try_load_cache(
"greet",
100,
&HashMap::default(),
Some(artifact_hash),
root,
true,
)
.is_none()
);
}
#[test]
fn prune_legacy_global_caches_removes_only_hashed_files() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path();
let legacy_prelude = dir.join("prelude_defs_4330e9_compiler_f709f8.bin");
let legacy_stdlib = dir.join("stdlib_defs_151b6b_compiler_f709f8_darwin_arm64.bin");
let stable_prelude = dir.join("prelude_defs.bin");
let stable_stdlib = dir.join("stdlib_defs_darwin_arm64.bin");
let other_stdlib = dir.join("stdlib_defs_linux_amd64.bin");
for path in [
&legacy_prelude,
&legacy_stdlib,
&stable_prelude,
&stable_stdlib,
&other_stdlib,
] {
std::fs::write(path, b"x").unwrap();
}
prune_legacy_global_caches(dir, "prelude_defs");
prune_legacy_global_caches(dir, "stdlib_defs");
assert!(!legacy_prelude.exists());
assert!(!legacy_stdlib.exists());
assert!(stable_prelude.exists());
assert!(stable_stdlib.exists());
assert!(other_stdlib.exists());
}
#[test]
fn prune_legacy_global_caches_missing_dir_is_noop() {
let tmp = tempfile::tempdir().unwrap();
prune_legacy_global_caches(&tmp.path().join("does_not_exist"), "prelude_defs");
}
#[test]
fn global_cache_temp_paths_are_unique() {
let base = Path::new("/cache/prelude_defs.bin");
let first = global_cache_temp_path(base);
let second = global_cache_temp_path(base);
assert_ne!(first, second);
}
}