use crate::state::{GrugState, Files, FileInfo};
use crate::arena::Arena;
use crate::types::FileId;
use crate::ast::*;
use crate::ntstring::{NTStrPtr, NTStr, NTBytes};
use crate::error::{Error, ErrorKind, SourceSpan};
use crate::mod_api::ModApi;
use crate::own_ptr::OwnPtr;
use crate::type_storage::TypeStorage;
use allocator_api2::vec::Vec;
use allocator_api2::boxed::Box as Box2;
use std::ffi::{OsStr, OsString};
use std::path::PathBuf;
use std::path::Path;
use std::sync::Arc;
use std::sync::mpsc::{Receiver, Sender};
const MAX_FILE_ENTITY_TYPE_LENGTH: usize = 420;
pub(crate) const SPACES_PER_INDENT: usize = 4;
pub mod tokenizer;
pub mod parser;
pub mod type_propagation;
use type_propagation::TypePropagator;
impl GrugState {
const FILES_PER_THREAD: usize = 8;
pub(crate) fn compiler_thread_fn(
receiver: Receiver<(Arena, &'static [&'static OsStr])>,
sender: Sender<(
Arena,
OwnPtr<'static, [(
Result<GrugAst<'static>, Error>,
// derived from input paths
// These paths are relative
&'static OsStr
)]>,
// Resources
// These paths are relative
&'static [&'static OsStr]
)>,
mods_dir_path: PathBuf,
mod_api: Arc<ModApi>,
) -> impl FnOnce() {
use crate::async_fs::{open_file_async_for_read, read_files_async};
move || {
let mut temp_arena = Arena::new();
let mut type_storage = TypeStorage::new();
for (arena, files) in receiver.iter() {
temp_arena.clear();
let mut resources = Vec::new_in(&arena);
fn combine_lifetimes<'a>(_: &'a Arena, input: &'static [&'static OsStr]) -> &'a [&'a OsStr] {input}
let files = combine_lifetimes(&arena, files);
let mut results = Vec::new_in(&arena);
let mut ok_files = Vec::new_in(&arena);
for file_path in files.iter().copied() {
let mut abs_path = mods_dir_path.clone();
abs_path.push(file_path);
match open_file_async_for_read(&abs_path) {
Ok(file) => {
ok_files.push((file, file_path));
},
Err(err) => {
results.push((Err(err), file_path));
}
}
}
let ok_files_data = read_files_async(ok_files.iter().map(|(file, path)| (file, *path)), &arena);
results.extend(ok_files_data.into_iter().zip(&ok_files).map(|(data, (_, path))| {
let path = *path;
let file_text = match data {
Ok(data) => data,
Err(err) => return (Err(err), path),
};
let (ast, current_resources) = match Self::compile_inner(
path,
file_text,
mods_dir_path.as_ref(),
&mod_api,
&arena,
&temp_arena,
&mut type_storage,
) {
Ok(data) => data,
Err(err) => return (Err(err), path)
};
resources.extend_from_slice(current_resources);
(Ok(ast), path)
}));
drop(ok_files);
let results = unsafe{std::mem::transmute::<
OwnPtr<[(
Result<
GrugAst<'_>,
Error
>,
&OsStr
)]>,
OwnPtr<'static, [(
Result<
GrugAst<'static>,
Error
>,
&'static OsStr
)]>
>(results.into_boxed_slice().into())};
let resources = unsafe{std::mem::transmute::<&[&OsStr], &'static[&'static OsStr]>(resources.leak())};
let Ok(()) = sender.send((arena, results, resources)) else {break;};
}
}
}
pub fn compile_grug_file(&self, path: impl AsRef<OsStr>) -> Result<FileId, Error> {
let path = path.as_ref();
let mut path_buf = self.mods_dir_path.clone();
path_buf.push("/");
path_buf.push(path);
let file_text = match std::fs::read_to_string(path_buf) {
Ok(file_text) => file_text,
Err(err) => return Err(Error::from_io_error(err, path))
};
self.compile_grug_file_from_str(path, &file_text)
}
pub fn compile_grug_file_from_str(&self, path: impl AsRef<OsStr>, file_text: &str) -> Result<FileId, Error> {
use super::frontend::*;
use crate::async_fs::{verify_file_data};
let path = path.as_ref();
let mut arena = self.arenas.borrow_mut().pop().unwrap_or_default();
let file_text = arena.copy_bytes_into_nt(file_text.as_bytes());
let file_text = verify_file_data(file_text, path)?;
let id = (|| {
let (file, resources) = Self::compile_inner(
path,
file_text,
&self.mods_dir_path,
&self.mod_api,
&arena,
&arena,
&mut *self.type_storage.borrow_mut(),
)?;
let mut self_resources = self.resources.borrow_mut();
for resource in resources {
if !self_resources.contains(*resource) {self_resources.insert(OsString::from(resource));}
}
let id = self.get_or_insert_script_id(path.as_ref());
self.backend.insert_file(id, &file);
let mut script_entities = self.script_entities.borrow_mut();
if (id.to_inner() as usize) < script_entities.len() {
for entity in &script_entities[id.to_inner() as usize] {
self.backend.init_entity(self, unsafe{&*entity.as_ptr()});
}
} else if id.to_inner() as usize == script_entities.len() {
script_entities.push(std::vec::Vec::new());
} else {
unreachable!();
}
Ok(id)
})();
arena.clear();
self.arenas.borrow_mut().push(arena);
id
}
pub fn compile_all_files(&self) -> Files {
let arena = self.arenas.borrow_mut().pop().unwrap_or_else(Arena::new);
let mut file_paths = Vec::new_in(&arena);
let mut files = std::vec::Vec::new();
let mods_dir_len = if self.mods_dir_path.as_encoded_bytes().last().is_some_and(|x| *x != b'\\' && *x != b'/') {self.mods_dir_path.len() + 1} else {self.mods_dir_path.len()};
for mod_dir in std::fs::read_dir(&self.mods_dir_path).expect("Could not read mods directory") {
let Ok(mod_dir) = mod_dir else {
panic!("unable to read directory: {:?}", mod_dir);
};
let mut entries_to_check = std::vec::Vec::from([mod_dir]);
while let Some(next_entry) = entries_to_check.pop() {
if next_entry.metadata().expect("could not read metadata").is_dir() {
let next_entry_path = next_entry.path();
for entry in std::fs::read_dir(&next_entry_path).expect("Could not read mods directory") {
let Ok(entry) = entry else {
panic!("unable to read entry: {:?}", entry);
};
entries_to_check.push(entry);
}
} else {
let entry_path = next_entry.path();
if let Some(extension) = entry_path.extension() && extension == "grug" {
let rel_path = unsafe{OsStr::from_encoded_bytes_unchecked(&entry_path.as_os_str().as_encoded_bytes()[mods_dir_len..])};
let rel_path = arena.copy_osstr_into(rel_path);
file_paths.push(rel_path);
};
}
}
}
let mut next_thread = self.compiler_senders.iter().cycle();
let sent_count = file_paths.len();
let mut recv_count = 0;
for chunk in file_paths.chunks(Self::FILES_PER_THREAD) {
let cur_arena = self.arenas.borrow_mut().pop().unwrap_or_else(Arena::new);
let chunk = cur_arena.slice_from_iter(chunk.iter().map(|item| cur_arena.copy_osstr_into(item.as_ref())));
let chunk = unsafe{std::mem::transmute::<&[&OsStr], &'static [&'static OsStr]>(chunk)};
next_thread.next().expect("at least one compiler thread").send((cur_arena, chunk)).expect("send succeeds");
}
while recv_count < sent_count {
let (mut current_arena, results, resources) = self.compiler_receiver.recv().unwrap();
recv_count += results.len();
for (result, path) in results {
let result = match result {
Ok(ast) => {
let id = self.get_or_insert_script_id(path.as_ref());
self.backend.insert_file(id, &ast);
let mut script_entities = self.script_entities.borrow_mut();
if (id.to_inner() as usize) < script_entities.len() {
for entity in &script_entities[id.to_inner() as usize] {
self.backend.init_entity(self, unsafe{&*entity.as_ptr()});
}
} else if id.to_inner() as usize == script_entities.len() {
script_entities.push(std::vec::Vec::new());
} else {
unreachable!();
}
Ok(id)
}
Err(err) => {
let err = err.inner().copy_into(&arena);
Err(err)
}
};
let path = <OsStr as AsRef<Path>>::as_ref(path);
let mod_dir_path = path.parent().expect("must have at least component in path").components().next().unwrap().as_os_str();
let info = FileInfo::new_in(
path.as_os_str(),
path.file_name().unwrap(),
mod_dir_path,
get_entity_type(path.as_os_str()).unwrap_or(""),
path.file_prefix().unwrap(),
result,
&arena
);
files.push(info);
}
let mut self_resources = self.resources.borrow_mut();
for resource in resources {
if !self_resources.contains(*resource) {
self_resources.insert(OsString::from(resource));
}
}
current_arena.clear();
self.arenas.borrow_mut().push(current_arena);
}
drop(file_paths);
Files {
inner: unsafe{std::mem::transmute::<OwnPtr<[FileInfo]>, OwnPtr<'static, [FileInfo]>>(files.into_boxed_slice().into())},
_arena: arena,
}
}
pub fn update_files(&self) -> (std::vec::Vec<OsString>, Files) {
let arena = self.arenas.borrow_mut().pop().unwrap_or_else(Arena::new);
let mut file_paths = Vec::new_in(&arena);
let mut updated_resources = std::vec::Vec::new();
let mut grug_files = std::vec::Vec::new();
for change in self.changes.try_iter() {
let file_name = change.expect("File IO error");
if let Some(extension) = <OsStr as AsRef<Path>>::as_ref(&file_name).extension() && extension == "grug" {
if !file_paths.contains(&&*file_name) {
let rel_name = arena.copy_osstr_into(file_name.as_ref());
file_paths.push(rel_name);
}
}
if self.resources.borrow().contains(&file_name) {
if !updated_resources.contains(&file_name) {
updated_resources.push(file_name);
}
}
}
let mut next_thread = self.compiler_senders.iter().cycle();
let sent_count = file_paths.len();
let mut recv_count = 0;
for chunk in file_paths.chunks(Self::FILES_PER_THREAD) {
let cur_arena = self.arenas.borrow_mut().pop().unwrap_or_else(Arena::new);
let chunk = cur_arena.slice_from_iter(chunk.iter().map(|item| cur_arena.copy_osstr_into(item.as_ref())));
let chunk = unsafe{std::mem::transmute::<&[&OsStr], &'static [&'static OsStr]>(chunk)};
next_thread.next().expect("at least one compiler thread").send((cur_arena, chunk)).expect("send succeeds");
}
while recv_count < sent_count {
let (mut current_arena, results, resources) = self.compiler_receiver.recv().unwrap();
recv_count += results.len();
for (result, path) in results {
let result = match result {
Ok(ast) => {
let id = self.get_or_insert_script_id(path.as_ref());
self.backend.insert_file(id, &ast);
let mut script_entities = self.script_entities.borrow_mut();
if (id.to_inner() as usize) < script_entities.len() {
for entity in &script_entities[id.to_inner() as usize] {
self.backend.init_entity(self, unsafe{&*entity.as_ptr()});
}
} else if id.to_inner() as usize == script_entities.len() {
script_entities.push(std::vec::Vec::new());
} else {
unreachable!();
}
Ok(id)
}
Err(err) => {
let err = err.inner().copy_into(&arena);
Err(err)
}
};
let path = <OsStr as AsRef<Path>>::as_ref(path);
let mod_dir_path = path.parent().expect("must have at least one component in path").components().next().unwrap().as_os_str();
let info = FileInfo::new_in(
path.as_os_str(),
path.file_name().unwrap(),
mod_dir_path,
get_entity_type(path.as_os_str()).unwrap_or(""),
path.file_prefix().unwrap(),
result,
&arena
);
grug_files.push(info);
}
let mut self_resources = self.resources.borrow_mut();
for resource in resources {
if !self_resources.contains(*resource) {
self_resources.insert(OsString::from(resource));
}
}
current_arena.clear();
self.arenas.borrow_mut().push(current_arena);
}
drop(file_paths);
let grug_files = Files {
inner: unsafe{std::mem::transmute::<OwnPtr<[FileInfo]>, OwnPtr<'static, [FileInfo]>>(grug_files.into_boxed_slice().into())},
_arena: arena,
};
(updated_resources, grug_files)
}
fn compile_inner<'arena>(
path: &'arena OsStr,
file_text: &'arena NTStr,
mods_dir_path: &'arena OsStr,
mod_api: &'arena ModApi,
arena: &'arena Arena,
temp_arena: &'_ Arena,
type_storage: &mut TypeStorage,
) -> Result<(GrugAst<'arena>, &'arena [&'arena OsStr]), Error> {
let mod_name = get_mod_name(path);
let entity_type = get_entity_type(path)?;
if file_text.len() == 0 {
return Err(Error::new(
ErrorKind::EMPTY_FILE,
"",
path,
"",
SourceSpan {offset: 0, line: 1},
format_args!("File is empty")
));
}
let tokens = tokenizer::tokenize(file_text, arena, path)?;
let ast = parser::parse(tokens.leak(), arena, file_text, path)?;
let entity = mod_api.entities().get(entity_type).ok_or_else(||
Error::new(
ErrorKind::FILE_NAME_ERROR,
"",
path,
"",
SourceSpan{offset: 0, line: 0},
format_args!("Entity '{}' is not registered in the mod_api.json", entity_type),
)
)?;
let (ast, resources) = TypePropagator::fill_result_types(
entity,
mod_api,
mod_name,
mods_dir_path,
file_text,
path,
entity_type,
ast,
arena,
temp_arena,
type_storage,
)?;
let mut member_variables = Vec::new_in(arena);
let mut on_functions = Vec::new_in(arena);
on_functions.extend((0..entity.export_fns.len()).map(|_| None));
let mut helper_functions = Vec::new_in(arena);
ast.global_statements.into_iter().for_each(|statement| {
match statement {
GlobalStatement::Variable(st@MemberVariable {..}) => member_variables.push(st),
GlobalStatement::OnFunction(st@OnFunction {..}) => {
let (i, _) = entity.get_export_fn(st.name.to_str()).unwrap();
on_functions[i] = Some(&*Box2::leak(Box2::new_in(st, arena)));
}
GlobalStatement::HelperFunction(st@HelperFunction{..}) => helper_functions.push(st),
_ => (),
}
});
let file_path = unsafe{NTBytes::from_bytes_unchecked(arena.copy_bytes_into_nt(path.as_encoded_bytes()))};
let file = GrugAst{
members: member_variables.leak(),
on_functions: on_functions.leak(),
helper_functions: helper_functions.leak(),
file_text: file_text.as_ntstrptr(),
file_path
};
Ok((file, resources))
}
}
#[derive(Debug)]
pub(crate) enum GlobalStatement<'a> {
Variable(MemberVariable<'a>),
OnFunction(OnFunction<'a>),
HelperFunction(HelperFunction<'a>),
Comment{
value: NTStrPtr<'a>,
},
EmptyLine,
}
fn get_mod_name (path: &OsStr) -> &OsStr {
let path = path.as_encoded_bytes();
let mut slash_len = 0;
for (i, ch) in path.iter().enumerate() {
if *ch == b'/' || *ch == b'\\' {slash_len = i; break;}
}
unsafe{OsStr::from_encoded_bytes_unchecked(&path[..slash_len])}
}
fn get_entity_type(path: &OsStr) -> Result<&str, Error> {
let mut dot_pos = None;
let mut dash_pos = None;
let path_bytes = path.as_encoded_bytes();
let file_name = <OsStr as AsRef<Path>>::as_ref(path).file_name().unwrap_or("".as_ref());
for (i, ch) in path_bytes.iter().enumerate().rev() {
match (ch, dot_pos, dash_pos) {
(b'.', None, None) => dot_pos = Some(i),
(b'-', None, None) =>
return Err(Error::new(
ErrorKind::FILE_NAME_ERROR,
"",
path,
"",
SourceSpan{offset: 0, line: 0},
format_args!("'{}' is missing a period in its name", file_name.display())
)),
(b'-', Some(_), None) => {dash_pos = Some(i); break;},
_ => (),
}
}
let (dot_pos, dash_pos) = match (dot_pos, dash_pos) {
(Some(dot_pos), Some(dash_pos)) if dot_pos == dash_pos + 1 => {
return Err(Error::new(
ErrorKind::FILE_NAME_ERROR,
"",
path,
"",
SourceSpan{offset: 0, line: 0},
format_args!("'{}' is missing an entity type in its name", file_name.display())
));
}
(Some(dot_pos), Some(dash_pos)) => (dot_pos, dash_pos),
_ => {
return Err(Error::new(
ErrorKind::FILE_NAME_ERROR,
"",
path,
"",
SourceSpan{offset: 0, line: 0},
format_args!("'{}' is missing an entity type in its name", file_name.display())
));
}
};
let entity_type = unsafe{OsStr::from_encoded_bytes_unchecked(&path_bytes[(dash_pos + 1)..dot_pos])};
if entity_type.len() > MAX_FILE_ENTITY_TYPE_LENGTH {
return Err(Error::new(
ErrorKind::FILE_NAME_ERROR,
"",
path,
"",
SourceSpan{offset: 0, line: 0},
format_args!("There are more than {} characters \n\
in the entity type of '{}', exceeding MAX_FILE_ENTITY_TYPE_LENGTH",
entity_type.len(), path.display()
)
));
}
check_custom_id_is_pascal(entity_type, path)
}
fn check_custom_id_is_pascal<'a>(entity_type: &'a OsStr, path: &'_ OsStr) -> Result<&'a str, Error> {
let entity_type = entity_type.to_str().ok_or_else(||
Error::new(
ErrorKind::FILE_NAME_ERROR,
"",
entity_type,
"",
SourceSpan{offset: 0, line: 0},
format_args!("'{}' is not valid utf8",
entity_type.display()
)
)
)?;
let mut chars = entity_type.chars();
if let Some(first) = chars.next() && !first.is_uppercase() {
return Err(Error::new(
ErrorKind::FILE_NAME_ERROR,
"",
path,
"",
SourceSpan{offset: 0, line: 0},
format_args!("'{entity_type}' seems like a custom ID type, but it doesn't start in Uppercase")
));
}
for ch in chars {
if !(ch.is_uppercase() || ch.is_lowercase() || ch.is_ascii_digit()) {
return Err(Error::new(
ErrorKind::FILE_NAME_ERROR,
"",
path,
"",
SourceSpan{offset: 0, line: 0},
format_args!("'{entity_type}' seems like a custom ID type, but it contains '{ch}', which isn't uppercase, lowercase, or a digit", )
));
}
}
Ok(entity_type)
}