use std::any::Any;
use std::collections::HashMap;
use std::io::{self, Read};
use std::path::{Path, PathBuf};
use std::sync::{Arc, LazyLock, Mutex};
use serde::Serialize;
use tauri::{AppHandle, Emitter, Manager, State};
use typst::diag::{FileResult, PackageError, Severity, SourceDiagnostic, SourceResult, Warned};
use typst::foundations::{Bytes, Datetime, Duration, Output};
use typst::layout::{Frame, FrameItem};
use typst::syntax::package::PackageSpec;
use typst::syntax::{FileId, RootedPath, Source, VirtualPath, VirtualRoot};
use typst::text::{Font, FontBook};
use typst::utils::{LazyHash, hash128};
use typst::{Feature, Library, LibraryExt, World, WorldExt};
use typst_html::HtmlDocument;
use typst_kit::datetime::Time;
use typst_kit::downloader::{Downloader, SystemDownloader};
use typst_kit::files::{FileLoader, FileStore, FsRoot};
use typst_kit::fonts::{self, FontPath, FontStore};
use typst_kit::packages::SystemPackages;
use typst_layout::PagedDocument;
use typst_svg::SvgOptions;
use crate::theme;
use crate::workspace::Workspace;
const MAX_PROBLEMS: usize = 10;
static LIBRARY: LazyLock<LazyHash<Library>> = LazyLock::new(|| {
LazyHash::new(
Library::builder()
.with_features([Feature::Html].into_iter().collect())
.build(),
)
});
pub fn font_store(themes: &Path) -> FontStore {
let mut store = FontStore::new();
store.extend(fonts::system());
store.extend(fonts::embedded());
store.extend(fonts::scan(themes));
store
}
#[derive(Debug, Clone, PartialEq)]
pub struct UsedFont {
pub family: String,
pub file: PathBuf,
pub restricted: bool,
}
fn fonts_in(frame: &Frame, found: &mut Vec<Font>) {
for (_, item) in frame.items() {
match item {
FrameItem::Group(group) => fonts_in(&group.frame, found),
FrameItem::Text(text) if !found.contains(text.font.font()) => {
found.push(text.font.font().clone())
}
_ => {}
}
}
}
#[derive(Debug, Serialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct Diagnostic {
severity: &'static str,
message: String,
from: usize,
to: usize,
}
#[derive(Debug, PartialEq)]
pub struct Problem {
pub line: Option<usize>,
pub message: String,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct RenderedPage {
index: usize,
svg: String,
width: f64,
height: f64,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Compiled {
page_count: Option<usize>,
pages: Vec<RenderedPage>,
diagnostics: Vec<Diagnostic>,
}
#[derive(Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct CompiledEvent<'a> {
page: &'a str,
seq: u64,
#[serde(flatten)]
result: &'a Compiled,
}
struct Announcing<F> {
inner: SystemDownloader,
announce: F,
}
impl<F: Fn(&str) + Send + Sync + 'static> Downloader for Announcing<F> {
fn stream(&self, key: &dyn Any, url: &str) -> io::Result<(Option<usize>, Box<dyn Read>)> {
if let Some(spec) = key.downcast_ref::<PackageSpec>() {
(self.announce)(&spec.to_string());
}
self.inner.stream(key, url)
}
}
struct Loader {
main: FileId,
text: String,
wrapper: Option<(FileId, String)>,
project: FsRoot,
themes: PathBuf,
packages: SystemPackages,
}
impl FileLoader for Loader {
fn load(&self, id: FileId) -> FileResult<Bytes> {
if id == self.main {
return Ok(Bytes::from_string(self.text.clone()));
}
if let Some((wrapper, text)) = &self.wrapper
&& id == *wrapper
{
return Ok(Bytes::from_string(text.clone()));
}
match id.root() {
VirtualRoot::Package(spec) if spec.namespace == "themes" => {
let package = self
.themes
.join(spec.name.as_str())
.join(spec.version.to_string())
.join("typst");
if !theme::valid_id(&spec.name) || !package.is_dir() {
return Err(PackageError::NotFound(spec.clone()).into());
}
FsRoot::new(package).load(id.vpath())
}
VirtualRoot::Package(spec) => self.packages.obtain(spec)?.load(id.vpath()),
VirtualRoot::Project => self.project.load(id.vpath()),
}
}
}
pub struct PageWorld {
main: FileId,
files: FileStore<Loader>,
fonts: Arc<FontStore>,
time: Time,
hashes: Vec<u128>,
used: Vec<UsedFont>,
folders: Vec<String>,
}
impl World for PageWorld {
fn library(&self) -> &LazyHash<Library> {
&LIBRARY
}
fn book(&self) -> &LazyHash<FontBook> {
self.fonts.book()
}
fn main(&self) -> FileId {
self.files
.loader()
.wrapper
.as_ref()
.map_or(self.main, |(wrapper, _)| *wrapper)
}
fn source(&self, id: FileId) -> FileResult<Source> {
self.files.source(id)
}
fn file(&self, id: FileId) -> FileResult<Bytes> {
self.files.file(id)
}
fn font(&self, index: usize) -> Option<Font> {
self.fonts.font(index)
}
fn today(&self, offset: Option<Duration>) -> Option<Datetime> {
self.time.today(offset)
}
}
impl PageWorld {
pub fn new(
project: &Path,
main: &str,
themes: &Path,
fonts: Arc<FontStore>,
announce_download: impl Fn(&str) + Send + Sync + 'static,
) -> Result<Self, String> {
let path = VirtualPath::new(main).map_err(|error| error.to_string())?;
let main = FileId::new(RootedPath::new(VirtualRoot::Project, path));
let downloader = Announcing {
inner: SystemDownloader::new(concat!("znote/", env!("CARGO_PKG_VERSION"))),
announce: announce_download,
};
let loader = Loader {
main,
text: String::new(),
wrapper: None,
project: FsRoot::new(project.to_path_buf()),
themes: themes.to_path_buf(),
packages: SystemPackages::new(downloader),
};
Ok(Self {
main,
files: FileStore::new(loader),
fonts,
time: Time::system(),
hashes: Vec::new(),
used: Vec::new(),
folders: [project, themes]
.map(|folder| folder.to_string_lossy().into_owned())
.into_iter()
.chain(
std::env::var("USERPROFILE")
.or_else(|_| std::env::var("HOME"))
.ok(),
)
.filter(|folder| folder.len() > 1)
.collect(),
})
}
pub fn forget_sent_pages(&mut self) {
self.hashes.clear();
}
fn typeset<T: Output>(&mut self, text: String) -> Warned<SourceResult<T>> {
self.files.loader_mut().text = text;
self.files.reset();
self.time.reset();
typst::compile::<T>(self)
}
pub fn document(&mut self, text: String) -> Result<PagedDocument, String> {
self.typeset(text)
.output
.map_err(|_| "Fix the problems on this page before you export it.".to_string())
}
pub fn html(
&mut self,
text: String,
prelude: &str,
) -> Result<(HtmlDocument, Vec<String>), String> {
let path = self.main.vpath().get_with_slash();
let quoted = path.replace('\\', "\\\\").replace('"', "\\\"");
let wrapper = FileId::unique(RootedPath::new(
VirtualRoot::Project,
VirtualPath::new("/.znote-export.typ").map_err(|error| error.to_string())?,
));
self.files.loader_mut().wrapper =
Some((wrapper, format!("{prelude}\n#include \"{quoted}\"\n")));
let warned = self.typeset::<HtmlDocument>(text);
self.files.loader_mut().wrapper = None;
let warnings = warned
.warnings
.iter()
.map(|warning| warning.message.to_string())
.collect();
match warned.output {
Ok(document) => Ok((document, warnings)),
Err(errors) => Err(errors
.first()
.map(|error| error.message.to_string())
.unwrap_or_default()),
}
}
pub fn compile(&mut self, text: String) -> Compiled {
let warned = self.typeset::<PagedDocument>(text);
let mut diagnostics: Vec<Diagnostic> = warned
.warnings
.iter()
.filter(|d| !self.template_font_fallback(d))
.map(|d| self.diagnostic(d))
.collect();
let document = match warned.output {
Ok(document) => document,
Err(errors) => {
diagnostics.extend(errors.iter().map(|d| self.diagnostic(d)));
return Compiled {
page_count: None,
pages: Vec::new(),
diagnostics,
};
}
};
let mut fonts = Vec::new();
document
.pages()
.iter()
.for_each(|page| fonts_in(&page.frame, &mut fonts));
self.used = fonts
.iter()
.filter_map(|font| self.used_font(font))
.collect();
let hashes: Vec<u128> = document.pages().iter().map(hash128).collect();
let pages = document
.pages()
.iter()
.enumerate()
.filter(|(index, _)| self.hashes.get(*index) != Some(&hashes[*index]))
.map(|(index, page)| RenderedPage {
index,
svg: typst_svg::svg(page, &SvgOptions::default()),
width: page.frame.width().to_pt(),
height: page.frame.height().to_pt(),
})
.collect();
self.hashes = hashes;
Compiled {
page_count: Some(self.hashes.len()),
pages,
diagnostics,
}
}
pub fn problems(&mut self, text: String) -> Vec<Problem> {
let Err(errors) = self.typeset::<PagedDocument>(text).output else {
return Vec::new();
};
errors
.iter()
.take(MAX_PROBLEMS)
.map(|error| {
let message = self
.folders
.iter()
.fold(self.diagnostic(error).message, |message, folder| {
message.replace(folder.as_str(), "")
});
Problem {
line: self.line(error),
message,
}
})
.collect()
}
fn line(&self, diagnostic: &SourceDiagnostic) -> Option<usize> {
if diagnostic.span.id()? != self.main {
return None;
}
let start = self.range(diagnostic.span)?.start;
Some(self.source(self.main).ok()?.lines().byte_to_line(start)? + 1)
}
fn used_font(&self, font: &Font) -> Option<UsedFont> {
let info = font.info();
let index = self
.fonts
.book()
.select(&info.family.to_lowercase(), info.variant)?;
let source: &dyn Any = self.fonts.source(index)?;
let file = source.downcast_ref::<FontPath>()?.path.clone();
Some(UsedFont {
family: info.family.clone(),
file,
restricted: crate::export::restricted(font),
})
}
pub fn used_fonts(&self) -> &[UsedFont] {
&self.used
}
fn template_font_fallback(&self, diagnostic: &SourceDiagnostic) -> bool {
diagnostic.span.id() != Some(self.main)
&& diagnostic.message.starts_with("unknown font family")
}
fn diagnostic(&self, diagnostic: &SourceDiagnostic) -> Diagnostic {
let mut message = diagnostic.message.to_string();
for hint in &diagnostic.hints {
message.push_str("\nHint: ");
message.push_str(&hint.v);
}
let (from, to) = self.utf16_range(diagnostic).unwrap_or((0, 0));
Diagnostic {
severity: match diagnostic.severity {
Severity::Error => "error",
Severity::Warning => "warning",
},
message,
from,
to,
}
}
fn utf16_range(&self, diagnostic: &SourceDiagnostic) -> Option<(usize, usize)> {
if diagnostic.span.id()? != self.main {
return None;
}
let range = self.range(diagnostic.span)?;
let source = self.source(self.main).ok()?;
let lines = source.lines();
Some((
lines.byte_to_utf16(range.start)?,
lines.byte_to_utf16(range.end)?,
))
}
}
#[derive(Default)]
pub struct Compiler {
latest: Mutex<HashMap<String, u64>>,
world: Mutex<Option<(PathBuf, PageWorld)>>,
fonts: Mutex<Option<Arc<FontStore>>>,
}
impl Compiler {
pub fn forget_themes(&self) -> Result<(), String> {
*self.world.lock().map_err(|error| error.to_string())? = None;
*self.fonts.lock().map_err(|error| error.to_string())? = None;
Ok(())
}
}
fn new_world(app: &AppHandle, project: &Path, main: &str) -> Result<PageWorld, String> {
let announcer = app.clone();
let themes = theme::folder(app)?;
let fonts = app
.state::<Compiler>()
.fonts
.lock()
.map_err(|error| error.to_string())?
.get_or_insert_with(|| {
let mut store = font_store(&themes);
if let Some(carried) = crate::project::fonts(app) {
store.extend(fonts::scan(&carried));
}
Arc::new(store)
})
.clone();
PageWorld::new(project, main, &themes, fonts, move |package| {
let _ = announcer.emit("downloading", package);
})
}
pub fn problems(app: &AppHandle, page: &str, text: String) -> Vec<Problem> {
let check = || {
let (project, main) = app.state::<Workspace>().typst_source(page)?;
let mut world = new_world(app, &project, &main)?;
crate::guarded("", || Ok(world.problems(text)))
};
check().unwrap_or_default()
}
type OpenWorld = Option<(PathBuf, PageWorld)>;
fn open_world<'a>(
app: &AppHandle,
open: &'a mut OpenWorld,
page: &str,
) -> Result<&'a mut PageWorld, String> {
let (project, main) = app.state::<Workspace>().typst_source(page)?;
let file = project.join(&main);
if open.as_ref().is_none_or(|(current, _)| *current != file) {
*open = Some((file, new_world(app, &project, &main)?));
}
open.as_mut()
.map(|(_, world)| world)
.ok_or_else(|| "The page could not be opened.".to_string())
}
pub fn with_page<T>(
app: &AppHandle,
page: &str,
work: impl FnOnce(&mut PageWorld) -> Result<T, String>,
) -> Result<T, String> {
let compiler = app.state::<Compiler>();
let mut open = compiler.world.lock().map_err(|error| error.to_string())?;
let world = open_world(app, &mut open, page)?;
crate::guarded(
"This page could not be read. Undo your last change and try again.",
|| work(world),
)
}
fn compile_page(
app: &AppHandle,
page: &str,
seq: u64,
text: String,
fresh: bool,
) -> Result<(), String> {
let compiler = app.state::<Compiler>();
let mut open = compiler.world.lock().map_err(|error| error.to_string())?;
let superseded = compiler
.latest
.lock()
.map_err(|error| error.to_string())?
.get(page)
!= Some(&seq);
if superseded {
return Ok(());
}
let world = open_world(app, &mut open, page)?;
if fresh {
world.forget_sent_pages();
}
let result = crate::guarded(
"This page could not be read. Undo your last change and try again.",
|| Ok(world.compile(text)),
)?;
if result.page_count.is_some() {
crate::project::fonts_used(app, page, world.used_fonts());
}
app.emit(
"compiled",
CompiledEvent {
page,
seq,
result: &result,
},
)
.map_err(|error| error.to_string())
}
#[tauri::command]
pub fn compile(
app: AppHandle,
compiler: State<Compiler>,
page: String,
seq: u64,
text: String,
fresh: bool,
) -> Result<(), String> {
compiler
.latest
.lock()
.map_err(|error| error.to_string())?
.insert(page.clone(), seq);
std::thread::spawn(move || {
if let Err(message) = compile_page(&app, &page, seq, text, fresh) {
let _ = app.emit("compile-failed", message);
}
});
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
static FONTS: LazyLock<Arc<FontStore>> = LazyLock::new(|| {
Arc::new(font_store(
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("no-themes")
.as_path(),
))
});
fn world() -> PageWorld {
let folder = std::env::temp_dir();
PageWorld::new(
&folder,
"main.typ",
&folder.join("no-themes"),
FONTS.clone(),
|_| {},
)
.unwrap()
}
#[test]
fn problems_name_the_line_of_each_error_and_a_good_page_has_none() {
let mut world = world();
assert!(world.problems("= Fine\n".into()).is_empty());
assert_eq!(
world.problems("#image(\"missing.png\")\n".into()),
[Problem {
line: Some(1),
message: "file not found (searched at /missing.png)".into()
}]
);
let found = world.problems("= Title\n\n#foo\n".into());
assert_eq!(found.len(), 1);
assert_eq!(found[0].line, Some(3));
assert!(
found[0].message.starts_with("unknown variable: foo"),
"{found:?}"
);
}
#[test]
fn a_theme_package_is_read_from_the_themes_folder_with_its_fonts() {
let themes =
std::env::temp_dir().join(format!("znote-compile-themes-{}", std::process::id()));
let package = themes.join("acme").join("0.1.0").join("typst");
std::fs::create_dir_all(package.join("fonts")).unwrap();
std::fs::write(
package.join("typst.toml"),
"[package]\nname = \"acme\"\nversion = \"0.1.0\"\nentrypoint = \"lib.typ\"\n",
)
.unwrap();
std::fs::write(
package.join("lib.typ"),
"#let brand = text(font: (\"Not Shipped\", \"Libertinus Serif\"))[Acme]",
)
.unwrap();
let font = fonts::embedded().next().unwrap().0;
std::fs::write(package.join("fonts").join("copy.ttf"), font.data()).unwrap();
let fonts = Arc::new(font_store(&themes));
assert!(FONTS.book().families().count() > 0);
assert!(
fonts
.source(
FONTS
.book()
.families()
.map(|(_, infos)| infos.count())
.sum::<usize>()
)
.is_some()
);
let mut world =
PageWorld::new(&std::env::temp_dir(), "main.typ", &themes, fonts, |_| {}).unwrap();
let result = world.compile("#import \"@themes/acme:0.1.0\": brand\n#brand".into());
assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
assert_eq!(result.page_count, Some(1));
let own = world.compile("#set text(font: \"Not Shipped\")\nHello".into());
assert!(
own.diagnostics[0].message.contains("unknown font family"),
"{:?}",
own.diagnostics
);
let missing = world.compile("#import \"@themes/other:0.1.0\": brand".into());
assert_eq!(missing.page_count, None);
assert!(
missing.diagnostics[0].message.contains("package not found"),
"{:?}",
missing.diagnostics
);
}
#[test]
fn a_page_reads_other_files_from_its_project_folder() {
let project = std::env::temp_dir().join(format!("znote-compile-{}", std::process::id()));
std::fs::create_dir_all(project.join("Notes")).unwrap();
std::fs::write(project.join("shared.typ"), "#let greeting = [Hello]").unwrap();
let mut world = PageWorld::new(
&project,
"Notes/Page.typ",
&project.join("no-themes"),
FONTS.clone(),
|_| {},
)
.unwrap();
let result = world.compile("#import \"../shared.typ\": greeting\n#greeting".into());
assert!(result.diagnostics.is_empty(), "{:?}", result.diagnostics);
assert_eq!(result.page_count, Some(1));
let escaping = world.compile("#read(\"../../outside.txt\")".into());
assert_eq!(escaping.page_count, None);
}
#[test]
fn valid_source_gives_its_pages() {
let result = world().compile("One #pagebreak() Two #pagebreak() Three".into());
assert_eq!(result.page_count, Some(3));
assert_eq!(result.pages.len(), 3);
assert!(result.pages[0].svg.starts_with("<svg"));
assert!(result.pages[0].width > 0.0);
assert!(result.diagnostics.is_empty());
}
#[test]
fn invalid_source_gives_a_diagnostic_with_a_range() {
let result = world().compile("é #nope".into());
assert_eq!(result.page_count, None);
assert_eq!(result.diagnostics.len(), 1);
let diagnostic = &result.diagnostics[0];
assert_eq!(diagnostic.severity, "error");
assert_eq!((diagnostic.from, diagnostic.to), (3, 7));
}
#[test]
fn only_changed_pages_are_sent_again() {
let mut world = world();
world.compile("One #pagebreak() Two".into());
let result = world.compile("One #pagebreak() Changed".into());
assert_eq!(result.page_count, Some(2));
assert_eq!(
result
.pages
.iter()
.map(|page| page.index)
.collect::<Vec<_>>(),
[1]
);
let unchanged = world.compile("One #pagebreak() Changed".into());
assert_eq!(unchanged.page_count, Some(2));
assert!(unchanged.pages.is_empty());
world.forget_sent_pages();
assert_eq!(
world.compile("One #pagebreak() Changed".into()).pages.len(),
2
);
}
#[test]
#[ignore = "needs the local data folder and the network"]
fn every_theme_in_the_data_folder_installs_and_its_templates_compile() {
let data = Path::new(env!("CARGO_MANIFEST_DIR")).join("../data");
let index = std::fs::read(data.join("index.json")).unwrap();
let entries: Vec<serde_json::Value> = serde_json::from_slice(&index).unwrap();
assert!(!entries.is_empty());
assert_eq!(theme::catalog(&index).unwrap().len(), entries.len());
let folder =
std::env::temp_dir().join(format!("znote-compile-data-{}", std::process::id()));
let themes = folder.join("themes");
for entry in &entries {
let id = entry["id"].as_str().unwrap();
theme::install(&themes, &data.join("themes").join(format!("{id}.znx"))).unwrap();
}
assert_eq!(theme::list(&themes).len(), entries.len());
let looks = Path::new(env!("CARGO_MANIFEST_DIR")).join("../data-looks");
let index = std::fs::read(looks.join("index.json")).unwrap();
let offered = zsuite_theme::catalog(&index).unwrap();
let listed: Vec<serde_json::Value> = serde_json::from_slice(&index).unwrap();
assert_eq!(offered.len(), listed.len());
for look in &offered {
zsuite_theme::install(
&theme::looks_of(&themes),
&looks.join("themes").join(format!("{}.zth", look.id)),
)
.unwrap();
}
let fonts = Arc::new(font_store(&themes));
for template in entries
.iter()
.flat_map(|entry| entry["typst"]["templates"].as_array().into_iter().flatten())
{
let mut world =
PageWorld::new(&folder, "main.typ", &themes, fonts.clone(), |_| {}).unwrap();
let result = world.compile(format!(
"{}\n= Heading\nSome text.\n",
template["snippet"].as_str().unwrap()
));
let errors: Vec<_> = result
.diagnostics
.iter()
.filter(|problem| problem.severity == "error")
.collect();
assert!(
result.page_count.is_some() && errors.is_empty(),
"{}: {errors:?}",
template["label"]
);
}
}
}