use std::{
fs,
path::{Path, PathBuf},
process::Command,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use corsa::{
Result, TsgoError,
api::{ApiClient, ApiMode, ApiSpawnConfig, SymbolHandle, UpdateSnapshotParams},
fast::{CompactString, SmallVec},
};
use serde_json::json;
use crate::{
args::{Cli, Suite},
dataset::DatasetCase,
measure::measure_with_warmup,
process::run_command,
stats::Stats,
};
#[derive(Clone, Debug)]
pub struct ToolRow {
pub workload: CompactString,
pub dataset: CompactString,
pub tool: CompactString,
pub stats: Stats,
}
struct ToolSupport {
workspace_root: PathBuf,
typescript_go_root: PathBuf,
node_command: CompactString,
tsc_script: PathBuf,
eslint_script: PathBuf,
eslint_config: PathBuf,
tsgolint_script: PathBuf,
}
struct OverlayConfig {
dir: OverlayDir,
path: PathBuf,
}
struct OverlayDir {
path: PathBuf,
}
struct WorkflowSession {
client: ApiClient,
snapshot: corsa::api::ManagedSnapshot,
project: corsa::api::ProjectHandle,
file: CompactString,
target: BenchTarget,
}
struct BenchTarget {
position: u32,
symbol: SymbolHandle,
}
pub async fn run(cli: &Cli, datasets: &[DatasetCase]) -> Result<SmallVec<[ToolRow; 32]>> {
let mut rows = SmallVec::<[ToolRow; 32]>::new();
let support = ToolSupport::discover(cli)?;
let run_project_check = cli.suites.contains(&Suite::ProjectCheck);
let run_workflow = cli.suites.contains(&Suite::Workflow);
for dataset in datasets {
let overlay = OverlayConfig::create(cli, dataset)?;
if run_project_check {
rows.extend(run_project_check_suite(cli, dataset, &support, &overlay).await?);
}
if run_workflow {
rows.extend(run_workflow_suite(cli, dataset).await?);
}
}
Ok(rows)
}
async fn run_project_check_suite(
cli: &Cli,
dataset: &DatasetCase,
support: &ToolSupport,
overlay: &OverlayConfig,
) -> Result<SmallVec<[ToolRow; 8]>> {
let mut rows = SmallVec::<[ToolRow; 8]>::new();
let timeout = Duration::from_millis(cli.timeout_ms);
rows.push(row(
"project_check",
dataset,
"tsc",
measure_with_warmup(cli.warmup_iterations, cli.iterations, || async {
let mut command = tsc_command(support, overlay);
run_command(&mut command, timeout, &[0], "tsc")
})
.await?,
));
rows.push(row(
"project_check",
dataset,
"tsgo",
measure_with_warmup(cli.warmup_iterations, cli.iterations, || async {
let mut command = tsgo_command(cli, support, overlay);
run_command(&mut command, timeout, &[0], "tsgo")
})
.await?,
));
rows.push(row(
"project_check",
dataset,
"typescript-eslint",
measure_with_warmup(cli.warmup_iterations, cli.iterations, || async {
let mut command = eslint_command(dataset, support, overlay);
run_command(&mut command, timeout, &[0, 1], "typescript-eslint")
})
.await?,
));
rows.push(row(
"project_check",
dataset,
"tsgolint",
measure_with_warmup(cli.warmup_iterations, cli.iterations, || async {
let mut command = tsgolint_command(support, overlay);
run_command(&mut command, timeout, &[0, 1], "tsgolint")
})
.await?,
));
Ok(rows)
}
async fn run_workflow_suite(cli: &Cli, dataset: &DatasetCase) -> Result<SmallVec<[ToolRow; 4]>> {
let mut rows = SmallVec::<[ToolRow; 4]>::new();
rows.push(row(
"editor_workflow",
dataset,
"corsa-msgpack-cold",
workflow_cold(cli, dataset).await?,
));
rows.push(row(
"editor_workflow",
dataset,
"corsa-msgpack-warm",
workflow_warm(cli, dataset).await?,
));
Ok(rows)
}
fn row(workload: &str, dataset: &DatasetCase, tool: &str, stats: Stats) -> ToolRow {
ToolRow {
workload: CompactString::from(workload),
dataset: dataset.label.clone(),
tool: CompactString::from(tool),
stats,
}
}
fn tsc_command(support: &ToolSupport, overlay: &OverlayConfig) -> Command {
let mut command = Command::new(support.node_command.as_str());
command
.current_dir(&support.typescript_go_root)
.arg(&support.tsc_script)
.arg("--pretty")
.arg("false")
.arg("--noEmit")
.arg("-p")
.arg(&overlay.path);
command
}
fn tsgo_command(cli: &Cli, support: &ToolSupport, overlay: &OverlayConfig) -> Command {
let mut command = Command::new(&cli.tsgo_path);
command
.current_dir(&support.typescript_go_root)
.arg("--pretty")
.arg("false")
.arg("--noEmit")
.arg("-p")
.arg(&overlay.path);
command
}
fn eslint_command(
dataset: &DatasetCase,
support: &ToolSupport,
overlay: &OverlayConfig,
) -> Command {
let mut command = Command::new(support.node_command.as_str());
command
.current_dir(&support.workspace_root)
.arg(&support.eslint_script)
.arg("--config")
.arg(&support.eslint_config)
.arg("--no-config-lookup")
.env("TSGO_RS_BENCH_TSCONFIG", &overlay.path);
for file in &dataset.source_files {
command.arg(file.as_str());
}
command
}
fn tsgolint_command(support: &ToolSupport, overlay: &OverlayConfig) -> Command {
let mut command = Command::new(support.node_command.as_str());
command
.current_dir(&overlay.dir.path)
.arg(&support.tsgolint_script)
.arg("--tsconfig")
.arg(&overlay.path);
command
}
async fn workflow_cold(cli: &Cli, dataset: &DatasetCase) -> Result<Stats> {
measure_with_warmup(0, cli.iterations, || async {
let session = open_workflow_session(cli, dataset).await?;
let workflow = run_editor_workflow(&session).await;
let cleanup = close_workflow_session(session).await;
workflow?;
cleanup
})
.await
}
async fn workflow_warm(cli: &Cli, dataset: &DatasetCase) -> Result<Stats> {
let session = open_workflow_session(cli, dataset).await?;
let measured = measure_with_warmup(cli.warmup_iterations, cli.iterations, || async {
run_editor_workflow(&session).await
})
.await;
let cleanup = close_workflow_session(session).await;
match (measured, cleanup) {
(Ok(stats), Ok(())) => Ok(stats),
(Err(error), _) => Err(error),
(Ok(_), Err(error)) => Err(error),
}
}
async fn open_workflow_session(cli: &Cli, dataset: &DatasetCase) -> Result<WorkflowSession> {
let client = ApiClient::spawn(
ApiSpawnConfig::new(&cli.tsgo_path)
.with_cwd(&cli.root_dir)
.with_mode(ApiMode::SyncMsgpackStdio),
)
.await?;
let snapshot = client
.update_snapshot(UpdateSnapshotParams {
open_project: Some(dataset.config_wire.to_string()),
file_changes: None,
overlay_changes: None,
})
.await?;
let project = snapshot.projects[0].id.clone();
let target =
discover_bench_target(&client, &snapshot, &project, dataset.primary_file.as_str()).await?;
Ok(WorkflowSession {
client,
snapshot,
project,
file: dataset.primary_file.clone(),
target,
})
}
async fn run_editor_workflow(session: &WorkflowSession) -> Result<()> {
let _ = session
.client
.get_default_project_for_file(session.snapshot.handle.clone(), session.file.as_str())
.await?;
let _ = session
.client
.get_source_file(
session.snapshot.handle.clone(),
session.project.clone(),
session.file.as_str(),
)
.await?;
let _ = session
.client
.get_symbol_at_position(
session.snapshot.handle.clone(),
session.project.clone(),
session.file.as_str(),
session.target.position,
)
.await?;
let _ = session
.client
.get_type_of_symbol(
session.snapshot.handle.clone(),
session.project.clone(),
session.target.symbol.clone(),
)
.await?;
let ty = session
.client
.get_type_at_position(
session.snapshot.handle.clone(),
session.project.clone(),
session.file.as_str(),
session.target.position,
)
.await?
.ok_or(TsgoError::Protocol(
"workflow target no longer resolves to a type".into(),
))?;
let _ = session
.client
.type_to_string(
session.snapshot.handle.clone(),
session.project.clone(),
ty.id,
None,
None,
)
.await?;
Ok(())
}
async fn close_workflow_session(session: WorkflowSession) -> Result<()> {
let release = session.snapshot.release().await;
let close = session.client.close().await;
release?;
close
}
async fn discover_bench_target(
client: &ApiClient,
snapshot: &corsa::api::ManagedSnapshot,
project: &corsa::api::ProjectHandle,
file: &str,
) -> Result<BenchTarget> {
let source = client
.get_source_file(snapshot.handle.clone(), project.clone(), file)
.await?
.ok_or(TsgoError::Protocol(
"benchmark dataset is missing its primary file".into(),
))?;
let text = String::from_utf8_lossy(source.as_bytes());
for (position, token) in identifier_positions(text.as_ref()) {
if token.len() <= 1 || is_noise_identifier(token) {
continue;
}
if let Some(symbol) = client
.get_symbol_at_position(snapshot.handle.clone(), project.clone(), file, position)
.await?
{
return Ok(BenchTarget {
position,
symbol: symbol.id,
});
}
}
Err(TsgoError::Protocol(
"failed to discover a benchmarkable symbol in the primary file".into(),
))
}
fn identifier_positions(text: &str) -> impl Iterator<Item = (u32, &str)> {
let mut items = SmallVec::<[(u32, &str); 128]>::new();
let bytes = text.as_bytes();
let mut index = 0_usize;
while index < bytes.len() {
if !is_identifier_start(bytes[index]) {
index += 1;
continue;
}
let start = index;
index += 1;
while index < bytes.len() && is_identifier_continue(bytes[index]) {
index += 1;
}
items.push((
u32::try_from(start).unwrap_or(u32::MAX),
&text[start..index],
));
}
items.into_iter()
}
fn is_identifier_start(byte: u8) -> bool {
byte.is_ascii_alphabetic() || matches!(byte, b'_' | b'$')
}
fn is_identifier_continue(byte: u8) -> bool {
is_identifier_start(byte) || byte.is_ascii_digit()
}
fn is_noise_identifier(token: &str) -> bool {
matches!(
token,
"const"
| "let"
| "var"
| "function"
| "class"
| "interface"
| "type"
| "import"
| "export"
| "from"
| "return"
| "if"
| "else"
| "for"
| "while"
| "switch"
| "case"
| "default"
| "extends"
| "implements"
| "new"
| "true"
| "false"
| "null"
| "undefined"
)
}
impl ToolSupport {
fn discover(cli: &Cli) -> Result<Self> {
let workspace_root = cli.root_dir.clone();
let typescript_go_root = workspace_root.join("ref/typescript-go");
let tsc_script = typescript_go_root.join("node_modules/typescript/bin/tsc");
if !tsc_script.exists() {
return Err(TsgoError::Protocol(CompactString::from(
"missing ref/typescript-go/node_modules/typescript/bin/tsc; run `vp run -w bench_tooling_setup` first",
)));
}
let cli_compare_root = workspace_root.join("bench/cli_compare");
let eslint_script = cli_compare_root.join("node_modules/eslint/bin/eslint.js");
if !eslint_script.exists() {
return Err(TsgoError::Protocol(CompactString::from(
"missing bench/cli_compare/node_modules/eslint/bin/eslint.js; run `vp run -w bench_tooling_setup` first",
)));
}
let eslint_config = cli_compare_root.join("eslint.config.mjs");
let tsgolint_script = cli_compare_root.join("node_modules/oxlint-tsgolint/bin/tsgolint.js");
if !tsgolint_script.exists() {
return Err(TsgoError::Protocol(CompactString::from(
"missing bench/cli_compare/node_modules/oxlint-tsgolint/bin/tsgolint.js; run `vp run -w bench_tooling_setup` first",
)));
}
Ok(Self {
workspace_root,
typescript_go_root,
node_command: cli.node_command.clone(),
tsc_script,
eslint_script,
eslint_config,
tsgolint_script,
})
}
}
impl OverlayConfig {
fn create(cli: &Cli, dataset: &DatasetCase) -> Result<Self> {
let dir = OverlayDir::create(&cli.root_dir)?;
let path = dir.path.join(format!("{}.json", dataset.label.as_str()));
let extends = relative_path(&dir.path, &dataset.config_path);
fs::write(
&path,
serde_json::to_vec_pretty(&json!({
"extends": extends,
"compilerOptions": {
"customConditions": ["@typescript/source"]
}
}))?,
)?;
fs::write(
dir.path.join(".oxlintrc.json"),
serde_json::to_vec_pretty(&json!({
"options": {
"typeAware": true,
},
"rules": {
"typescript/await-thenable": "error",
"typescript/no-floating-promises": "error",
"typescript/no-misused-promises": "error",
"typescript/no-unnecessary-condition": "error",
"typescript/no-unnecessary-type-assertion": "error",
}
}))?,
)?;
Ok(Self { dir, path })
}
}
impl OverlayDir {
fn create(root_dir: &Path) -> Result<Self> {
let suffix = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|elapsed| elapsed.as_nanos())
.unwrap_or(0);
let path = root_dir
.join("ref/typescript-go/.cache/bench_tooling_compare")
.join(format!("overlay-{}-{suffix}", std::process::id()));
fs::create_dir_all(&path)?;
Ok(Self { path })
}
}
impl Drop for OverlayDir {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.path);
}
}
fn relative_path(from_dir: &Path, to: &Path) -> PathBuf {
let from_components = from_dir.components().collect::<Vec<_>>();
let to_components = to.components().collect::<Vec<_>>();
let mut shared = 0_usize;
while shared < from_components.len()
&& shared < to_components.len()
&& from_components[shared] == to_components[shared]
{
shared += 1;
}
let mut path = PathBuf::new();
for _ in shared..from_components.len() {
path.push("..");
}
for component in &to_components[shared..] {
path.push(component.as_os_str());
}
path
}