use std::fmt::Write as _;
use std::path::{Path, PathBuf};
use incin_core::exec::{Capabilities, CapabilityQuery, LayoutClass, MathMode, SupportLevel};
use incin_core::shapes::OperationKind;
use incin_core::tensor::device::{DeviceId, DeviceKind};
use incin_core::tensor::dtype::DTypeId;
use serde::Serialize;
pub const SCHEMA_VERSION: u32 = 1;
const DEVICE_ORDER: &[DeviceKind] = &[DeviceKind::Cpu, DeviceKind::Cuda, DeviceKind::Wgpu];
pub trait Host {
fn incin_version(&self) -> String;
fn rustc_version(&self) -> Option<String>;
fn features(&self) -> Vec<Feature>;
fn cpu_isa(&self) -> Vec<IsaFeature>;
fn compiled_in(&self, kind: DeviceKind) -> bool;
fn probe(&self, kind: DeviceKind) -> Option<DeviceId>;
fn caches(&self) -> Vec<Cache>;
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Feature {
pub name: String,
pub enabled: bool,
}
impl Feature {
#[must_use]
pub fn new(name: &str, enabled: bool) -> Self {
Self {
name: name.to_string(),
enabled,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct IsaFeature {
pub name: String,
pub available: bool,
}
impl IsaFeature {
#[must_use]
pub fn new(name: &str, available: bool) -> Self {
Self {
name: name.to_string(),
available,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum CacheState {
Writable,
NotWritable,
Absent,
Unset,
NotCompiled,
}
impl CacheState {
const fn as_str(self) -> &'static str {
match self {
Self::Writable => "writable",
Self::NotWritable => "not writable",
Self::Absent => "absent",
Self::Unset => "unset",
Self::NotCompiled => "not compiled",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Cache {
pub name: String,
pub path: Option<String>,
pub state: CacheState,
pub detail: Option<String>,
}
impl Cache {
#[must_use]
pub fn new(name: &str, path: Option<PathBuf>, state: CacheState) -> Self {
Self {
name: name.to_string(),
path: path.map(|p| p.display().to_string()),
state,
detail: None,
}
}
#[must_use]
pub fn with_detail(mut self, detail: &str) -> Self {
self.detail = Some(detail.to_string());
self
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct DeviceReport {
pub kind: String,
pub compiled_in: bool,
pub available: bool,
pub ordinal: Option<usize>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Probe {
pub device: String,
pub operation: String,
pub dtype: String,
pub rank: usize,
pub training: bool,
pub support: String,
pub reason: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Severity {
Note,
Warning,
Error,
}
impl Severity {
const fn as_str(self) -> &'static str {
match self {
Self::Note => "note",
Self::Warning => "warning",
Self::Error => "error",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Finding {
pub severity: Severity,
pub code: String,
pub message: String,
pub remedy: Option<String>,
}
impl Finding {
fn new(severity: Severity, code: &str, message: String, remedy: Option<&str>) -> Self {
Self {
severity,
code: code.to_string(),
message,
remedy: remedy.map(ToString::to_string),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Toolchain {
pub incin: String,
pub rustc: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Report {
pub schema_version: u32,
pub toolchain: Toolchain,
pub features: Vec<Feature>,
pub cpu_isa: Vec<IsaFeature>,
pub devices: Vec<DeviceReport>,
pub caches: Vec<Cache>,
pub probes: Vec<Probe>,
pub findings: Vec<Finding>,
}
struct ProbeSpec {
operation: OperationKind,
dtype: DTypeId,
rank: usize,
training: bool,
}
const fn spec(operation: OperationKind, dtype: DTypeId, rank: usize, training: bool) -> ProbeSpec {
ProbeSpec {
operation,
dtype,
rank,
training,
}
}
const PROBES: &[ProbeSpec] = &[
spec(OperationKind::Pointwise, DTypeId::F32, 2, false),
spec(OperationKind::Pointwise, DTypeId::F32, 2, true),
spec(OperationKind::MatMul, DTypeId::F32, 2, false),
spec(OperationKind::MatMul, DTypeId::F16, 2, false),
spec(OperationKind::Reduction, DTypeId::F32, 2, false),
spec(OperationKind::Reduction, DTypeId::F64, 2, false),
spec(OperationKind::Conv2d, DTypeId::F32, 4, true),
spec(OperationKind::Normalization, DTypeId::F32, 2, true),
];
impl Report {
#[must_use]
pub fn gather(host: &dyn Host) -> Self {
let toolchain = Toolchain {
incin: host.incin_version(),
rustc: host.rustc_version(),
};
let features = host.features();
let cpu_isa = host.cpu_isa();
let detected: Vec<Option<DeviceId>> =
DEVICE_ORDER.iter().map(|&kind| host.probe(kind)).collect();
let devices: Vec<DeviceReport> = DEVICE_ORDER
.iter()
.zip(&detected)
.map(|(&kind, device)| DeviceReport {
kind: device_name(kind).to_string(),
compiled_in: host.compiled_in(kind),
available: device.is_some(),
ordinal: device.map(DeviceId::ordinal),
})
.collect();
let caches = host.caches();
let probes: Vec<Probe> = DEVICE_ORDER
.iter()
.zip(&detected)
.filter(|(_, device)| device.is_some())
.flat_map(|(&kind, _)| PROBES.iter().map(move |probe| run_probe(kind, probe)))
.collect();
let findings = findings(&toolchain, &features, &cpu_isa, &devices, &caches);
Self {
schema_version: SCHEMA_VERSION,
toolchain,
features,
cpu_isa,
devices,
caches,
probes,
findings,
}
}
#[must_use]
pub fn exit_code(&self) -> i32 {
if self
.findings
.iter()
.any(|finding| finding.severity == Severity::Error)
{
EXIT_FINDINGS
} else {
EXIT_OK
}
}
#[must_use]
pub fn to_text(&self) -> String {
let mut out = String::new();
out.push_str("[toolchain]\n");
let _ = writeln!(out, "incin: {}", self.toolchain.incin);
let _ = writeln!(
out,
"rustc: {}",
self.toolchain.rustc.as_deref().unwrap_or("unknown")
);
out.push_str("\n[features]\n");
for feature in &self.features {
let _ = writeln!(
out,
"{}: {}",
feature.name,
if feature.enabled { "on" } else { "off" }
);
}
out.push_str("\n[cpu]\n");
if self.cpu_isa.is_empty() {
out.push_str("isa: none used by any kernel on this target\n");
} else {
for isa in &self.cpu_isa {
let _ = writeln!(
out,
"{}: {}",
isa.name,
if isa.available {
"available"
} else {
"unavailable (scalar path)"
}
);
}
}
out.push_str("\n[devices]\n");
for device in &self.devices {
let state = match (device.compiled_in, device.ordinal) {
(true, Some(ordinal)) => {
format!("compiled, available as {}:{ordinal}", device.kind)
}
(true, None) => "compiled, no device found".to_string(),
(false, Some(ordinal)) => {
format!("not compiled, but answered as {}:{ordinal}", device.kind)
}
(false, None) => "not compiled".to_string(),
};
let _ = writeln!(out, "{}: {state}", device.kind);
}
out.push_str("\n[caches]\n");
for cache in &self.caches {
let mut line = match &cache.path {
Some(path) => format!("{path} ({})", cache.state.as_str()),
None => cache.state.as_str().to_string(),
};
if let Some(detail) = &cache.detail {
line.push_str(&format!(" — {detail}"));
}
let _ = writeln!(out, "{}: {line}", cache.name);
}
out.push_str("\n[probes]\n");
if self.probes.is_empty() {
out.push_str("none: no device answered\n");
}
for probe in &self.probes {
let mode = if probe.training {
"training"
} else {
"inference"
};
let answer = match &probe.reason {
Some(reason) => format!("{} ({reason})", probe.support),
None => probe.support.clone(),
};
let _ = writeln!(
out,
"{} {} {} rank {} {mode}: {answer}",
probe.device, probe.operation, probe.dtype, probe.rank
);
}
out.push_str("\n[findings]\n");
if self.findings.is_empty() {
out.push_str("none\n");
}
for finding in &self.findings {
let _ = writeln!(
out,
"{} {}: {}",
finding.severity.as_str(),
finding.code,
finding.message
);
if let Some(remedy) = &finding.remedy {
let _ = writeln!(out, " remedy: {remedy}");
}
}
out
}
pub fn to_json(&self) -> Result<String, serde_json::Error> {
serde_json::to_string_pretty(self)
}
}
fn run_probe(kind: DeviceKind, spec: &ProbeSpec) -> Probe {
let query = CapabilityQuery {
operation: incin_core::exec::OperationIdentity::Builtin(spec.operation),
dtype: spec.dtype.descriptor(),
layout: LayoutClass::Contiguous,
rank: spec.rank,
training: spec.training,
math_mode: MathMode::Precise,
};
let level = incin_backends::capability::registry(kind).support(&query);
let (support, reason) = match level {
SupportLevel::Native => ("native", None),
SupportLevel::Composed => ("composed", None),
SupportLevel::Fallback => ("fallback", None),
SupportLevel::Unsupported(why) => ("unsupported", Some(why.to_string())),
};
Probe {
device: device_name(kind).to_string(),
operation: spec.operation.name().to_string(),
dtype: dtype_name(spec.dtype).to_string(),
rank: spec.rank,
training: spec.training,
support: support.to_string(),
reason,
}
}
fn findings(
toolchain: &Toolchain,
_features: &[Feature],
cpu_isa: &[IsaFeature],
devices: &[DeviceReport],
caches: &[Cache],
) -> Vec<Finding> {
let mut out = Vec::new();
if devices.iter().all(|device| !device.compiled_in) {
out.push(Finding::new(
Severity::Error,
"no-backend-compiled",
"no backend family is compiled into this build, so nothing can execute".to_string(),
Some("rebuild with at least one of the cpu, cuda, or wgpu features"),
));
}
if toolchain.rustc.is_none() {
out.push(Finding::new(
Severity::Warning,
"toolchain-unknown",
"rustc could not be run, so the toolchain version is unknown".to_string(),
Some("check that rustc is on PATH"),
));
}
for device in devices {
if device.compiled_in && !device.available {
out.push(Finding::new(
Severity::Warning,
"backend-unavailable",
format!(
"{} is compiled into this build but no device answered",
device.kind
),
Some(match device.kind.as_str() {
"cuda" => "check that a driver is installed and visible to this process",
"wgpu" => "check that a non-software adapter is visible to this process",
_ => "check that the backend's runtime is present",
}),
));
}
}
for cache in caches {
if cache.state == CacheState::NotWritable {
out.push(Finding::new(
Severity::Warning,
"cache-not-writable",
format!(
"the {} cache at {} is not writable",
cache.name,
cache.path.as_deref().unwrap_or("an unknown path")
),
Some("fix the directory's permissions or point the cache elsewhere"),
));
}
}
for isa in cpu_isa {
if !isa.available {
out.push(Finding::new(
Severity::Note,
"isa-unavailable",
format!(
"{} is not available, so the CPU kernels that branch on it take their scalar path",
isa.name
),
None,
));
}
}
out
}
fn device_name(kind: DeviceKind) -> &'static str {
kind.name()
}
fn dtype_name(dtype: DTypeId) -> &'static str {
dtype.name()
}
#[derive(Debug, Clone, Copy, Default)]
pub struct HostMachine;
fn compiled_features() -> Vec<Feature> {
vec![
Feature::new("std", cfg!(feature = "std")),
Feature::new("nightly", cfg!(feature = "nightly")),
Feature::new("cpu", cfg!(feature = "cpu")),
Feature::new("cpu-blas", cfg!(feature = "cpu-blas")),
Feature::new("cuda", cfg!(feature = "cuda")),
Feature::new("wgpu", cfg!(feature = "wgpu")),
Feature::new("external-candle", cfg!(feature = "external-candle")),
Feature::new("metal", cfg!(feature = "metal")),
Feature::new("metal-mps", cfg!(feature = "metal-mps")),
Feature::new("autotune", cfg!(feature = "autotune")),
Feature::new("train", cfg!(feature = "train")),
Feature::new("distributed", cfg!(feature = "distributed")),
Feature::new(
"distributed-reference",
cfg!(feature = "distributed-reference"),
),
Feature::new("distributed-nccl", cfg!(feature = "distributed-nccl")),
Feature::new("telemetry", cfg!(feature = "telemetry")),
Feature::new("test-utils", cfg!(feature = "test-utils")),
Feature::new("backend-authoring", cfg!(feature = "backend-authoring")),
Feature::new("compiled", cfg!(feature = "compiled")),
Feature::new("data-hub", cfg!(feature = "data-hub")),
Feature::new("hardware-tests", cfg!(feature = "hardware-tests")),
]
}
fn detected_isa() -> Vec<IsaFeature> {
#[cfg(target_arch = "x86_64")]
{
vec![IsaFeature::new(
"avx2",
std::arch::is_x86_feature_detected!("avx2"),
)]
}
#[cfg(target_arch = "aarch64")]
{
vec![IsaFeature::new("neon", true)]
}
#[cfg(target_arch = "wasm32")]
{
vec![IsaFeature::new("simd128", cfg!(target_feature = "simd128"))]
}
#[cfg(not(any(
target_arch = "x86_64",
target_arch = "aarch64",
target_arch = "wasm32"
)))]
{
Vec::new()
}
}
#[must_use]
pub fn cache_state(path: &Path) -> CacheState {
match std::fs::metadata(path) {
Ok(metadata) if metadata.permissions().readonly() => CacheState::NotWritable,
Ok(_) => CacheState::Writable,
Err(_) => CacheState::Absent,
}
}
#[cfg(feature = "telemetry")]
fn telemetry_cache() -> Cache {
match incin_telemetry::run_dir::default_run_dir_path() {
Ok(path) => {
let state = cache_state(&path);
Cache::new("telemetry-runs", Some(path), state)
}
Err(_) => Cache::new("telemetry-runs", None, CacheState::Unset)
.with_detail("no XDG data directory could be resolved"),
}
}
#[cfg(not(feature = "telemetry"))]
fn telemetry_cache() -> Cache {
Cache::new("telemetry-runs", None, CacheState::NotCompiled)
.with_detail("enable the telemetry feature")
}
impl Host for HostMachine {
fn incin_version(&self) -> String {
env!("CARGO_PKG_VERSION").to_string()
}
fn rustc_version(&self) -> Option<String> {
let output = std::process::Command::new("rustc")
.arg("--version")
.output()
.ok()?;
if !output.status.success() {
return None;
}
let text = String::from_utf8(output.stdout).ok()?;
let trimmed = text.trim();
(!trimmed.is_empty()).then(|| trimmed.to_string())
}
fn features(&self) -> Vec<Feature> {
compiled_features()
}
fn cpu_isa(&self) -> Vec<IsaFeature> {
detected_isa()
}
fn compiled_in(&self, kind: DeviceKind) -> bool {
incin_backends::detect::is_compiled_in(kind)
}
fn probe(&self, kind: DeviceKind) -> Option<DeviceId> {
incin_backends::detect::probe(kind)
}
fn caches(&self) -> Vec<Cache> {
let hub = match std::env::var_os("INCIN_HUB_CACHE_DIR") {
Some(dir) => {
let path = PathBuf::from(dir);
let state = cache_state(&path);
Cache::new("hub", Some(path), state)
}
None => Cache::new("hub", None, CacheState::Unset)
.with_detail("INCIN_HUB_CACHE_DIR is unset; hf-hub chooses"),
};
vec![hub, telemetry_cache()]
}
}
pub const EXIT_OK: i32 = 0;
pub const EXIT_FINDINGS: i32 = 1;
pub const EXIT_USAGE: i32 = 2;
#[must_use]
pub fn run(args: &[String]) -> (String, i32) {
let mut json = false;
for arg in args {
match arg.as_str() {
"--json" => json = true,
other => {
return (
format!("error: unknown argument `{other}` for doctor (expected --json)\n"),
EXIT_USAGE,
);
}
}
}
let report = Report::gather(&HostMachine);
let rendered = if json {
match report.to_json() {
Ok(text) => format!("{text}\n"),
Err(error) => {
return (
format!("error: could not render JSON: {error}\n"),
EXIT_USAGE,
);
}
}
} else {
report.to_text()
};
(rendered, report.exit_code())
}