#![allow(clippy::missing_errors_doc)]
use std::io::{BufRead, BufReader, Write};
use std::path::PathBuf;
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::thread::JoinHandle;
use serde::{Deserialize, Serialize};
use crate::backend::{
Backend, BackendCapabilities, BackendError, DispatchBackend, KernelArg, KernelHandle,
};
use crate::sched::notify_ring::{NotifyHub, NotifySendResult, NotifySender};
const BRIDGE_PY: &str = include_str!("tpu_jax_bridge.py");
pub struct TpuJaxBackend {
device_id: u32,
caps: BackendCapabilities,
devices: Vec<String>,
hub: NotifyHub<BridgeRequest>,
tx: NotifySender<BridgeRequest>,
owner: Option<JoinHandle<()>>,
}
struct BridgeState {
child: Option<Child>,
stdin: ChildStdin,
stdout: BufReader<ChildStdout>,
script_path: PathBuf,
devices: Vec<String>,
}
struct BridgeRequest {
body: String,
reply: std::sync::mpsc::Sender<Result<String, BackendError>>,
}
static REPLIES_UNCLAIMED: AtomicU64 = AtomicU64::new(0);
pub fn replies_unclaimed() -> u64 {
REPLIES_UNCLAIMED.load(Ordering::Relaxed)
}
impl std::fmt::Debug for TpuJaxBackend {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TpuJaxBackend")
.field("device_id", &self.device_id)
.finish()
}
}
impl TpuJaxBackend {
pub fn new() -> Result<Self, BackendError> {
Self::with_device(0)
}
pub fn with_device(device_id: u32) -> Result<Self, BackendError> {
let script_path = write_bridge_script()?;
let (state, pong) = open_bridge(script_path)?;
let devices = pong.devices;
const BRIDGE_RING_CAPACITY: usize = 256;
let hub = NotifyHub::<BridgeRequest>::new(BRIDGE_RING_CAPACITY, 1);
let tx = hub.sender();
let hub_for_owner = hub.clone();
let owner = std::thread::Builder::new()
.name(format!("flynnel-tpu-jax-{device_id}"))
.spawn(move || owner_loop(state, &hub_for_owner))
.map_err(|_| BackendError::DeviceUnavailable(Backend::Tpu { device_id }))?;
Ok(Self {
device_id,
caps: probe_capabilities(),
devices,
hub,
tx,
owner: Some(owner),
})
}
pub fn devices(&self) -> Vec<String> {
self.devices.clone()
}
}
fn owner_loop(mut state: BridgeState, hub: &NotifyHub<BridgeRequest>) {
let rx = hub.register_consumer();
while let Some(request) = rx.recv() {
let outcome = exchange(&mut state, &request.body);
if let Err(unclaimed) = request.reply.send(outcome) {
REPLIES_UNCLAIMED.fetch_add(1, Ordering::Relaxed);
drop(unclaimed);
}
}
let farewell = serde_json::json!({"op": "shutdown"});
match writeln!(state.stdin, "{farewell}") {
Ok(()) => {}
Err(gone) => drop(gone),
}
if let Some(mut child) = state.child.take() {
match child.wait() {
Ok(status) => {
if !status.success() {
notice!("tpu_jax: the bridge process ended with {status}");
}
}
Err(unwaitable) => {
notice!("tpu_jax: the bridge process could not be waited for: {unwaitable}");
}
}
}
match std::fs::remove_file(&state.script_path) {
Ok(()) => {}
Err(absent) => drop(absent),
}
}
impl Drop for TpuJaxBackend {
fn drop(&mut self) {
self.hub.shutdown();
if let Some(owner) = self.owner.take() {
match owner.join() {
Ok(()) => {}
Err(panicked) => drop(panicked),
}
}
}
}
impl DispatchBackend for TpuJaxBackend {
fn id(&self) -> Backend {
Backend::Tpu {
device_id: self.device_id,
}
}
fn capabilities(&self) -> BackendCapabilities {
self.caps
}
fn dispatch_parallel_for(&self, count: u32, work: &(dyn Fn(u32) + Send + Sync)) {
if count == 0 {
return;
}
std::thread::scope(|scope| {
let threads = (count as usize).min(
std::thread::available_parallelism()
.map(std::num::NonZeroUsize::get)
.unwrap_or(1),
);
let chunks = count.div_ceil(threads as u32);
for t in 0..threads as u32 {
let lo = t.saturating_mul(chunks);
let hi = (lo + chunks).min(count);
if lo >= hi {
continue;
}
scope.spawn(move || {
for i in lo..hi {
work(i);
}
});
}
});
}
fn dispatch_one(&self, work: Box<dyn FnOnce() + Send>) {
std::thread::spawn(work);
}
fn register_kernel(&self, name: &str, source: &[u8]) -> Result<KernelHandle, BackendError> {
let source_str = std::str::from_utf8(source).map_err(|e| {
BackendError::KernelCompile(format!("source must be UTF-8 Python: {e}"))
})?;
let req = RegisterRequest {
op: "register",
name,
source: source_str,
};
let resp: RegisterResponse = transact(&self.tx, &req)?;
if !resp.ok {
return Err(BackendError::KernelCompile(
resp.error.unwrap_or_else(|| "register failed".into()),
));
}
let handle =
resp.handle.ok_or_else(|| BackendError::KernelCompile("missing handle".into()))?;
Ok(KernelHandle(handle))
}
fn dispatch_kernel(
&self,
handle: KernelHandle,
count: u32,
args: &[KernelArg<'_>],
) -> Result<(), BackendError> {
let json_args = args
.iter()
.map(arg_to_json)
.collect::<Result<Vec<_>, _>>()?;
let req = DispatchRequest {
op: "dispatch",
handle: handle.0,
count,
args: json_args,
};
let resp: PlainResponse = transact(&self.tx, &req)?;
if !resp.ok {
return Err(BackendError::Launch(
resp.error.unwrap_or_else(|| "dispatch failed".into()),
));
}
Ok(())
}
}
#[derive(Serialize)]
struct PingRequest<'a> {
op: &'a str,
}
#[derive(Deserialize)]
struct PingResponse {
ok: bool,
#[serde(default)]
devices: Vec<String>,
#[serde(default)]
error: Option<String>,
#[serde(default)]
#[allow(dead_code)]
jax_version: Option<String>,
}
#[derive(Serialize)]
struct RegisterRequest<'a> {
op: &'a str,
name: &'a str,
source: &'a str,
}
#[derive(Deserialize)]
struct RegisterResponse {
ok: bool,
#[serde(default)]
handle: Option<u64>,
#[serde(default)]
error: Option<String>,
}
#[derive(Serialize)]
struct DispatchRequest<'a> {
op: &'a str,
handle: u64,
count: u32,
args: Vec<serde_json::Value>,
}
#[derive(Deserialize)]
struct PlainResponse {
ok: bool,
#[serde(default)]
error: Option<String>,
}
fn write_bridge_script() -> Result<PathBuf, BackendError> {
let mut path = std::env::temp_dir();
let pid = std::process::id();
path.push(format!("flynnel_tpu_jax_bridge_{pid}.py"));
std::fs::write(&path, BRIDGE_PY).map_err(|e| {
BackendError::DeviceUnavailable(Backend::Tpu { device_id: 0 })
.map_io_context(format!("write bridge script: {e}"))
})?;
Ok(path)
}
fn open_bridge(script_path: PathBuf) -> Result<(BridgeState, PingResponse), BackendError> {
let mut last = String::from("no python3 or python interpreter on PATH");
for interpreter in ["python3", "python"] {
let started = Command::new(interpreter)
.arg(&script_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn();
let mut child = match started {
Ok(child) => child,
Err(absent) => {
last = format!("{interpreter}: {absent}");
continue;
}
};
let (Some(stdin), Some(stdout)) = (child.stdin.take(), child.stdout.take()) else {
last = format!("{interpreter}: started without both pipes");
reap(interpreter, child);
continue;
};
let mut state = BridgeState {
child: Some(child),
stdin,
stdout: BufReader::new(stdout),
script_path: script_path.clone(),
devices: Vec::new(),
};
match ping_handshake(&mut state) {
Ok(pong) => {
state.devices = pong.devices.clone();
return Ok((state, pong));
}
Err(refused) => {
last = format!("{interpreter}: {refused}");
if let Some(child) = state.child.take() {
reap(interpreter, child);
}
}
}
}
match std::fs::remove_file(&script_path) {
Ok(()) => {}
Err(absent) => drop(absent),
}
Err(BackendError::DeviceUnavailable(Backend::Tpu { device_id: 0 }).map_io_context(last))
}
fn reap(interpreter: &str, mut child: Child) {
match child.kill() {
Ok(()) => {}
Err(gone) => {
notice!("tpu_jax: the {interpreter} child was already gone: {gone}");
}
}
match child.wait() {
Ok(_ended) => {}
Err(unwaitable) => {
notice!("tpu_jax: the {interpreter} child could not be waited for: {unwaitable}");
}
}
}
fn ping_handshake(state: &mut BridgeState) -> Result<PingResponse, BackendError> {
let req = PingRequest { op: "ping" };
let req_json = serde_json::to_string(&req).map_err(|e| {
BackendError::DeviceUnavailable(Backend::Tpu { device_id: 0 })
.map_io_context(format!("ping serialize: {e}"))
})?;
writeln!(state.stdin, "{req_json}").map_err(|e| {
BackendError::DeviceUnavailable(Backend::Tpu { device_id: 0 })
.map_io_context(format!("ping stdin write: {e}"))
})?;
state.stdin.flush().map_err(|e| {
BackendError::DeviceUnavailable(Backend::Tpu { device_id: 0 })
.map_io_context(format!("ping stdin flush: {e}"))
})?;
let mut line = String::new();
state.stdout.read_line(&mut line).map_err(|e| {
BackendError::DeviceUnavailable(Backend::Tpu { device_id: 0 })
.map_io_context(format!("ping stdout read: {e}"))
})?;
if line.trim().is_empty() {
return Err(BackendError::DeviceUnavailable(Backend::Tpu { device_id: 0 })
.map_io_context("ping: bridge sent empty response".into()));
}
let pong: PingResponse = serde_json::from_str(line.trim()).map_err(|e| {
BackendError::DeviceUnavailable(Backend::Tpu { device_id: 0 })
.map_io_context(format!("ping parse `{}`: {e}", line.trim()))
})?;
if !pong.ok {
return Err(BackendError::DeviceUnavailable(Backend::Tpu { device_id: 0 })
.map_io_context(
pong.error.unwrap_or_else(|| "ping reported not-ok".into()),
));
}
if pong.devices.is_empty() {
return Err(BackendError::DeviceUnavailable(Backend::Tpu { device_id: 0 })
.map_io_context("ping: jax reported zero devices".into()));
}
Ok(pong)
}
fn exchange(state: &mut BridgeState, body: &str) -> Result<String, BackendError> {
writeln!(state.stdin, "{body}")
.map_err(|e| BackendError::Launch(format!("stdin write: {e}")))?;
state
.stdin
.flush()
.map_err(|e| BackendError::Launch(format!("stdin flush: {e}")))?;
let mut line = String::new();
state
.stdout
.read_line(&mut line)
.map_err(|e| BackendError::Launch(format!("stdout read: {e}")))?;
Ok(line)
}
fn transact<Req, Resp>(tx: &NotifySender<BridgeRequest>, req: &Req) -> Result<Resp, BackendError>
where
Req: Serialize,
Resp: for<'de> Deserialize<'de>,
{
let body = serde_json::to_string(req)
.map_err(|e| BackendError::Launch(format!("request serialize: {e}")))?;
let (reply, answer) = std::sync::mpsc::channel();
match tx.send(BridgeRequest { body, reply }) {
NotifySendResult::Ok => {}
NotifySendResult::Closed(refused) => {
drop(refused);
return Err(BackendError::Launch(
"the bridge is shut down, so this request was not sent".into(),
));
}
}
let line = answer
.recv()
.map_err(|_| BackendError::Launch("the bridge thread ended before answering".into()))??;
serde_json::from_str(line.trim())
.map_err(|e| BackendError::Launch(format!("response parse `{}`: {e}", line.trim())))
}
fn arg_to_json(arg: &KernelArg<'_>) -> Result<serde_json::Value, BackendError> {
let v = match *arg {
KernelArg::I32(v) => serde_json::json!({"i32": v}),
KernelArg::I64(v) => serde_json::json!({"i64": v}),
KernelArg::U32(v) => serde_json::json!({"u32": v}),
KernelArg::U64(v) => serde_json::json!({"u64": v}),
KernelArg::F32(v) => serde_json::json!({"f32": v}),
KernelArg::F64(v) => serde_json::json!({"f64": v}),
KernelArg::DevicePtr(p) => serde_json::json!({"device_ptr": p as u64}),
KernelArg::HostSlice(_) | KernelArg::Buffer(_) => return Err(BackendError::NotSupported),
};
Ok(v)
}
fn probe_capabilities() -> BackendCapabilities {
BackendCapabilities {
simt_width: 128,
max_threads_in_flight: 200_000,
launch_latency_ns: 100_000,
h2d_bw_bytes_per_sec: 25_000_000_000,
}
}
trait WithIoContext: Sized {
fn map_io_context(self, msg: String) -> Self;
}
impl WithIoContext for BackendError {
fn map_io_context(self, msg: String) -> Self {
match self {
BackendError::DeviceUnavailable(b) => {
notice!("[flynnel::tpu_jax] {}: {msg}", b.name());
BackendError::DeviceUnavailable(b)
}
other => other,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn construction_matches_python_jax_availability() {
let res = TpuJaxBackend::new();
match res {
Ok(b) => {
assert_eq!(b.id(), Backend::Tpu { device_id: 0 });
assert_eq!(b.capabilities().simt_width, 128);
let devices = b.devices();
assert!(
!devices.is_empty(),
"successful ping must report at least one device"
);
}
Err(BackendError::DeviceUnavailable(Backend::Tpu { device_id })) => {
assert_eq!(device_id, 0);
}
Err(other) => panic!("unexpected construction error: {other}"),
}
}
#[test]
fn capabilities_have_expected_shape() {
let caps = probe_capabilities();
assert_eq!(caps.simt_width, 128);
assert_eq!(caps.h2d_bw_bytes_per_sec, 25_000_000_000);
assert!(caps.launch_latency_ns >= 10_000);
}
}