use std::io::{BufRead, BufReader, Read, Write};
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::mpsc::{Receiver, channel};
use std::sync::{Arc, Mutex, PoisonError};
use crate::adapters::motherduck_bridge::constants::{
BRIDGE_SCRIPT, PYTHON_VARIABLE, STDERR_TAIL_BYTES, SUPPORTED_DUCKDB,
};
use crate::adapters::motherduck_bridge::errors::BridgeError;
use crate::adapters::motherduck_bridge::models::{BridgeProcess, Replies};
pub(crate) fn start(python: &str, hidden: Vec<String>) -> Result<Arc<BridgeProcess>, BridgeError> {
let mut child = Command::new(python)
.args(["-c", BRIDGE_SCRIPT])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|error| {
BridgeError::Setup(format!(
"MotherDuck connects through Python's duckdb package ({SUPPORTED_DUCKDB}), and {python} cannot run ({error}): install sqlload[motherduck] with pip or uv, or set {PYTHON_VARIABLE} to a Python that has {SUPPORTED_DUCKDB}"
))
})?;
let (Some(mut stdin), Some(stdout), Some(stderr)) =
(child.stdin.take(), child.stdout.take(), child.stderr.take())
else {
return Err(BridgeError::Setup(format!(
"{python} started without its pipes"
)));
};
let (lines, outgoing) = channel::<String>();
let replies = Arc::new(Mutex::new(Replies::default()));
let tail = Arc::new(Mutex::new(String::new()));
std::thread::spawn(move || {
for line in outgoing {
if stdin
.write_all(format!("{line}\n").as_bytes())
.and_then(|()| stdin.flush())
.is_err()
{
break;
}
}
drop(stdin);
let _ = child.wait();
});
let kept = tail.clone();
let reading_stderr = std::thread::spawn(move || keep_tail(stderr, &kept, &hidden));
let routed = replies.clone();
let (shown, kept) = (python.to_owned(), tail.clone());
std::thread::spawn(move || match route_replies(stdout, &routed) {
Some(broken) => fail(
&routed,
format!("the MotherDuck bridge on {shown} {broken}"),
),
None => {
let _ = reading_stderr.join();
stop(&routed, &shown, &kept);
}
});
Ok(Arc::new(BridgeProcess {
python: python.to_owned(),
lines,
replies,
next_request: AtomicU64::new(1),
next_cursor: AtomicU64::new(1),
stderr: tail,
}))
}
pub(crate) fn send(
process: &BridgeProcess,
mut request: serde_json::Value,
) -> Result<Receiver<Result<serde_json::Value, BridgeError>>, BridgeError> {
let id = process.next_request.fetch_add(1, Ordering::Relaxed);
request["id"] = id.into();
let (reply, received) = channel();
let mut replies = process
.replies
.lock()
.unwrap_or_else(PoisonError::into_inner);
if let Some(stopped) = &replies.stopped {
return Err(BridgeError::Stopped(stopped.clone()));
}
replies.waiting.insert(id, reply);
drop(replies);
if process.lines.send(request.to_string()).is_err() {
return Err(BridgeError::Stopped(format!(
"the MotherDuck bridge on {} is closed",
process.python
)));
}
Ok(received)
}
pub(crate) fn failure(reply: &serde_json::Value) -> Option<(String, String)> {
if reply["ok"].as_bool() == Some(true) {
return None;
}
let text = |key: &str| reply[key].as_str().unwrap_or_default().to_owned();
Some((text("error"), text("type")))
}
fn route_replies(stdout: impl Read, replies: &Mutex<Replies>) -> Option<String> {
for line in BufReader::new(stdout).lines() {
let line = match line {
Ok(line) => line,
Err(error) => return Some(format!("cannot be read: {error}")),
};
let reply = match serde_json::from_str::<serde_json::Value>(&line) {
Ok(reply) => reply,
Err(error) => return Some(format!("replied with something other than JSON ({error})")),
};
let waiting = reply["id"].as_u64().and_then(|id| {
replies
.lock()
.unwrap_or_else(PoisonError::into_inner)
.waiting
.remove(&id)
});
if let Some(waiting) = waiting {
let _ = waiting.send(Ok(reply));
}
}
None
}
fn stop(replies: &Mutex<Replies>, python: &str, tail: &Mutex<String>) {
let said = tail
.lock()
.unwrap_or_else(PoisonError::into_inner)
.trim()
.to_owned();
let stopped = match said.is_empty() {
true => format!("the MotherDuck bridge on {python} stopped"),
false => format!("the MotherDuck bridge on {python} stopped: {said}"),
};
fail(replies, stopped);
}
fn fail(replies: &Mutex<Replies>, stopped: String) {
let mut replies = replies.lock().unwrap_or_else(PoisonError::into_inner);
for (_, waiting) in replies.waiting.drain() {
let _ = waiting.send(Err(BridgeError::Stopped(stopped.clone())));
}
replies.stopped = Some(stopped);
}
fn keep_tail(stderr: impl Read, tail: &Mutex<String>, hidden: &[String]) {
for line in BufReader::new(stderr).lines() {
let Ok(mut line) = line else { break };
for secret in hidden.iter().filter(|secret| !secret.is_empty()) {
line = line.replace(secret.as_str(), "<redacted>");
}
tracing::debug!(line, "MotherDuck bridge");
let mut kept = tail.lock().unwrap_or_else(PoisonError::into_inner);
kept.push_str(&line);
kept.push('\n');
if kept.len() > STDERR_TAIL_BYTES {
let mut cut = kept.len() - STDERR_TAIL_BYTES;
while !kept.is_char_boundary(cut) {
cut += 1;
}
kept.replace_range(..cut, "");
}
}
}