clipasm 0.5.0

A strict, typed language for compiling and rendering Video and Audio graphs
Documentation
use std::io::{self, BufRead, BufReader};
use std::process::{Command, ExitStatus, Stdio};
use std::thread::JoinHandle;

use crate::diagnostic::{BuiltinDiagnostic, Diagnostic, Result};
use crate::process::{self as child_process, Child, ReaderError, RetainedOutput};
use crate::source::SourceSpan;

const CAPTURE_LIMIT: usize = 8 * 1024 * 1024;
const STDERR_TAIL_LIMIT: usize = 64 * 1024;

pub(crate) struct CapturedOutput {
    pub(crate) stdout: Vec<u8>,
    pub(crate) stderr: Vec<u8>,
}

pub(crate) fn capture(
    mut command: Command,
    diagnostic: BuiltinDiagnostic,
    span: &SourceSpan,
) -> Result<CapturedOutput> {
    let debug = format!("{command:?}");
    command
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let mut child = spawn(command, diagnostic, span, &debug)?;
    let stdout = child_process::take_stdout(&mut child).expect("piped media-tool stdout");
    let stderr = child_process::take_stderr(&mut child).expect("piped media-tool stderr");
    let stdout_reader =
        std::thread::spawn(move || child_process::read_prefix(stdout, CAPTURE_LIMIT));
    let stderr_reader = std::thread::spawn(move || child_process::read_tail(stderr, CAPTURE_LIMIT));
    let status = wait(&mut child, diagnostic, span, &debug);
    let stdout = join_reader(stdout_reader, "stdout", diagnostic, span, &debug);
    let stderr = join_reader(stderr_reader, "stderr", diagnostic, span, &debug);
    let status = status?;
    let stdout = stdout?;
    let stderr = stderr?;

    if !status.success() {
        return Err(exit_diagnostic(
            diagnostic,
            span,
            &debug,
            status,
            &stderr.bytes,
            stderr.truncated,
            CAPTURE_LIMIT,
        ));
    }
    if stdout.truncated {
        return Err(output_limit_diagnostic(
            diagnostic,
            span,
            &debug,
            "stdout",
            CAPTURE_LIMIT,
        ));
    }
    if stderr.truncated {
        return Err(output_limit_diagnostic(
            diagnostic,
            span,
            &debug,
            "stderr",
            CAPTURE_LIMIT,
        ));
    }

    Ok(CapturedOutput {
        stdout: stdout.bytes,
        stderr: stderr.bytes,
    })
}

pub(crate) fn run(
    mut command: Command,
    diagnostic: BuiltinDiagnostic,
    span: &SourceSpan,
) -> Result<()> {
    let debug = format!("{command:?}");
    command
        .stdin(Stdio::null())
        .stdout(Stdio::null())
        .stderr(Stdio::piped());
    let mut child = spawn(command, diagnostic, span, &debug)?;
    let stderr = child_process::take_stderr(&mut child).expect("piped media-tool stderr");
    let stderr_reader =
        std::thread::spawn(move || child_process::read_tail(stderr, STDERR_TAIL_LIMIT));
    let status = wait(&mut child, diagnostic, span, &debug);
    let stderr = join_reader(stderr_reader, "stderr", diagnostic, span, &debug);
    let status = status?;
    let stderr = stderr?;
    if !status.success() {
        return Err(exit_diagnostic(
            diagnostic,
            span,
            &debug,
            status,
            &stderr.bytes,
            stderr.truncated,
            STDERR_TAIL_LIMIT,
        ));
    }
    Ok(())
}

pub(crate) fn stream_stdout_lines(
    mut command: Command,
    line_limit: usize,
    diagnostic: BuiltinDiagnostic,
    span: &SourceSpan,
    mut visitor: impl FnMut(&[u8]) -> Result<()>,
) -> Result<()> {
    let debug = format!("{command:?}");
    command
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let mut child = spawn(command, diagnostic, span, &debug)?;
    let stdout = child_process::take_stdout(&mut child).expect("piped media-tool stdout");
    let stderr = child_process::take_stderr(&mut child).expect("piped media-tool stderr");
    let stderr_reader =
        std::thread::spawn(move || child_process::read_tail(stderr, STDERR_TAIL_LIMIT));

    let lines = for_each_bounded_line(
        BufReader::new(stdout),
        line_limit,
        diagnostic,
        span,
        &mut visitor,
    );
    if let Err(error) = lines {
        child_process::terminate(&mut child);
        let _ = join_reader(stderr_reader, "stderr", diagnostic, span, &debug);
        return Err(error.note(debug));
    }
    let status = wait(&mut child, diagnostic, span, &debug)?;
    let stderr = join_reader(stderr_reader, "stderr", diagnostic, span, &debug)?;
    if !status.success() {
        return Err(exit_diagnostic(
            diagnostic,
            span,
            &debug,
            status,
            &stderr.bytes,
            stderr.truncated,
            STDERR_TAIL_LIMIT,
        ));
    }
    Ok(())
}

fn spawn(
    command: Command,
    diagnostic: BuiltinDiagnostic,
    span: &SourceSpan,
    debug: &str,
) -> Result<Child> {
    child_process::spawn(command).map_err(|error| {
        Diagnostic::builtin(
            diagnostic,
            format!("could not start external media tool: {error}"),
            span.clone(),
        )
        .note(debug)
    })
}

fn wait(
    child: &mut Child,
    diagnostic: BuiltinDiagnostic,
    span: &SourceSpan,
    debug: &str,
) -> Result<ExitStatus> {
    child_process::wait(child).map_err(|error| {
        Diagnostic::builtin(
            diagnostic,
            format!("could not wait for external media tool: {error}"),
            span.clone(),
        )
        .note(debug)
    })
}

fn join_reader(
    reader: JoinHandle<io::Result<RetainedOutput>>,
    stream: &str,
    diagnostic: BuiltinDiagnostic,
    span: &SourceSpan,
    debug: &str,
) -> Result<RetainedOutput> {
    child_process::join_reader(reader).map_err(|error| {
        let message = match error {
            ReaderError::Panicked => format!("external media-tool {stream} reader panicked"),
            ReaderError::Io(error) => {
                format!("could not read external media-tool {stream}: {error}")
            }
        };
        Diagnostic::builtin(diagnostic, message, span.clone()).note(debug)
    })
}

fn for_each_bounded_line(
    mut reader: impl BufRead,
    limit: usize,
    diagnostic: BuiltinDiagnostic,
    span: &SourceSpan,
    visitor: &mut impl FnMut(&[u8]) -> Result<()>,
) -> Result<()> {
    let mut line = Vec::with_capacity(limit.min(256));
    let mut overflow = false;
    loop {
        let available = reader.fill_buf().map_err(|error| {
            Diagnostic::builtin(
                diagnostic,
                format!("could not read streamed media-tool output: {error}"),
                span.clone(),
            )
        })?;
        if available.is_empty() {
            if overflow {
                return Err(line_limit_diagnostic(limit, span));
            }
            if !line.is_empty() {
                visitor(&line)?;
            }
            return Ok(());
        }
        let newline = available.iter().position(|byte| *byte == b'\n');
        let segment_end = newline.unwrap_or(available.len());
        if !overflow {
            let remaining = limit.saturating_sub(line.len());
            let retained = segment_end.min(remaining);
            line.extend_from_slice(&available[..retained]);
            overflow = retained < segment_end;
        }
        let consumed = segment_end + usize::from(newline.is_some());
        reader.consume(consumed);
        if newline.is_some() {
            if overflow {
                return Err(line_limit_diagnostic(limit, span));
            }
            if line.last() == Some(&b'\r') {
                line.pop();
            }
            visitor(&line)?;
            line.clear();
        }
    }
}

fn line_limit_diagnostic(limit: usize, span: &SourceSpan) -> Diagnostic {
    Diagnostic::builtin(
        BuiltinDiagnostic::ToolOutputLimit,
        format!("external media-tool output line exceeds the {limit}-byte limit"),
        span.clone(),
    )
}

fn exit_diagnostic(
    diagnostic: BuiltinDiagnostic,
    span: &SourceSpan,
    debug: &str,
    status: ExitStatus,
    stderr: &[u8],
    truncated: bool,
    retained_limit: usize,
) -> Diagnostic {
    let stderr = String::from_utf8_lossy(stderr);
    let marker = if truncated {
        format!("[stderr truncated to final {retained_limit} bytes]\n")
    } else {
        String::new()
    };
    Diagnostic::builtin(
        diagnostic,
        format!(
            "external media tool exited with {status}\n{marker}{}",
            stderr.trim()
        ),
        span.clone(),
    )
    .note(debug)
}

fn output_limit_diagnostic(
    diagnostic: BuiltinDiagnostic,
    span: &SourceSpan,
    debug: &str,
    stream: &str,
    limit: usize,
) -> Diagnostic {
    Diagnostic::builtin(
        diagnostic,
        format!("external media-tool {stream} exceeds the {limit}-byte capture limit"),
        span.clone(),
    )
    .note(debug)
}

#[cfg(test)]
mod tests {
    use std::io::Read;

    use super::*;

    struct RepeatingReader {
        pattern: &'static [u8],
        repetitions: usize,
        offset: usize,
    }

    impl Read for RepeatingReader {
        fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
            if self.repetitions == 0 || buffer.is_empty() {
                return Ok(0);
            }
            let mut written = 0;
            while written < buffer.len() && self.repetitions > 0 {
                let remaining = &self.pattern[self.offset..];
                let count = remaining.len().min(buffer.len() - written);
                buffer[written..written + count].copy_from_slice(&remaining[..count]);
                written += count;
                self.offset += count;
                if self.offset == self.pattern.len() {
                    self.offset = 0;
                    self.repetitions -= 1;
                }
            }
            Ok(written)
        }
    }

    #[test]
    fn streamed_lines_do_not_accumulate_the_complete_output() {
        let reader = BufReader::with_capacity(
            17,
            RepeatingReader {
                pattern: b"1024\n",
                repetitions: 100_000,
                offset: 0,
            },
        );
        let mut lines = 0_u64;
        let span = SourceSpan::file_start("test");
        for_each_bounded_line(
            reader,
            16,
            BuiltinDiagnostic::ToolOutputLimit,
            &span,
            &mut |line| {
                assert_eq!(line, b"1024");
                lines += 1;
                Ok(())
            },
        )
        .expect("streamed lines");
        assert_eq!(lines, 100_000);
    }

    #[cfg(unix)]
    #[test]
    fn streamed_visitor_errors_terminate_the_child_and_remain_primary() {
        use std::time::{Duration, Instant};

        let mut command = Command::new("sh");
        command.args(["-c", "printf 'line\n'; sleep 5"]);
        let span = SourceSpan::file_start("test");
        let started = Instant::now();
        let error = stream_stdout_lines(command, 256, BuiltinDiagnostic::Ffprobe, &span, |_| {
            Err(Diagnostic::builtin(
                BuiltinDiagnostic::InvalidPlan,
                "visitor stopped",
                span.clone(),
            ))
        })
        .expect_err("visitor failure");

        assert_eq!(error.code, "E_INVALID_PLAN");
        assert_eq!(error.message, "visitor stopped");
        assert!(
            started.elapsed() < Duration::from_secs(2),
            "descendant process kept the stderr pipe open"
        );
    }

    #[test]
    fn streamed_lines_reject_one_overlong_record() {
        let reader = BufReader::new(std::io::Cursor::new(vec![b'x'; 257]));
        let span = SourceSpan::file_start("test");
        let error = for_each_bounded_line(
            reader,
            256,
            BuiltinDiagnostic::ToolOutputLimit,
            &span,
            &mut |_| Ok(()),
        )
        .expect_err("long line");
        assert_eq!(error.code, "E_TOOL_OUTPUT_LIMIT");
    }
}