use super::{BackendSnapshot, MediaSource, VideoError, VideoState};
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use std::collections::VecDeque;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use windows::Win32::Media::MediaFoundation::*;
use windows::Win32::System::Com::StructuredStorage::PROPVARIANT;
use windows::Win32::System::Com::{COINIT_APARTMENTTHREADED, CoInitializeEx, CoUninitialize};
use windows::core::{GUID, HSTRING};
const HNS_PER_SECOND: f64 = 10_000_000.0;
const VIDEO_END_TOLERANCE_HNS: i64 = 800_000;
struct DecodedVideoFrame {
timestamp_hns: i64,
rgba: Vec<u8>,
}
struct WindowsMediaRuntime {
com_initialized: bool,
media_foundation_started: bool,
}
impl WindowsMediaRuntime {
fn new() -> Result<Self, VideoError> {
unsafe {
CoInitializeEx(None, COINIT_APARTMENTTHREADED)
.ok()
.map_err(|error| backend_error("failed to initialize COM", error))?;
if let Err(error) = MFStartup(MF_VERSION, MFSTARTUP_FULL) {
CoUninitialize();
return Err(backend_error("failed to start Media Foundation", error));
}
}
Ok(Self {
com_initialized: true,
media_foundation_started: true,
})
}
}
impl Drop for WindowsMediaRuntime {
fn drop(&mut self) {
unsafe {
if self.media_foundation_started {
let _ = MFShutdown();
}
if self.com_initialized {
CoUninitialize();
}
}
}
}
struct AudioOutput {
stream: cpal::Stream,
queue: Arc<Mutex<VecDeque<f32>>>,
volume: Arc<AtomicU32>,
muted: Arc<AtomicBool>,
playback_rate: Arc<AtomicU64>,
sample_rate: u32,
channels: usize,
}
pub(crate) struct WindowsVideoBackend {
reader: Option<IMFSourceReader>,
runtime: Option<WindowsMediaRuntime>,
audio: Option<AudioOutput>,
width: u32,
height: u32,
stride: usize,
bottom_up: bool,
duration: Option<Duration>,
base_position_hns: i64,
started_at: Option<Instant>,
pending_frame: Option<DecodedVideoFrame>,
has_presented_frame: bool,
video_ended: bool,
audio_ended: bool,
playing: bool,
looping: bool,
muted: bool,
volume: f32,
playback_rate: f64,
}
impl WindowsVideoBackend {
pub(crate) fn open(source: &MediaSource) -> Result<Self, VideoError> {
let runtime = WindowsMediaRuntime::new()?;
let source = resolve_source(source)?;
let reader = unsafe {
let mut attributes = None;
MFCreateAttributes(&mut attributes, 2)
.map_err(|error| backend_error("failed to create reader attributes", error))?;
let attributes = attributes.ok_or_else(|| {
VideoError::Backend("Media Foundation returned no reader attributes".to_string())
})?;
attributes
.SetUINT32(&MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, 1)
.map_err(|error| backend_error("failed to enable video processing", error))?;
attributes
.SetUINT32(&MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1)
.map_err(|error| backend_error("failed to enable hardware decoding", error))?;
MFCreateSourceReaderFromURL(&HSTRING::from(source), &attributes)
.map_err(|error| backend_error("failed to open Windows video source", error))?
};
unsafe {
reader
.SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS.0 as u32, false)
.map_err(|error| backend_error("failed to reset media streams", error))?;
reader
.SetStreamSelection(MF_SOURCE_READER_FIRST_VIDEO_STREAM.0 as u32, true)
.map_err(|error| {
backend_error("video source has no decodable video stream", error)
})?;
let video_type = MFCreateMediaType()
.map_err(|error| backend_error("failed to create video media type", error))?;
video_type
.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)
.and_then(|_| video_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_RGB32))
.map_err(|error| backend_error("failed to configure RGB video output", error))?;
reader
.SetCurrentMediaType(
MF_SOURCE_READER_FIRST_VIDEO_STREAM.0 as u32,
None,
&video_type,
)
.map_err(|error| backend_error("failed to enable Windows video decoder", error))?;
}
let (width, height, stride, bottom_up) = read_video_layout(&reader)?;
let duration = read_duration(&reader);
let audio = create_audio_output(&reader)?;
let mut backend = Self {
reader: Some(reader),
runtime: Some(runtime),
audio,
width,
height,
stride,
bottom_up,
duration,
base_position_hns: 0,
started_at: None,
pending_frame: None,
has_presented_frame: false,
video_ended: false,
audio_ended: false,
playing: false,
looping: false,
muted: false,
volume: 1.0,
playback_rate: 1.0,
};
backend.pending_frame = backend.decode_video_frame()?;
backend.fill_audio_queue()?;
Ok(backend)
}
pub(crate) fn close(&mut self) {
if let Some(audio) = self.audio.take() {
let _ = audio.stream.pause();
}
self.reader.take();
self.runtime.take();
self.playing = false;
}
pub(crate) fn update(&mut self) -> Result<BackendSnapshot, VideoError> {
let position_hns = self.current_position_hns();
let mut rgba = None;
if self.pending_frame.is_none() && !self.video_ended {
self.pending_frame = self.decode_video_frame()?;
}
while let Some(frame) = self.pending_frame.take() {
let should_present = !self.has_presented_frame || frame.timestamp_hns <= position_hns;
if !should_present {
self.pending_frame = Some(frame);
break;
}
rgba = Some(frame.rgba);
self.has_presented_frame = true;
if self.video_ended {
break;
}
self.pending_frame = self.decode_video_frame()?;
}
self.fill_audio_queue()?;
let duration_hns = self
.duration
.map(|duration| duration.as_secs_f64() * HNS_PER_SECOND);
let ended = self.video_ended
&& self.pending_frame.is_none()
&& duration_hns
.map(|duration| position_hns as f64 >= duration - VIDEO_END_TOLERANCE_HNS as f64)
.unwrap_or(true);
if ended && self.looping {
self.seek(Duration::ZERO)?;
self.play()?;
} else if ended {
self.freeze_clock();
self.playing = false;
if let Some(audio) = &self.audio {
let _ = audio.stream.pause();
}
}
let state = if ended && !self.looping {
VideoState::Ended
} else if self.playing {
VideoState::Playing
} else if position_hns > 0 {
VideoState::Paused
} else {
VideoState::Ready
};
Ok(BackendSnapshot {
state,
duration: self.duration,
position: Duration::from_secs_f64(position_hns.max(0) as f64 / HNS_PER_SECOND),
dimensions: Some((self.width, self.height)),
rgba,
#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
external_video: None,
})
}
pub(crate) fn play(&mut self) -> Result<(), VideoError> {
if self.playing {
return Ok(());
}
self.started_at = Some(Instant::now());
self.playing = true;
if let Some(audio) = &self.audio {
audio
.stream
.play()
.map_err(|error| backend_error("failed to start Windows video audio", error))?;
}
Ok(())
}
pub(crate) fn pause(&mut self) -> Result<(), VideoError> {
self.freeze_clock();
self.playing = false;
if let Some(audio) = &self.audio {
audio
.stream
.pause()
.map_err(|error| backend_error("failed to pause Windows video audio", error))?;
}
Ok(())
}
pub(crate) fn stop(&mut self) -> Result<(), VideoError> {
self.pause()?;
self.seek(Duration::ZERO)
}
pub(crate) fn seek(&mut self, position: Duration) -> Result<(), VideoError> {
let position_hns =
(position.as_secs_f64() * HNS_PER_SECOND).clamp(0.0, i64::MAX as f64) as i64;
let value = PROPVARIANT::from(position_hns);
unsafe {
self.reader()?
.Flush(MF_SOURCE_READER_ALL_STREAMS.0 as u32)
.map_err(|error| backend_error("failed to flush Windows video decoder", error))?;
self.reader()?
.SetCurrentPosition(&GUID::zeroed(), &value)
.map_err(|error| backend_error("failed to seek Windows video", error))?;
}
if let Some(audio) = &self.audio
&& let Ok(mut queue) = audio.queue.lock()
{
queue.clear();
}
self.base_position_hns = position_hns;
self.started_at = self.playing.then(Instant::now);
self.pending_frame = None;
self.has_presented_frame = false;
self.video_ended = false;
self.audio_ended = false;
Ok(())
}
pub(crate) fn set_loop(&mut self, looping: bool) -> Result<(), VideoError> {
self.looping = looping;
Ok(())
}
pub(crate) fn set_muted(&mut self, muted: bool) -> Result<(), VideoError> {
self.muted = muted;
if let Some(audio) = &self.audio {
audio.muted.store(muted, Ordering::Relaxed);
}
Ok(())
}
pub(crate) fn set_volume(&mut self, volume: f32) -> Result<(), VideoError> {
self.volume = volume.clamp(0.0, 1.0);
if let Some(audio) = &self.audio {
audio.volume.store(self.volume.to_bits(), Ordering::Relaxed);
}
Ok(())
}
pub(crate) fn set_playback_rate(&mut self, rate: f64) -> Result<(), VideoError> {
let position = self.current_position_hns();
self.base_position_hns = position;
self.started_at = self.playing.then(Instant::now);
self.playback_rate = rate;
if let Some(audio) = &self.audio {
audio.playback_rate.store(rate.to_bits(), Ordering::Relaxed);
}
Ok(())
}
fn reader(&self) -> Result<&IMFSourceReader, VideoError> {
self.reader
.as_ref()
.ok_or(VideoError::InvalidState("Windows video player is closed"))
}
fn current_position_hns(&self) -> i64 {
let elapsed = self
.started_at
.filter(|_| self.playing)
.map(|started| started.elapsed().as_secs_f64())
.unwrap_or(0.0);
self.base_position_hns.saturating_add(
(elapsed * self.playback_rate * HNS_PER_SECOND).clamp(0.0, i64::MAX as f64) as i64,
)
}
fn freeze_clock(&mut self) {
self.base_position_hns = self.current_position_hns();
self.started_at = None;
}
fn decode_video_frame(&mut self) -> Result<Option<DecodedVideoFrame>, VideoError> {
for _ in 0..32 {
let mut flags = 0;
let mut timestamp_hns = 0;
let mut sample = None;
unsafe {
self.reader()?.ReadSample(
MF_SOURCE_READER_FIRST_VIDEO_STREAM.0 as u32,
0,
None,
Some(&mut flags),
Some(&mut timestamp_hns),
Some(&mut sample),
)
}
.map_err(|error| backend_error("failed to decode Windows video frame", error))?;
if flags & MF_SOURCE_READERF_CURRENTMEDIATYPECHANGED.0 as u32 != 0 {
let (width, height, stride, bottom_up) = read_video_layout(self.reader()?)?;
self.width = width;
self.height = height;
self.stride = stride;
self.bottom_up = bottom_up;
}
if flags & MF_SOURCE_READERF_ENDOFSTREAM.0 as u32 != 0 {
self.video_ended = true;
}
let Some(sample) = sample else {
if self.video_ended {
return Ok(None);
}
continue;
};
let rgba = copy_video_sample(
&sample,
self.width,
self.height,
self.stride,
self.bottom_up,
)?;
return Ok(Some(DecodedVideoFrame {
timestamp_hns,
rgba,
}));
}
Ok(None)
}
fn fill_audio_queue(&mut self) -> Result<(), VideoError> {
let Some(audio) = &self.audio else {
return Ok(());
};
if self.audio_ended {
return Ok(());
}
let target_samples = audio.sample_rate as usize * audio.channels;
let queued = audio.queue.lock().map(|queue| queue.len()).unwrap_or(0);
if queued >= target_samples {
return Ok(());
}
for _ in 0..64 {
let mut flags = 0;
let mut sample = None;
unsafe {
self.reader()?.ReadSample(
MF_SOURCE_READER_FIRST_AUDIO_STREAM.0 as u32,
0,
None,
Some(&mut flags),
None,
Some(&mut sample),
)
}
.map_err(|error| backend_error("failed to decode Windows video audio", error))?;
if flags & MF_SOURCE_READERF_ENDOFSTREAM.0 as u32 != 0 {
self.audio_ended = true;
}
if let Some(sample) = sample {
append_audio_sample(&sample, &audio.queue)?;
}
let queued = audio.queue.lock().map(|queue| queue.len()).unwrap_or(0);
if queued >= target_samples || self.audio_ended {
break;
}
}
Ok(())
}
}
fn read_video_layout(reader: &IMFSourceReader) -> Result<(u32, u32, usize, bool), VideoError> {
let media_type = unsafe {
reader
.GetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM.0 as u32)
.map_err(|error| backend_error("failed to read Windows video format", error))?
};
let size = unsafe { media_type.GetUINT64(&MF_MT_FRAME_SIZE) }
.map_err(|error| backend_error("Windows video has no frame size", error))?;
let width = (size >> 32) as u32;
let height = size as u32;
if width == 0 || height == 0 {
return Err(VideoError::Backend(
"Windows video reported an empty frame size".to_string(),
));
}
let raw_stride = unsafe { media_type.GetUINT32(&MF_MT_DEFAULT_STRIDE) }
.unwrap_or(width.saturating_mul(4)) as i32;
let bottom_up = raw_stride < 0;
let stride = raw_stride.unsigned_abs().max(width.saturating_mul(4)) as usize;
Ok((width, height, stride, bottom_up))
}
fn read_duration(reader: &IMFSourceReader) -> Option<Duration> {
let value = unsafe {
reader
.GetPresentationAttribute(MF_SOURCE_READER_MEDIASOURCE.0 as u32, &MF_PD_DURATION)
.ok()?
};
let duration_hns = u64::try_from(&value)
.ok()
.or_else(|| i64::try_from(&value).ok().map(|value| value.max(0) as u64))?;
Some(Duration::from_secs_f64(
duration_hns as f64 / HNS_PER_SECOND,
))
}
fn copy_video_sample(
sample: &IMFSample,
width: u32,
height: u32,
configured_stride: usize,
bottom_up: bool,
) -> Result<Vec<u8>, VideoError> {
let buffer = unsafe { sample.ConvertToContiguousBuffer() }
.map_err(|error| backend_error("failed to join Windows video buffers", error))?;
let mut data = std::ptr::null_mut();
let mut length = 0;
unsafe { buffer.Lock(&mut data, None, Some(&mut length)) }
.map_err(|error| backend_error("failed to lock Windows video frame", error))?;
let result = (|| {
if data.is_null() {
return Err(VideoError::Backend(
"Windows video decoder returned an empty frame".to_string(),
));
}
let bytes = unsafe { std::slice::from_raw_parts(data, length as usize) };
let minimum_stride = width as usize * 4;
let stride = configured_stride
.max(minimum_stride)
.min(bytes.len().checked_div(height as usize).unwrap_or(0));
if stride < minimum_stride || bytes.len() < stride * height as usize {
return Err(VideoError::Backend(
"Windows video frame has an unsupported memory layout".to_string(),
));
}
let mut rgba = vec![0; width as usize * height as usize * 4];
for row in 0..height as usize {
let source_row = if bottom_up {
height as usize - 1 - row
} else {
row
};
let source = &bytes[source_row * stride..source_row * stride + minimum_stride];
let target = &mut rgba[row * minimum_stride..(row + 1) * minimum_stride];
for (bgra, rgba) in source.chunks_exact(4).zip(target.chunks_exact_mut(4)) {
rgba.copy_from_slice(&[bgra[2], bgra[1], bgra[0], 255]);
}
}
Ok(rgba)
})();
let _ = unsafe { buffer.Unlock() };
result
}
fn create_audio_output(reader: &IMFSourceReader) -> Result<Option<AudioOutput>, VideoError> {
let host = cpal::default_host();
let Some(device) = host.default_output_device() else {
return Ok(None);
};
let config = device
.default_output_config()
.map_err(|error| backend_error("failed to query Windows audio output", error))?;
let sample_format = config.sample_format();
let sample_rate = config.sample_rate();
let channels = config.channels() as usize;
let audio_type = unsafe { MFCreateMediaType() }
.map_err(|error| backend_error("failed to create audio media type", error))?;
let block_alignment = channels as u32 * 4;
let configure_result = unsafe {
reader
.SetStreamSelection(MF_SOURCE_READER_FIRST_AUDIO_STREAM.0 as u32, true)
.and_then(|_| audio_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Audio))
.and_then(|_| audio_type.SetGUID(&MF_MT_SUBTYPE, &MFAudioFormat_Float))
.and_then(|_| audio_type.SetUINT32(&MF_MT_AUDIO_NUM_CHANNELS, channels as u32))
.and_then(|_| audio_type.SetUINT32(&MF_MT_AUDIO_SAMPLES_PER_SECOND, sample_rate))
.and_then(|_| audio_type.SetUINT32(&MF_MT_AUDIO_BITS_PER_SAMPLE, 32))
.and_then(|_| audio_type.SetUINT32(&MF_MT_AUDIO_BLOCK_ALIGNMENT, block_alignment))
.and_then(|_| {
audio_type.SetUINT32(
&MF_MT_AUDIO_AVG_BYTES_PER_SECOND,
sample_rate.saturating_mul(block_alignment),
)
})
.and_then(|_| audio_type.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1))
.and_then(|_| {
reader.SetCurrentMediaType(
MF_SOURCE_READER_FIRST_AUDIO_STREAM.0 as u32,
None,
&audio_type,
)
})
};
if configure_result.is_err() {
return Ok(None);
}
let queue = Arc::new(Mutex::new(VecDeque::<f32>::new()));
let volume = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
let muted = Arc::new(AtomicBool::new(false));
let playback_rate = Arc::new(AtomicU64::new(1.0_f64.to_bits()));
let stream_config: cpal::StreamConfig = config.into();
let stream = match sample_format {
cpal::SampleFormat::F32 => build_audio_stream::<f32>(
&device,
&stream_config,
Arc::clone(&queue),
Arc::clone(&volume),
Arc::clone(&muted),
Arc::clone(&playback_rate),
)?,
cpal::SampleFormat::I16 => build_audio_stream::<i16>(
&device,
&stream_config,
Arc::clone(&queue),
Arc::clone(&volume),
Arc::clone(&muted),
Arc::clone(&playback_rate),
)?,
cpal::SampleFormat::U16 => build_audio_stream::<u16>(
&device,
&stream_config,
Arc::clone(&queue),
Arc::clone(&volume),
Arc::clone(&muted),
Arc::clone(&playback_rate),
)?,
_ => return Ok(None),
};
Ok(Some(AudioOutput {
stream,
queue,
volume,
muted,
playback_rate,
sample_rate,
channels,
}))
}
fn build_audio_stream<T>(
device: &cpal::Device,
config: &cpal::StreamConfig,
queue: Arc<Mutex<VecDeque<f32>>>,
volume: Arc<AtomicU32>,
muted: Arc<AtomicBool>,
playback_rate: Arc<AtomicU64>,
) -> Result<cpal::Stream, VideoError>
where
T: cpal::SizedSample + cpal::FromSample<f32>,
{
let channels = config.channels as usize;
let mut phase = 1.0_f64;
let mut current_frame = vec![0.0_f32; channels];
device
.build_output_stream(
config,
move |output: &mut [T], _| {
let gain = if muted.load(Ordering::Relaxed) {
0.0
} else {
f32::from_bits(volume.load(Ordering::Relaxed))
};
let rate = f64::from_bits(playback_rate.load(Ordering::Relaxed)).max(0.01);
let Ok(mut queue) = queue.try_lock() else {
for sample in output {
*sample = T::from_sample(0.0);
}
return;
};
for output_frame in output.chunks_mut(channels) {
while phase >= 1.0 {
for sample in &mut current_frame {
*sample = queue.pop_front().unwrap_or(0.0);
}
phase -= 1.0;
}
for (output, input) in output_frame.iter_mut().zip(¤t_frame) {
*output = T::from_sample((*input * gain).clamp(-1.0, 1.0));
}
phase += rate;
}
},
|error| eprintln!("[spot][video] Windows audio stream error: {error}"),
None,
)
.map_err(|error| backend_error("failed to create Windows video audio stream", error))
}
fn append_audio_sample(
sample: &IMFSample,
queue: &Arc<Mutex<VecDeque<f32>>>,
) -> Result<(), VideoError> {
let buffer = unsafe { sample.ConvertToContiguousBuffer() }
.map_err(|error| backend_error("failed to join Windows audio buffers", error))?;
let mut data = std::ptr::null_mut();
let mut length = 0;
unsafe { buffer.Lock(&mut data, None, Some(&mut length)) }
.map_err(|error| backend_error("failed to lock Windows audio sample", error))?;
if !data.is_null() {
let bytes = unsafe { std::slice::from_raw_parts(data, length as usize) };
if let Ok(mut queue) = queue.lock() {
queue.extend(
bytes
.chunks_exact(4)
.map(|sample| f32::from_le_bytes(sample.try_into().unwrap())),
);
}
}
let _ = unsafe { buffer.Unlock() };
Ok(())
}
fn resolve_source(source: &MediaSource) -> Result<String, VideoError> {
match source {
MediaSource::Url(url) => Ok(url.clone()),
MediaSource::File(path) => resolve_file(path),
MediaSource::Asset(path) => {
let asset_path = Path::new(path.strip_prefix("./").unwrap_or(path));
let executable_path = std::env::current_exe()
.ok()
.and_then(|executable| executable.parent().map(|parent| parent.join(asset_path)));
if let Some(path) = executable_path.filter(|path| path.exists()) {
resolve_file(&path)
} else {
resolve_file(asset_path)
}
}
}
}
fn resolve_file(path: &Path) -> Result<String, VideoError> {
let path: PathBuf = path.canonicalize().map_err(|error| {
VideoError::Backend(format!(
"failed to resolve Windows video path '{}': {error}",
path.display()
))
})?;
Ok(path.to_string_lossy().into_owned())
}
fn backend_error(context: &str, error: impl std::fmt::Display) -> VideoError {
VideoError::Backend(format!("{context}: {error}"))
}