use super::LOG_TARGET;
use crate::error::Error::{FilterDescUtf8, FilterNameUtf8};
use crate::error::{FilterGraphParseError, Result};
use ffmpeg_sys_next::AVMediaType::{AVMEDIA_TYPE_AUDIO, AVMEDIA_TYPE_UNKNOWN, AVMEDIA_TYPE_VIDEO};
use ffmpeg_sys_next::{
av_freep, av_opt_get, avfilter_init_dict, avfilter_pad_get_name, avfilter_pad_get_type,
AVFilterContext, AVFilterGraphSegment, AVFilterPadParams, AVMediaType,
AVFILTER_FLAG_DYNAMIC_INPUTS, AVFILTER_FLAG_DYNAMIC_OUTPUTS, AV_OPT_SEARCH_CHILDREN,
};
use log::warn;
use std::ffi::{c_char, c_void, CStr};
use std::ptr::null_mut;
pub(super) struct ProbedPad {
pub(super) linklabel: String,
pub(super) media_type: AVMediaType,
pub(super) name: String,
pub(super) node: (usize, usize),
}
pub(super) struct ProbedTopology {
pub(super) inputs: Vec<ProbedPad>,
pub(super) outputs: Vec<ProbedPad>,
pub(super) edges: Vec<NodeEdge>,
pub(super) filter_components: usize,
}
fn is_init_denied(filter_name: &[u8]) -> bool {
matches!(
filter_name,
b"movie" | b"amovie" | b"ladspa" | b"lv2" | b"libplacebo"
)
}
fn denied_pad_type(
filter_name: &str,
spec_types: Option<&[AVMediaType]>,
pad_idx: usize,
) -> AVMediaType {
if let Some(t) = spec_types.and_then(|types| types.get(pad_idx)) {
if *t != AVMEDIA_TYPE_UNKNOWN {
return *t;
}
}
match filter_name {
"movie" | "libplacebo" => AVMEDIA_TYPE_VIDEO,
"amovie" | "ladspa" | "lv2" => AVMEDIA_TYPE_AUDIO,
_ => AVMEDIA_TYPE_UNKNOWN,
}
}
fn spec_media_type(spec: &str) -> AVMediaType {
let rest = spec.strip_prefix('d').unwrap_or(spec);
match rest.chars().next() {
Some('v') => AVMEDIA_TYPE_VIDEO,
Some('a') => AVMEDIA_TYPE_AUDIO,
_ => AVMEDIA_TYPE_UNKNOWN,
}
}
unsafe fn movie_spec_types(f: *mut AVFilterContext) -> Option<Vec<AVMediaType>> {
let mut out: *mut u8 = null_mut();
let ret = av_opt_get(
f as *mut c_void,
c"streams".as_ptr(),
AV_OPT_SEARCH_CHILDREN,
&mut out,
);
if ret < 0 || out.is_null() {
return None;
}
let spec = CStr::from_ptr(out as *const c_char)
.to_str()
.ok()
.map(str::to_owned);
av_freep(&mut out as *mut *mut u8 as *mut c_void);
let spec = spec?;
if spec.is_empty() {
return None;
}
Some(spec.split('+').map(spec_media_type).collect())
}
pub(super) unsafe fn init_topology_filters(seg: *mut AVFilterGraphSegment) -> i32 {
for ci in 0..(*seg).nb_chains {
let ch = *(*seg).chains.add(ci);
for fi in 0..(*ch).nb_filters {
let p = *(*ch).filters.add(fi);
let f = (*p).filter;
if f.is_null() {
continue;
}
let flags = (*(*f).filter).flags;
if flags & (AVFILTER_FLAG_DYNAMIC_INPUTS | AVFILTER_FLAG_DYNAMIC_OUTPUTS) == 0 {
continue;
}
if is_init_denied(CStr::from_ptr((*(*f).filter).name).to_bytes()) {
continue;
}
let ret = avfilter_init_dict(f, null_mut());
if ret < 0 {
return ret;
}
}
}
0
}
pub(super) type NodeEdge = ((usize, usize), (usize, usize));
struct NodeState {
ctx: *mut AVFilterContext,
filter_name: String,
in_labels: Vec<Option<String>>,
out_labels: Vec<Option<String>>,
exact_in: bool,
exact_out: bool,
in_linked: Vec<bool>,
out_linked: Vec<bool>,
spec_types: Option<Vec<AVMediaType>>,
}
unsafe fn pad_labels(pads: *mut *mut AVFilterPadParams, nb: u32) -> Result<Vec<Option<String>>> {
let mut labels = Vec::with_capacity(nb as usize);
for i in 0..nb as usize {
let pp = *pads.add(i);
let label = (*pp).label;
if label.is_null() {
labels.push(None);
} else {
let s = CStr::from_ptr(label).to_str().map_err(|_| FilterDescUtf8)?;
labels.push(Some(s.to_string()));
}
}
Ok(labels)
}
unsafe fn build_nodes(seg: *mut AVFilterGraphSegment) -> Result<Vec<Vec<NodeState>>> {
let mut chains = Vec::with_capacity((*seg).nb_chains);
for ci in 0..(*seg).nb_chains {
let ch = *(*seg).chains.add(ci);
let mut nodes = Vec::with_capacity((*ch).nb_filters);
for fi in 0..(*ch).nb_filters {
let p = *(*ch).filters.add(fi);
let f = (*p).filter;
if f.is_null() {
nodes.push(NodeState {
ctx: null_mut(),
filter_name: String::new(),
in_labels: Vec::new(),
out_labels: Vec::new(),
exact_in: true,
exact_out: true,
in_linked: Vec::new(),
out_linked: Vec::new(),
spec_types: None,
});
continue;
}
let filter_name = CStr::from_ptr((*(*f).filter).name)
.to_str()
.map_err(|_| FilterNameUtf8)?
.to_string();
let in_labels = pad_labels((*p).inputs, (*p).nb_inputs)?;
let out_labels = pad_labels((*p).outputs, (*p).nb_outputs)?;
let flags = (*(*f).filter).flags;
let denied = is_init_denied(filter_name.as_bytes());
let real_in = (*f).nb_inputs as usize;
let real_out = (*f).nb_outputs as usize;
let (eff_in, exact_in) = if flags & AVFILTER_FLAG_DYNAMIC_INPUTS != 0 && denied {
(in_labels.len().max(1), false)
} else {
(real_in, true)
};
let spec_types = if denied && matches!(filter_name.as_str(), "movie" | "amovie") {
movie_spec_types(f)
} else {
None
};
let (eff_out, exact_out) = if flags & AVFILTER_FLAG_DYNAMIC_OUTPUTS != 0 && denied {
let assumed = spec_types.as_ref().map_or(1, |t| t.len().max(1));
(out_labels.len().max(assumed), false)
} else {
(real_out, true)
};
nodes.push(NodeState {
ctx: f,
filter_name,
in_labels,
out_labels,
exact_in,
exact_out,
in_linked: vec![false; eff_in],
out_linked: vec![false; eff_out],
spec_types,
});
}
chains.push(nodes);
}
Ok(chains)
}
fn find_unlinked_labeled(
nodes: &[Vec<NodeState>],
start_ci: usize,
start_fi: usize,
want_outputs: bool,
label: &str,
) -> Option<(usize, usize, usize)> {
for (ci, chain) in nodes.iter().enumerate().skip(start_ci) {
let fi0 = if ci == start_ci { start_fi } else { 0 };
for (fi, n) in chain.iter().enumerate().skip(fi0) {
if n.ctx.is_null() {
continue;
}
let (labels, linked) = if want_outputs {
(&n.out_labels, &n.out_linked)
} else {
(&n.in_labels, &n.in_linked)
};
for pi in 0..linked.len().min(labels.len()) {
if !linked[pi] && labels[pi].as_deref() == Some(label) {
return Some((ci, fi, pi));
}
}
}
}
None
}
unsafe fn real_pad_type(n: &NodeState, is_output: bool, pad_idx: usize) -> Option<AVMediaType> {
let f = n.ctx;
let (pads, real_nb) = if is_output {
((*f).output_pads, (*f).nb_outputs as usize)
} else {
((*f).input_pads, (*f).nb_inputs as usize)
};
(pad_idx < real_nb).then(|| avfilter_pad_get_type(pads, pad_idx as i32))
}
unsafe fn check_link_media_types(
nodes: &[Vec<NodeState>],
out: (usize, usize, usize),
inp: (usize, usize, usize),
) -> Result<()> {
let ot = real_pad_type(&nodes[out.0][out.1], true, out.2);
let it = real_pad_type(&nodes[inp.0][inp.1], false, inp.2);
if let (Some(ot), Some(it)) = (ot, it) {
if ot != it {
warn!(target: LOG_TARGET,
"Media type mismatch between the '{}' filter output pad {} and the '{}' filter input pad {}",
nodes[out.0][out.1].filter_name, out.2, nodes[inp.0][inp.1].filter_name, inp.2
);
return Err(FilterGraphParseError::InvalidArgument.into());
}
}
Ok(())
}
unsafe fn describe_pad(
n: &NodeState,
node: (usize, usize),
is_output: bool,
pad_idx: usize,
label: Option<String>,
) -> Result<ProbedPad> {
let f = n.ctx;
let (pads, real_nb) = if is_output {
((*f).output_pads, (*f).nb_outputs as usize)
} else {
((*f).input_pads, (*f).nb_inputs as usize)
};
let (media_type, name) = if pad_idx < real_nb {
let media_type = avfilter_pad_get_type(pads, pad_idx as i32);
let name = if real_nb > 1 {
n.filter_name.clone()
} else {
let pad_name = CStr::from_ptr(avfilter_pad_get_name(pads, pad_idx as i32))
.to_str()
.map_err(|_| FilterNameUtf8)?;
format!("{}:{}", n.filter_name, pad_name)
};
(media_type, name)
} else {
(
denied_pad_type(&n.filter_name, n.spec_types.as_deref(), pad_idx),
n.filter_name.clone(),
)
};
Ok(ProbedPad {
linklabel: label.unwrap_or_default(),
media_type,
name,
node,
})
}
fn link_inputs_mirror(
nodes: &mut [Vec<NodeState>],
ci: usize,
fi: usize,
open: &mut Vec<ProbedPad>,
edges: &mut Vec<NodeEdge>,
) -> Result<()> {
let (eff, exact, nb_labels) = {
let n = &nodes[ci][fi];
(n.in_linked.len(), n.exact_in, n.in_labels.len())
};
if exact && eff < nb_labels {
warn!(target: LOG_TARGET,
"More input link labels specified for filter '{}' than it has inputs: {} > {}",
nodes[ci][fi].filter_name, nb_labels, eff
);
return Err(FilterGraphParseError::InvalidArgument.into());
}
for pi in 0..eff {
if nodes[ci][fi].in_linked[pi] {
continue;
}
let label = nodes[ci][fi].in_labels.get(pi).cloned().flatten();
if let Some(lab) = label.as_deref() {
if let Some((cj, fj, pj)) = find_unlinked_labeled(nodes, ci, fi, true, lab) {
unsafe { check_link_media_types(nodes, (cj, fj, pj), (ci, fi, pi)) }?;
nodes[cj][fj].out_linked[pj] = true;
nodes[ci][fi].in_linked[pi] = true;
edges.push(((cj, fj), (ci, fi)));
continue;
}
}
open.push(unsafe { describe_pad(&nodes[ci][fi], (ci, fi), false, pi, label) }?);
}
Ok(())
}
fn link_outputs_mirror(
nodes: &mut [Vec<NodeState>],
ci: usize,
fi: usize,
open: &mut Vec<ProbedPad>,
edges: &mut Vec<NodeEdge>,
) -> Result<()> {
let (eff, exact, nb_labels) = {
let n = &nodes[ci][fi];
(n.out_linked.len(), n.exact_out, n.out_labels.len())
};
if exact && eff < nb_labels {
warn!(target: LOG_TARGET,
"More output link labels specified for filter '{}' than it has outputs: {} > {}",
nodes[ci][fi].filter_name, nb_labels, eff
);
return Err(FilterGraphParseError::InvalidArgument.into());
}
'pads: for pi in 0..eff {
if nodes[ci][fi].out_linked[pi] {
continue;
}
let label = nodes[ci][fi].out_labels.get(pi).cloned().flatten();
if let Some(lab) = label.as_deref() {
if let Some((cj, fj, pj)) = find_unlinked_labeled(nodes, ci, fi, false, lab) {
unsafe { check_link_media_types(nodes, (ci, fi, pi), (cj, fj, pj)) }?;
nodes[cj][fj].in_linked[pj] = true;
nodes[ci][fi].out_linked[pi] = true;
edges.push(((ci, fi), (cj, fj)));
continue 'pads;
}
} else {
for nfi in fi + 1..nodes[ci].len() {
if nodes[ci][nfi].ctx.is_null() {
continue;
}
let target = {
let cand = &nodes[ci][nfi];
(0..cand.in_linked.len()).find(|&pj| {
!cand.in_linked[pj] && cand.in_labels.get(pj).is_none_or(|l| l.is_none())
})
};
if let Some(pj) = target {
unsafe { check_link_media_types(nodes, (ci, fi, pi), (ci, nfi, pj)) }?;
nodes[ci][nfi].in_linked[pj] = true;
nodes[ci][fi].out_linked[pi] = true;
edges.push(((ci, fi), (ci, nfi)));
continue 'pads;
}
break;
}
}
open.push(unsafe { describe_pad(&nodes[ci][fi], (ci, fi), true, pi, label) }?);
}
Ok(())
}
pub(super) unsafe fn probe_open_pads(seg: *mut AVFilterGraphSegment) -> Result<ProbedTopology> {
let mut nodes = build_nodes(seg)?;
let mut inputs = Vec::new();
let mut outputs = Vec::new();
let mut edges = Vec::new();
for ci in 0..nodes.len() {
for fi in 0..nodes[ci].len() {
if nodes[ci][fi].ctx.is_null() {
continue;
}
link_inputs_mirror(&mut nodes, ci, fi, &mut inputs, &mut edges)?;
link_outputs_mirror(&mut nodes, ci, fi, &mut outputs, &mut edges)?;
}
}
let filter_components = count_components(&nodes, &edges);
Ok(ProbedTopology {
inputs,
outputs,
edges,
filter_components,
})
}
fn count_components(nodes: &[Vec<NodeState>], edges: &[NodeEdge]) -> usize {
let mut id = std::collections::HashMap::new();
for (ci, chain) in nodes.iter().enumerate() {
for (fi, n) in chain.iter().enumerate() {
if !n.ctx.is_null() {
let next = id.len();
id.insert((ci, fi), next);
}
}
}
let mut parent: Vec<usize> = (0..id.len()).collect();
fn find(parent: &mut [usize], mut x: usize) -> usize {
while parent[x] != x {
parent[x] = parent[parent[x]];
x = parent[x];
}
x
}
for (a, b) in edges {
let (Some(&ia), Some(&ib)) = (id.get(a), id.get(b)) else {
continue;
};
let (ra, rb) = (find(&mut parent, ia), find(&mut parent, ib));
if ra != rb {
parent[ra] = rb;
}
}
(0..parent.len())
.filter(|&i| find(&mut parent, i) == i)
.count()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::core::scheduler::filter_task::graph_opts_apply;
use ffmpeg_sys_next::{
avfilter_get_by_name, avfilter_graph_segment_create_filters, avfilter_graph_segment_free,
avfilter_graph_segment_init, avfilter_graph_segment_link, avfilter_graph_segment_parse,
AVFilterInOut,
};
use std::ffi::CString;
type PadSig = (String, AVMediaType, String);
fn sigs(pads: &[ProbedPad]) -> Vec<PadSig> {
pads.iter()
.map(|p| (p.linklabel.clone(), p.media_type, p.name.clone()))
.collect()
}
unsafe fn parse_and_create(
desc: &CString,
) -> Result<(crate::raw::FilterGraph, *mut AVFilterGraphSegment), i32> {
let graph = crate::raw::FilterGraph::alloc().expect("graph alloc");
(*graph.as_ptr()).nb_threads = 1;
let mut seg = null_mut();
let mut ret = avfilter_graph_segment_parse(graph.as_ptr(), desc.as_ptr(), 0, &mut seg);
if ret < 0 {
return Err(ret);
}
ret = avfilter_graph_segment_create_filters(seg, 0);
if ret >= 0 {
ret = graph_opts_apply(seg);
}
if ret < 0 {
avfilter_graph_segment_free(&mut seg);
return Err(ret);
}
Ok((graph, seg))
}
unsafe fn upstream_pads(desc: &str) -> Result<(Vec<PadSig>, Vec<PadSig>), i32> {
let desc = CString::new(desc).unwrap();
let (_graph, mut seg) = parse_and_create(&desc)?;
let mut ret = avfilter_graph_segment_init(seg, 0);
if ret < 0 {
avfilter_graph_segment_free(&mut seg);
return Err(ret);
}
let mut inputs = crate::raw::FilterInOut::empty();
let mut outputs = crate::raw::FilterInOut::empty();
ret = avfilter_graph_segment_link(seg, 0, inputs.as_out_ptr(), outputs.as_out_ptr());
avfilter_graph_segment_free(&mut seg);
if ret < 0 {
return Err(ret);
}
Ok((
inout_sigs(inputs.as_ptr(), false),
inout_sigs(outputs.as_ptr(), true),
))
}
unsafe fn inout_sigs(mut cur: *mut AVFilterInOut, is_output: bool) -> Vec<PadSig> {
let mut sigs = Vec::new();
while !cur.is_null() {
let label = if (*cur).name.is_null() {
String::new()
} else {
CStr::from_ptr((*cur).name).to_str().unwrap().to_string()
};
let f = (*cur).filter_ctx;
let (pads, nb) = if is_output {
((*f).output_pads, (*f).nb_outputs)
} else {
((*f).input_pads, (*f).nb_inputs)
};
let media_type = avfilter_pad_get_type(pads, (*cur).pad_idx);
let fname = CStr::from_ptr((*(*f).filter).name).to_str().unwrap();
let name = if nb > 1 {
fname.to_string()
} else {
let pad = CStr::from_ptr(avfilter_pad_get_name(pads, (*cur).pad_idx))
.to_str()
.unwrap();
format!("{fname}:{pad}")
};
sigs.push((label, media_type, name));
cur = (*cur).next;
}
sigs
}
unsafe fn probe_pads(desc: &str) -> std::result::Result<ProbedTopology, String> {
let desc = CString::new(desc).unwrap();
let (_graph, mut seg) = parse_and_create(&desc).map_err(|e| format!("pre: {e}"))?;
let ret = init_topology_filters(seg);
if ret < 0 {
avfilter_graph_segment_free(&mut seg);
return Err(format!("init_topology_filters: {ret}"));
}
let topo = probe_open_pads(seg);
avfilter_graph_segment_free(&mut seg);
topo.map_err(|e| format!("probe: {e}"))
}
#[track_caller]
fn assert_parity(desc: &str) {
crate::core::initialize_ffmpeg();
unsafe {
let upstream = upstream_pads(desc).unwrap_or_else(|e| {
panic!("upstream path failed for {desc:?} (averror {e}); fix the test graph")
});
let probed = probe_pads(desc)
.unwrap_or_else(|e| panic!("probe failed for {desc:?} but upstream links: {e}"));
assert_eq!(
sigs(&probed.inputs),
upstream.0,
"open INPUT pads diverge from graphparser for {desc:?}"
);
assert_eq!(
sigs(&probed.outputs),
upstream.1,
"open OUTPUT pads diverge from graphparser for {desc:?}"
);
}
}
#[track_caller]
fn assert_both_reject(desc: &str) {
crate::core::initialize_ffmpeg();
unsafe {
assert!(
upstream_pads(desc).is_err(),
"upstream unexpectedly links {desc:?}"
);
assert!(
probe_pads(desc).is_err(),
"probe accepted {desc:?} which upstream rejects at parse/link"
);
}
}
#[test]
fn parity_single_filter_open_both_ends() {
assert_parity("scale=320:240");
}
#[test]
fn parity_implicit_chain() {
assert_parity("scale=320:240,hflip");
}
#[test]
fn parity_labeled_passthrough() {
assert_parity("[in]yadif[out]");
}
#[test]
fn parity_split_multichain_labels() {
assert_parity("[0:v]split[a][b];[a]hflip[x];[x][b]overlay[out]");
}
#[test]
fn parity_surplus_split_outputs_stay_open() {
assert_parity("split=3,hflip");
}
#[test]
fn parity_amix_dynamic_inputs() {
assert_parity("[0:a][1:a]amix=inputs=2[mixed]");
}
#[test]
fn parity_concat_dynamic_both_directions() {
assert_parity("[0:v][0:a][1:v][1:a]concat=n=2:v=1:a=1[v][a]");
}
#[test]
fn parity_forward_cross_chain_label() {
assert_parity("anull[l];[l]anull");
}
#[test]
fn parity_sws_flags_prefix() {
assert_parity("sws_flags=bilinear;[0:v]scale=100:100[out]");
}
#[test]
fn parity_channelsplit_dynamic_outputs() {
assert_parity("channelsplit=channel_layout=stereo");
}
#[test]
fn parity_partially_labeled_dynamic_outputs() {
assert_parity("split[a],hflip;[a]vflip[out]");
}
#[test]
fn arity_too_many_input_labels_rejected_by_both() {
assert_both_reject("[0:v][1:v]hflip[out]");
}
#[test]
fn arity_too_many_output_labels_rejected_by_both() {
assert_both_reject("[x]hflip[a][b]");
}
#[test]
fn cross_media_labeled_link_rejected_by_both() {
assert_both_reject("[0:a]anull[x];[x]hflip[out]");
}
#[test]
fn cross_media_implicit_link_rejected_by_both() {
assert_both_reject("anull,hflip");
}
#[test]
fn movie_probe_succeeds_without_touching_the_missing_file() {
crate::core::initialize_ffmpeg();
let desc = "movie=/nonexistent/ez_probe_fixture.mkv[wm];[base][wm]overlay[out]";
unsafe {
assert!(upstream_pads(desc).is_err(), "fixture unexpectedly exists");
let topo = probe_pads(desc).expect("probe must not need the movie file");
assert_eq!(
sigs(&topo.inputs),
vec![("base".into(), AVMEDIA_TYPE_VIDEO, "overlay".into())]
);
assert_eq!(
sigs(&topo.outputs),
vec![("out".into(), AVMEDIA_TYPE_VIDEO, "overlay:default".into())]
);
}
}
#[test]
fn movie_open_pad_types_follow_the_streams_spec() {
crate::core::initialize_ffmpeg();
let desc = "movie=/nonexistent/ez_probe_fixture.mkv:s=dv+da[v][a]";
unsafe {
let topo = probe_pads(desc).expect("probe must not need the movie file");
assert!(topo.inputs.is_empty());
assert_eq!(
sigs(&topo.outputs),
vec![
("v".into(), AVMEDIA_TYPE_VIDEO, "movie".into()),
("a".into(), AVMEDIA_TYPE_AUDIO, "movie".into()),
]
);
}
}
#[test]
fn amovie_open_pad_defaults_to_audio() {
crate::core::initialize_ffmpeg();
let desc = "amovie=/nonexistent/ez_probe_fixture.wav[m]";
unsafe {
let topo = probe_pads(desc).expect("probe must not need the amovie file");
assert_eq!(
sigs(&topo.outputs),
vec![("m".into(), AVMEDIA_TYPE_AUDIO, "amovie".into())]
);
}
}
#[test]
fn lv2_chain_head_assumes_exactly_one_input_pad() {
crate::core::initialize_ffmpeg();
unsafe {
let name = CString::new("lv2").unwrap();
if avfilter_get_by_name(name.as_ptr()).is_null() {
return; }
let topo = probe_pads("lv2=plugin=x[out]")
.expect("probe must not load the (bogus) LV2 plugin");
assert_eq!(topo.inputs.len(), 1, "one assumed input pad, no more");
assert_eq!(topo.inputs[0].name, "lv2");
assert_eq!(topo.inputs[0].media_type, AVMEDIA_TYPE_AUDIO);
assert_eq!(
sigs(&topo.outputs),
vec![("out".into(), AVMEDIA_TYPE_AUDIO, "lv2".into())]
);
}
}
}