dash_mpd/
stpp.rs

1//! Support for the STPP subtitle format
2//
3// This module provides support for TTML (Timed Text Markup Language) subtitles that are encoded
4// using the STPP codec and packed in fragmented MP4 segments. These subtitles are provided as a
5// separate media stream of fMP4 segments, that the media player retrieves incrementally.
6//
7// This module implements:
8//
9//  - extracting the XML-formatted TTML fragments from an MP4 fragment
10//
11//  - parsing the TTML fragment to extract <style>, <region> and <p> elements, appending them to the
12//  StppDocument object
13//
14//  - serializing to a single merged TTML subtitle file
15//
16// We only support the text-only IMSC1 profile for TTML subtitles ("stpp.ttml.im1t"); the image-only
17// profile ("stpp.ttml.im1i") is not supported.
18//
19// An example of the XML content in a TTML/STPP fragment:
20//
21// <?xml version="1.0" encoding="utf-8"?>
22// <tt xmlns="http://www.w3.org/ns/ttml" xmlns:ttm="http://www.w3.org/ns/ttml#metadata"
23//     xmlns:tts="http://www.w3.org/ns/ttml#styling" xml:lang="fr">
24//   <head>
25//     <metadata><ttm:title></ttm:title><ttm:desc></ttm:desc><ttm:copyright></ttm:copyright></metadata>
26//     <styling>
27//       <style xml:id="basic" tts:backgroundColor="transparent" tts:color="white"
28//              tts:fontFamily="proportionalSansSerif" tts:fontSize="16px" tts:textAlign="center" />
29//     </styling>
30//     <layout>
31//       <region style="basic" xml:id="speaker" tts:displayAlign="center" tts:extent="80% 10%"
32//              tts:origin="10% 85%" />
33//     </layout>
34//   </head>
35//   <body>
36//     <div xml:lang="fr">
37//       <p begin="00:03:44.500" end="00:03:45.700" region="speaker">Rien d&apos;inquiétant.</p>
38//       <p begin="00:03:46.000" end="00:03:47.000" region="speaker">Thom.</p>
39//       <p begin="00:03:51.000" end="00:03:52.000" region="speaker">Elle est là.</p>
40//     </div>
41//   </body>
42// </tt>
43//
44// Here an example fMP4 segment that contains TTML data
45//  https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel-ttml.ism/dash/tears-of-steel-ttml-textstream_fra=1000-52.m4s
46//
47// References:
48//   https://en.wikipedia.org/wiki/Timed_Text_Markup_Language
49//   https://www.w3.org/TR/ttml-imsc1.0.1/
50
51
52use std::io::Cursor;
53use xot::{Xot, output};
54use xot::xmlname::NameStrInfo;
55use xmlparser::{ElementEnd, Token, Tokenizer};
56use tracing::{trace, warn, error};
57use bytes::Bytes;
58use crate::DashMpdError;
59
60
61#[derive(Clone, Debug)]
62pub struct StppDocument {
63    xot: Xot,
64    styles: Vec<xot::Node>,
65    regions: Vec<xot::Node>,
66    // A paragraph is a single subtitle cue.
67    paragraphs: Vec<xot::Node>,
68    warned_binary_contents: bool,
69}
70
71impl Default for StppDocument {
72    fn default() -> Self {
73        Self::new()
74    }
75}
76
77// We need to extract and new styles and layouts and append them to the head of the final XML file,
78// and also append all the <p> tags for the final XML.
79impl StppDocument {
80    #[must_use]
81    pub fn new() -> StppDocument {
82        StppDocument {
83            xot: Xot::new(),
84            styles: Vec::new(),
85            regions: Vec::new(),
86            paragraphs: Vec::new(),
87            warned_binary_contents: false,
88        }
89    }
90
91
92    // Extract XML content from a fragmented MP4 segment (argument bytes) and add its contents to
93    // the content accumulated in the parent StppDocument. The content is typically in an mdat box,
94    // or sometimes an stpp box.
95    //
96    // Decode boxes present in an fMP4 segment: https://media-analyzer.pro/analyzer
97    pub fn add_from_mp4(&mut self, bytes: &Bytes) -> Result<(), DashMpdError> {
98        use mp4_atom::ReadFrom;
99
100        let mut buf = Cursor::new(bytes);
101        loop {
102            match Option::<mp4_atom::Any>::read_from(&mut buf) {
103                Ok(maybe_atom) => {
104                    match maybe_atom {
105                        // Some(mp4_atom::Any::Stpp(_stpp)) => (),
106                        Some(mp4_atom::Any::Mdat(mdat)) => if let Ok(xml) = str::from_utf8(&mdat.data) {
107                            self.add_content(xml)?;
108                        },
109                        Some(_) => (),
110                        None => break,
111                    }
112                },
113                Err(e) => warn!("Malformed MP4 box: {e:?}"),
114            }
115        }
116        Ok(())
117    }
118
119    // Extract XML content from the binary data in bytes.
120    pub fn add_bytes(&mut self, bytes: &Bytes) -> Result<(), DashMpdError> {
121        if let Ok(xml) = str::from_utf8(bytes) {
122            self.add_content(xml)?;
123        } else {
124            // This could be using the image-only profile, which we can't handle. Make sure we only
125            // display this warning a single time for each subtitle stream.
126            if !self.warned_binary_contents {
127                warn!("Ignoring invalid XML in STPP subs: {}", String::from_utf8_lossy(bytes));
128                self.warned_binary_contents = true;
129            }
130        }
131        Ok(())
132    }
133
134    fn find_child_named(&mut self, node: xot::Node, name: &str) -> Option<xot::Node> {
135        self.xot.children(node)
136            .find(|n| self.xot.node_name_ref(*n)
137                  .is_ok_and(
138                      |nn| nn.is_some_and(
139                          |nnn| nnn.local_name().eq(name))))
140    }
141
142
143    // Parse the TTML XML in xml and add its contents to the content accumulated in the parent
144    // StppDocument. TTML fragments will often contain redundant style and region elements to allow
145    // a media player to jump to a random point in the subtitle stream without requiring it to load
146    // all the previous subtitle segments. We filter these out.
147    pub fn add_content(&mut self, xml: &str) -> Result<(), DashMpdError> {
148        trace!("adding STPP content {xml}");
149        let mut clean_xml = xml;
150        let epos = identify_xml_endpos(xml)
151            .map_err(|_| DashMpdError::Parsing(String::from("calculating XML endpos")))?;
152        if epos < xml.len() {
153            clean_xml = &clean_xml[0..epos];
154        }
155        let root = self.xot.parse(clean_xml)
156            .map_err(|e| {
157                error!("Failure parsing STPP XML: {e:?}");
158                error!("Failing XML: {xml}");
159                DashMpdError::Parsing(String::from("parsing STPP XML"))
160            })?;
161        let tt = self.xot.document_element(root)
162            .map_err(|_| DashMpdError::Parsing(String::from("extracting STPP XML root")))?;
163        if !self.xot.element(tt).is_some_and(
164            |n| self.xot.name_ns_str(n.name()).0.eq("tt")) {
165            warn!("Missing tt root element in STPP XML: {xml}");
166            return Ok(());
167        }
168        let xml_ns = self.xot.add_namespace("http://www.w3.org/XML/1998/namespace");
169        let id_name = self.xot.add_name_ns("id", xml_ns);
170        if let Some(head) = self.find_child_named(tt, "head") {
171            for d in self.xot.descendants(head) {
172                if self.xot.element(d).is_some_and(|n| self.xot.name_ns_str(n.name()).0.eq("style")) {
173                    // Only add the style if it's not already defined (filter based on xml:id attribute)
174                    if let Some(new_id) = self.xot.attributes(d).get(id_name) {
175                        if !self.styles.iter().any(|s| self.xot.attributes(*s)
176                                                   .get(id_name)
177                                                   .is_some_and(|id| id.eq(new_id))) {
178                            self.styles.push(d);
179                        }
180                    } else {
181                        // don't attempt to dedup if there is no @id
182                        self.styles.push(d);
183                    }
184                }
185            }
186            for d in self.xot.descendants(head) {
187                if self.xot.element(d).is_some_and(|n| self.xot.name_ns_str(n.name()).0.eq("region")) {
188                    if let Some(new_id) = self.xot.attributes(d).get(id_name) {
189                        if !self.regions.iter().any(|s| self.xot.attributes(*s)
190                                                    .get(id_name).is_some_and(|id| id.eq(new_id))) {
191                            self.regions.push(d);
192                        }
193                    } else {
194                        // don't attempt to dedup if there is no @id
195                        self.regions.push(d);
196                    }
197                }
198            }
199        }
200        if let Some(body) = self.find_child_named(tt, "body") {
201            // Add all children of the body: these might be <div> nodes or <p> nodes.
202            for d in self.xot.children(body) {
203                self.paragraphs.push(d);
204            }
205        }
206        Ok(())
207    }
208
209    // Generate a complete TTML document corresponding to the merge of all the fragments seen so
210    // far. Note that we can't implement this using the fmt::Display trait for StppDocument, because
211    // we need a mutable reference to self, which is not available for Display.
212    #[allow(clippy::inherent_to_string)]
213    pub fn to_string(&mut self) -> String {
214        let empty_xml = r#"<tt xmlns="http://www.w3.org/ns/ttml" xmlns:ttm="http://www.w3.org/ns/ttml#metadata" xmlns:tts="http://www.w3.org/ns/ttml#styling" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="fr"></tt>"#;
215        let ttml_ns = self.xot.add_namespace("http://www.w3.org/ns/ttml");
216        let root = self.xot.parse(empty_xml).unwrap();
217        let tt = self.xot.document_element(root).unwrap();
218        let head_name = self.xot.add_name_ns("head", ttml_ns);
219        let head = self.xot.new_element(head_name);
220        self.xot.create_missing_prefixes(head).unwrap();
221        let _ = self.xot.append(tt, head);
222        let body_name = self.xot.add_name_ns("body", ttml_ns);
223        let body = self.xot.new_element(body_name);
224        self.xot.create_missing_prefixes(body).unwrap();
225        let _ = self.xot.append(tt, body);
226        let div_name = self.xot.add_name_ns("div", ttml_ns);
227        let div = self.xot.new_element(div_name);
228        let _ = self.xot.append(body, div);
229        let styling_name = self.xot.add_name_ns("styling", ttml_ns);
230        let styling = self.xot.new_element(styling_name);
231        self.xot.create_missing_prefixes(styling).unwrap();
232        let _ = self.xot.append(head, styling);
233        for s in &self.styles {
234            let new = self.xot.clone_with_prefixes(*s);
235            let _ = self.xot.append(styling, new);
236        }
237        let layout_name = self.xot.add_name_ns("layout", ttml_ns);
238        let layout = self.xot.new_element(layout_name);
239        self.xot.create_missing_prefixes(layout).unwrap();
240        let _ = self.xot.append(head,layout);
241        for r in &self.regions {
242            let new = self.xot.clone_with_prefixes(*r);
243            let _ = self.xot.append(layout, new);
244        }
245        for p in &self.paragraphs {
246            let new = self.xot.clone_with_prefixes(*p);
247            let _ = self.xot.append(div, new);
248        }
249        self.xot.create_missing_prefixes(tt).unwrap();
250        self.xot.deduplicate_namespaces(tt);
251        self.xot.serialize_xml_string(output::xml::Parameters {
252            declaration: Some(output::xml::Declaration {
253                encoding: Some("UTF-8".to_string()),
254                ..Default::default()
255            }),
256            ..Default::default()
257        }, tt).unwrap()
258    }
259}
260
261
262// Argument str contains a well-formed XML document potentially followed by trailing content. Return
263// the string position corresponding to the end of XML content.
264fn identify_xml_endpos(input: &str) -> Result<usize, xmlparser::Error> {
265    let mut depth = 0;
266    let mut xml_end;
267    for token in Tokenizer::from(input) {
268        let token = token?;
269        xml_end = token.span().end();
270        match token {
271            Token::ElementStart { .. } => depth += 1,
272            Token::ElementEnd { end: ElementEnd::Close(..), .. } => {
273                depth -= 1;
274                if depth == 0 {
275                    // We've closed the root element.
276                    return Ok(xml_end);
277                }
278            },
279            Token::ElementEnd { end: ElementEnd::Empty, .. } => {
280                depth -= 1;
281                if depth == 0 {
282                    // Root element was <root/>.
283                    return Ok(xml_end);
284                }
285            },
286            _ => {}
287        }
288    }
289    Err(xmlparser::Error::UnknownToken(
290        xmlparser::TextPos::new(1, 1),
291    ))
292}
293
dash_mpd/stpp.rs

dash_mpd/
stpp.rs
