ebustl-parser 0.7.0

//! Validation of the rules in Tech 3264 that relate several fields or blocks.

use std::collections::{BTreeSet, HashMap};
use std::fmt;

use crate::text_field::TF_UNUSED_SPACE;
use crate::{DisplayStandardCode, InvalidFieldValue, Stl, Subtitle, TextElement, Time};

/// A deviation from the spec, found by [`Stl::validate`].
///
/// # Example
///
/// ```rust
/// let stl = ebustl_parser::parse_stl_from_file("stls/test.stl").unwrap();
/// for issue in stl.validate() {
///     println!("{issue}");
/// }
/// ```
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ValidationIssue {
    tti_index: Option<usize>,
    field: &'static str,
    msg: String,
}

impl ValidationIssue {
    fn gsi(field: &'static str, msg: String) -> Self {
        Self {
            tti_index: None,
            field,
            msg,
        }
    }

    fn tti(tti_index: usize, field: &'static str, msg: String) -> Self {
        Self {
            tti_index: Some(tti_index),
            field,
            msg,
        }
    }

    /// The index in [`Stl::ttis`] of the TTI block with the issue, or `None` for the GSI block.
    pub fn tti_index(&self) -> Option<usize> {
        self.tti_index
    }

    /// Mnemonic of the field in the spec, e.g. "TNB".
    pub fn field(&self) -> &'static str {
        self.field
    }

    /// What deviates from the spec.
    pub fn msg(&self) -> &str {
        &self.msg
    }
}

impl fmt::Display for ValidationIssue {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self.tti_index {
            None => write!(f, "GSI block, {}: {}", self.field, self.msg),
            Some(index) => write!(f, "TTI block {index}, {}: {}", self.field, self.msg),
        }
    }
}

impl Stl {
    /// Checks the subtitle list against the rules in the spec, and returns the deviations
    /// found. An empty list means that no deviation was found.
    ///
    /// The fields are checked one at a time as by the validating accessors, e.g.
    /// [`GsiBlock::creation_date`](crate::GsiBlock::creation_date), and together:
    ///
    /// - TND is 1-9 and DSN 1 up to TND.
    /// - TNB, TNS and TNG match the TTI blocks, when the subtitle list is on a single disk.
    /// - TCF is the TCI of the first subtitle.
    /// - Subtitle Numbers are continuous and ascending, and TCI ascending within a subtitle group.
    /// - Extension blocks have ascending EBN, the last block of a subtitle has EBN FFh and its
    ///   Text Field ends with unused space (8Fh). EBN F0h-FDh are reserved.
    /// - TCI, TCO, JC and CF of the first block of each subtitle are valid, and TCO is not
    ///   before TCI.
    /// - VP is 1-23 for teletext and at most MNR for open subtitles.
    /// - The Text Field only has control codes for the Display Standard Code, and no reserved
    ///   codes.
    ///
    /// Time codes are expected to increase with time, as the spec requires ascending TCI
    /// (section 4.3.1). A subtitle list that passes midnight, where time codes start over from
    /// 00:00:00:00, or that has cuts in the time codes is therefore reported. Comment subtitles
    /// (CF 01h) are not included in the time code order and TCF.
    ///
    /// # Example
    ///
    /// ```rust
    /// use ebustl_parser::{Stl, Time, TtiFormat};
    ///
    /// let time = |seconds| Time { hours: 10, minutes: 0, seconds, frames: 0 };
    /// let format = |vp| TtiFormat { jc: 2, vp, dh: false };
    /// let mut stl = Stl::new();
    /// stl.add_sub(time(1), time(3), "Valid", format(20)).unwrap();
    /// assert!(stl.validate().is_empty());
    ///
    /// // Teletext has rows 1-23
    /// stl.add_sub(time(4), time(6), "Below the screen", format(30)).unwrap();
    /// let issues = stl.validate();
    /// assert_eq!(1, issues.len());
    /// assert_eq!("VP", issues[0].field());
    /// assert_eq!(Some(1), issues[0].tti_index());
    /// ```
    pub fn validate(&self) -> Vec<ValidationIssue> {
        let mut issues = Vec::new();
        self.check_gsi_fields(&mut issues);
        self.check_disks(&mut issues);
        self.check_totals(&mut issues);
        self.check_first_in_cue(&mut issues);
        self.check_subtitles(&mut issues);
        issues
    }

    fn check_gsi_fields(&self, issues: &mut Vec<ValidationIssue>) {
        let gsi = &self.gsi;
        push_error(issues, None, gsi.language());
        push_error(issues, None, gsi.creation_date());
        push_error(issues, None, gsi.revision_date());
        push_error(issues, None, gsi.revision_number());
        push_error(issues, None, gsi.time_code_start_of_programme());
        push_error(issues, None, gsi.time_code_first_in_cue());
        push_error(issues, None, gsi.country_of_origin());
    }

    fn check_disks(&self, issues: &mut Vec<ValidationIssue>) {
        let (tnd, dsn) = (self.gsi.tnd, self.gsi.dsn);
        if !(1..=9).contains(&tnd) {
            issues.push(ValidationIssue::gsi("TND", format!("{tnd} is not 1-9")));
        }
        if dsn == 0 || dsn > tnd {
            issues.push(ValidationIssue::gsi("DSN", format!("{dsn} is not 1-{tnd}")));
        }
    }

    /// TNB, TNS and TNG count the whole subtitle list, which may span several disks.
    fn check_totals(&self, issues: &mut Vec<ValidationIssue>) {
        if self.gsi.tnd != 1 {
            return;
        }
        let blocks = self.ttis.len();
        if usize::try_from(self.gsi.tnb) != Ok(blocks) {
            let msg = format!("{} but there are {blocks} TTI blocks", self.gsi.tnb);
            issues.push(ValidationIssue::gsi("TNB", msg));
        }
        let subtitles: Vec<Subtitle> = self.subtitles().filter(has_text).collect();
        if usize::try_from(self.gsi.tns) != Ok(subtitles.len()) {
            let msg = format!(
                "{} but there are {} subtitles",
                self.gsi.tns,
                subtitles.len()
            );
            issues.push(ValidationIssue::gsi("TNS", msg));
        }
        let groups: BTreeSet<u8> = subtitles
            .iter()
            .map(|subtitle| subtitle.header_block().sgn)
            .collect();
        if usize::from(self.gsi.tng) != groups.len() {
            let msg = format!(
                "{} but there are {} subtitle groups",
                self.gsi.tng,
                groups.len()
            );
            issues.push(ValidationIssue::gsi("TNG", msg));
        }
    }

    fn check_first_in_cue(&self, issues: &mut Vec<ValidationIssue>) {
        let Ok(Some(tcf)) = self.gsi.time_code_first_in_cue() else {
            return;
        };
        let first = self
            .subtitles()
            .find(|subtitle| has_text(subtitle) && !subtitle.is_comment());
        if let Some(first) = first {
            let tci = first.header_block().tci;
            if tci != tcf {
                let msg = format!(
                    "{} is not the TCI {} of the first subtitle",
                    tcf.hhmmssff(),
                    tci.hhmmssff()
                );
                issues.push(ValidationIssue::gsi("TCF", msg));
            }
        }
    }

    fn check_subtitles(&self, issues: &mut Vec<ValidationIssue>) {
        let mut first_index = 0;
        let mut previous_sn = None;
        // The latest Time Code In of each subtitle group
        let mut group_tci: HashMap<u8, Time> = HashMap::new();
        for subtitle in self.subtitles() {
            let sn = subtitle.subtitle_number();
            if let Some(previous) = previous_sn
                && u32::from(sn) != u32::from(previous) + 1
            {
                let msg = format!("{sn} does not follow {previous}");
                issues.push(ValidationIssue::tti(first_index, "SN", msg));
            }
            previous_sn = Some(sn);

            self.check_blocks(&subtitle, first_index, issues);
            if has_text(&subtitle) {
                let header_index = first_index + header_offset(&subtitle);
                self.check_header(&subtitle, header_index, issues);
                self.check_text_field(&subtitle, header_index, issues);
                if !subtitle.is_comment() {
                    check_tci_order(&subtitle, header_index, &mut group_tci, issues);
                }
            }
            first_index += subtitle.blocks().len();
        }
    }

    /// Extension Block Numbers and the terminating unused space of a subtitle.
    fn check_blocks(
        &self,
        subtitle: &Subtitle,
        first_index: usize,
        issues: &mut Vec<ValidationIssue>,
    ) {
        let mut previous_extension = None;
        let mut last_text_block = None;
        for (index, block) in (first_index..).zip(subtitle.blocks()) {
            let ebn = block.ebn;
            if (0xF0..=0xFD).contains(&ebn) {
                let msg = format!("{ebn:02X}h is reserved for future use");
                issues.push(ValidationIssue::tti(index, "EBN", msg));
            } else if !block.is_text_block() {
                continue;
            } else if !block.is_last_block() {
                if previous_extension.is_some_and(|previous| ebn <= previous) {
                    let msg = format!("{ebn:02X}h is not in ascending order");
                    issues.push(ValidationIssue::tti(index, "EBN", msg));
                }
                previous_extension = Some(ebn);
            }
            if block.is_text_block() {
                last_text_block = Some((index, block));
            }
        }
        let Some((index, block)) = last_text_block else {
            return;
        };
        if !block.is_last_block() {
            let msg = format!("{:02X}h, the last block of a subtitle has FFh", block.ebn);
            issues.push(ValidationIssue::tti(index, "EBN", msg));
        }
        if block.tf.last() != Some(&TF_UNUSED_SPACE) {
            let msg = "the last block of a subtitle does not end with unused space (8Fh)";
            issues.push(ValidationIssue::tti(index, "TF", msg.to_string()));
        }
    }

    /// The fields in bytes 4-15, which only the first block of a subtitle carries.
    fn check_header(&self, subtitle: &Subtitle, index: usize, issues: &mut Vec<ValidationIssue>) {
        let header = subtitle.header_block();
        let dfc = &self.gsi.dfc;
        push_error(issues, Some(index), header.justification());
        push_error(issues, Some(index), header.comment_flag());
        let tci = header.time_code_in(dfc);
        let tco = header.time_code_out(dfc);
        if let (Ok(tci), Ok(tco)) = (&tci, &tco)
            && tco < tci
        {
            let msg = format!("{} is before TCI {}", tco.hhmmssff(), tci.hhmmssff());
            issues.push(ValidationIssue::tti(index, "TCO", msg));
        }
        push_error(issues, Some(index), tci);
        push_error(issues, Some(index), tco);

        let vp = header.vp;
        match self.gsi.dsc {
            DisplayStandardCode::Level1Teletext | DisplayStandardCode::Level2Teletext
                if !(1..=23).contains(&vp) =>
            {
                let msg = format!("{vp} is not a teletext row 1-23");
                issues.push(ValidationIssue::tti(index, "VP", msg));
            }
            DisplayStandardCode::OpenSubtitling if u16::from(vp) > self.gsi.mnr => {
                let msg = format!("{vp} is above MNR {}", self.gsi.mnr);
                issues.push(ValidationIssue::tti(index, "VP", msg));
            }
            _ => {}
        }
    }

    /// Control codes that do not belong to the Display Standard Code, and reserved codes.
    fn check_text_field(
        &self,
        subtitle: &Subtitle,
        index: usize,
        issues: &mut Vec<ValidationIssue>,
    ) {
        let elements = subtitle.text_elements();
        let dsc = &self.gsi.dsc;
        let open = *dsc == DisplayStandardCode::OpenSubtitling;
        let teletext = matches!(
            dsc,
            DisplayStandardCode::Level1Teletext | DisplayStandardCode::Level2Teletext
        );
        if open
            && elements
                .iter()
                .any(|e| matches!(e, TextElement::Teletext(_)))
        {
            let msg = "teletext control codes in an open subtitle".to_string();
            issues.push(ValidationIssue::tti(index, "TF", msg));
        }
        if teletext && elements.iter().any(|e| matches!(e, TextElement::Open(_))) {
            let msg = "open subtitle control codes in a teletext subtitle".to_string();
            issues.push(ValidationIssue::tti(index, "TF", msg));
        }
        for element in &elements {
            if let TextElement::Reserved(code) = element {
                let msg = format!("{code:02X}h is reserved for future use");
                issues.push(ValidationIssue::tti(index, "TF", msg));
            }
        }
    }
}

/// Time Code In must not go back within a subtitle group (section 4.3.1).
fn check_tci_order(
    subtitle: &Subtitle,
    index: usize,
    group_tci: &mut HashMap<u8, Time>,
    issues: &mut Vec<ValidationIssue>,
) {
    let header = subtitle.header_block();
    if let Some(previous) = group_tci.get(&header.sgn)
        && header.tci < *previous
    {
        let msg = format!(
            "{} is before {} of the previous subtitle in the group",
            header.tci.hhmmssff(),
            previous.hhmmssff()
        );
        issues.push(ValidationIssue::tti(index, "TCI", msg));
    }
    group_tci.insert(header.sgn, header.tci);
}

fn push_error<T, V>(
    issues: &mut Vec<ValidationIssue>,
    tti_index: Option<usize>,
    result: Result<T, InvalidFieldValue<V>>,
) {
    if let Err(err) = result {
        issues.push(ValidationIssue {
            tti_index,
            field: err.field(),
            msg: err.msg().to_string(),
        });
    }
}

/// Returns `true` unless the subtitle only has user data or reserved blocks.
fn has_text(subtitle: &Subtitle) -> bool {
    subtitle.text_blocks().next().is_some()
}

/// Position of the header block within the blocks of the subtitle.
fn header_offset(subtitle: &Subtitle) -> usize {
    let header = subtitle.header_block();
    subtitle
        .blocks()
        .iter()
        .position(|block| std::ptr::eq(block, header))
        .unwrap_or(0)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::{CumulativeStatus, TtiBlock, TtiFormat};

    fn time(minutes: u8, seconds: u8) -> Time {
        Time {
            hours: 10,
            minutes,
            seconds,
            frames: 0,
        }
    }

    /// A valid teletext subtitle list with `count` subtitles, two seconds each
    fn stl(count: u8) -> Stl {
        let mut stl = Stl::new();
        for i in 0..count {
            let format = TtiFormat {
                jc: 2,
                vp: 20,
                dh: true,
            };
            let text = format!("Subtitle {i}\nSecond row");
            stl.add_sub(time(0, 2 * i), time(0, 2 * i + 1), &text, format)
                .unwrap();
        }
        stl
    }

    /// The (TTI index, field) of each issue
    fn issues(stl: &Stl) -> Vec<(Option<usize>, &'static str)> {
        stl.validate()
            .iter()
            .map(|issue| (issue.tti_index(), issue.field()))
            .collect()
    }

    fn user_data_block(sn: u16) -> TtiBlock {
        let mut block = TtiBlock::new(
            sn,
            time(0, 0),
            time(0, 0),
            "user data",
            TtiFormat {
                jc: 0,
                vp: 0,
                dh: false,
            },
            crate::CharacterCodeTable::Latin,
        );
        block.ebn = 0xFE;
        block
    }

    #[test]
    fn written_subtitle_list_is_valid() {
        let mut stl = stl(3);
        let long_text = "A subtitle long enough to need extension blocks. ".repeat(5);
        let format = TtiFormat {
            jc: 1,
            vp: 1,
            dh: false,
        };
        stl.add_sub(time(1, 0), time(1, 5), &long_text, format)
            .unwrap();

        assert!(stl.ttis.len() > 4, "extension blocks expected");
        assert_eq!(Vec::<ValidationIssue>::new(), stl.validate());
    }

    #[test]
    fn test_file_is_valid() {
        let stl = crate::parse_stl_from_file("stls/test.stl").unwrap();
        assert_eq!(Vec::<ValidationIssue>::new(), stl.validate());
    }

    #[test]
    fn invalid_gsi_field() {
        let mut stl = stl(1);
        stl.gsi.cd = "911341".to_string();
        let issues = stl.validate();
        assert_eq!(1, issues.len());
        assert_eq!(
            "GSI block, CD: \"911341\" is not an existing date",
            issues[0].to_string()
        );
    }

    #[test]
    fn disk_numbers() {
        let mut stl = stl(1);
        stl.gsi.tnd = 0;
        stl.gsi.dsn = 0;
        assert_eq!(vec![(None, "TND"), (None, "DSN")], issues(&stl));

        stl.gsi.tnd = 2;
        stl.gsi.dsn = 3;
        assert_eq!(vec![(None, "DSN")], issues(&stl));
    }

    #[test]
    fn totals() {
        let mut stl = stl(2);
        stl.gsi.tnb = 3;
        stl.gsi.tns = 1;
        stl.gsi.tng = 2;
        assert_eq!(
            vec![(None, "TNB"), (None, "TNS"), (None, "TNG")],
            issues(&stl)
        );
    }

    #[test]
    fn totals_are_not_checked_on_several_disks() {
        let mut stl = stl(2);
        stl.gsi.tnb = 1;
        stl.gsi.tnd = 2;
        assert!(stl.validate().is_empty());
    }

    #[test]
    fn totals_do_not_count_user_data_only_subtitles() {
        let mut stl = stl(1);
        stl.ttis.push(user_data_block(2));
        stl.gsi.tnb = 2;
        assert!(stl.validate().is_empty());
    }

    #[test]
    fn subtitle_groups_are_counted() {
        let mut stl = stl(2);
        stl.ttis[1].sgn = 1;
        assert_eq!(vec![(None, "TNG")], issues(&stl));
        stl.gsi.tng = 2;
        assert!(stl.validate().is_empty());
    }

    #[test]
    fn first_in_cue() {
        let mut stl = stl(2);
        stl.gsi.tcf = "10000100".to_string();
        let issues = stl.validate();
        assert_eq!(
            "GSI block, TCF: 10:00:01:00 is not the TCI 10:00:00:00 of the first subtitle",
            issues[0].to_string()
        );
    }

    #[test]
    fn first_in_cue_skips_comments() {
        let mut stl = stl(2);
        stl.ttis[0].cf = 1;
        stl.gsi.tcf = "10000200".to_string();
        assert!(stl.validate().is_empty());
    }

    #[test]
    fn subtitle_numbers_are_continuous() {
        let mut stl = stl(3);
        stl.ttis[2].sn = 5;
        assert_eq!(vec![(Some(2), "SN")], issues(&stl));
    }

    #[test]
    fn time_code_in_is_ascending_within_group() {
        let mut stl = stl(3);
        stl.ttis[2].tci = time(0, 1);
        stl.ttis[2].tco = time(0, 2);
        assert_eq!(vec![(Some(2), "TCI")], issues(&stl));

        // Another subtitle group may start over
        stl.ttis[2].sgn = 1;
        stl.gsi.tng = 2;
        assert!(stl.validate().is_empty());
    }

    #[test]
    fn time_codes_passing_midnight_are_reported() {
        let mut stl = stl(2);
        let late = |seconds| Time {
            hours: 23,
            minutes: 59,
            seconds,
            frames: 0,
        };
        let early = |seconds| Time {
            hours: 0,
            minutes: 0,
            seconds,
            frames: 0,
        };
        stl.ttis[0].tci = late(58);
        stl.ttis[0].tco = early(1);
        stl.ttis[1].tci = early(2);
        stl.ttis[1].tco = early(3);
        stl.gsi.tcf = "23595800".to_string();
        assert_eq!(vec![(Some(0), "TCO"), (Some(1), "TCI")], issues(&stl));
    }

    #[test]
    fn equal_time_codes_are_allowed() {
        let mut stl = stl(2);
        stl.ttis[0].tco = stl.ttis[0].tci;
        stl.ttis[1].tci = stl.ttis[0].tci;
        stl.ttis[1].tco = stl.ttis[0].tci;
        assert!(stl.validate().is_empty());
    }

    #[test]
    fn time_code_out_before_in() {
        let mut stl = stl(1);
        stl.ttis[0].tco = Time {
            hours: 9,
            ..time(0, 0)
        };
        let issues = stl.validate();
        assert_eq!(
            "TTI block 0, TCO: 09:00:00:00 is before TCI 10:00:00:00",
            issues[0].to_string()
        );
    }

    #[test]
    fn invalid_header_fields() {
        let mut stl = stl(1);
        stl.ttis[0].jc = 4;
        stl.ttis[0].cf = 2;
        stl.ttis[0].tci.frames = 25;
        // TCF no longer matches the changed TCI
        assert_eq!(
            vec![
                (None, "TCF"),
                (Some(0), "JC"),
                (Some(0), "CF"),
                (Some(0), "TCI")
            ],
            issues(&stl)
        );
    }

    #[test]
    fn only_the_first_block_carries_header_fields() {
        let mut stl = Stl::new();
        let format = TtiFormat {
            jc: 2,
            vp: 20,
            dh: false,
        };
        stl.add_sub(time(0, 0), time(0, 1), &"x".repeat(200), format)
            .unwrap();
        // Values in extension blocks have no meaning
        stl.ttis[1].jc = 0x20;
        stl.ttis[1].tci.frames = 99;
        stl.ttis[1].cs = CumulativeStatus::LastInSet;
        assert!(stl.validate().is_empty());
    }

    #[test]
    fn extension_block_numbers() {
        let mut stl = Stl::new();
        let format = TtiFormat {
            jc: 2,
            vp: 20,
            dh: false,
        };
        stl.add_sub(time(0, 0), time(0, 1), &"x".repeat(300), format)
            .unwrap();
        assert_eq!(
            vec![0x00, 0x01, 0xFF],
            stl.ttis.iter().map(|b| b.ebn).collect::<Vec<_>>()
        );

        stl.ttis[1].ebn = 0x00;
        assert_eq!(vec![(Some(1), "EBN")], issues(&stl));

        stl.ttis[1].ebn = 0xF0;
        assert_eq!(vec![(Some(1), "EBN")], issues(&stl));
    }

    #[test]
    fn subtitle_without_last_block() {
        let mut stl = stl(1);
        stl.ttis[0].ebn = 0x00;
        assert_eq!(vec![(Some(0), "EBN")], issues(&stl));
    }

    #[test]
    fn last_block_must_end_with_unused_space() {
        let mut stl = stl(1);
        stl.ttis[0].tf = vec![b'x'; crate::text_field::TF_LENGTH];
        assert_eq!(vec![(Some(0), "TF")], issues(&stl));
    }

    #[test]
    fn vertical_position_of_teletext() {
        let mut stl = stl(2);
        stl.ttis[0].vp = 0;
        stl.ttis[1].vp = 24;
        assert_eq!(vec![(Some(0), "VP"), (Some(1), "VP")], issues(&stl));
    }

    #[test]
    fn vertical_position_of_open_subtitles() {
        let mut stl = open_stl();
        stl.gsi.mnr = 10;
        stl.ttis[0].vp = 10;
        assert!(stl.validate().is_empty());
        stl.ttis[0].vp = 11;
        assert_eq!(vec![(Some(0), "VP")], issues(&stl));
    }

    /// A valid open subtitle list with one subtitle
    fn open_stl() -> Stl {
        let mut stl = Stl::new();
        stl.gsi.dsc = DisplayStandardCode::OpenSubtitling;
        let format = TtiFormat {
            jc: 2,
            vp: 5,
            dh: false,
        };
        stl.add_sub(time(0, 0), time(0, 1), "Open", format).unwrap();
        stl
    }

    #[test]
    fn control_codes_must_match_display_standard() {
        let mut stl = open_stl();
        assert!(stl.validate().is_empty());
        // Double height (0Dh) in an open subtitle
        stl.ttis[0].tf[0] = 0x0D;
        assert_eq!(vec![(Some(0), "TF")], issues(&stl));

        let mut stl = stl_with_first_code(0x80);
        assert_eq!(vec![(Some(0), "TF")], issues(&stl));
        stl.gsi.dsc = DisplayStandardCode::Blank;
        assert!(stl.validate().is_empty());
    }

    /// A teletext subtitle list with `code` as first byte of the Text Field
    fn stl_with_first_code(code: u8) -> Stl {
        let mut stl = stl(1);
        stl.ttis[0].tf[0] = code;
        stl
    }

    #[test]
    fn reserved_text_field_codes() {
        let stl = stl_with_first_code(0x90);
        let issues = stl.validate();
        assert_eq!(
            "TTI block 0, TF: 90h is reserved for future use",
            issues[0].to_string()
        );
    }
}