ot_tools_io/

arrangements.rs

/*
SPDX-License-Identifier: GPL-3.0-or-later
Copyright © 2024 Mike Robeson [dijksterhuis]
*/

//! Types and ser/de for `arr??.*` binary data files.
//!
//! Proper checksum calculations are not yet implemented. But this doesn't seem
//! to have an impact on loading arrangements onto the Octatrack.

mod deserialize;
mod serialize;

use crate::{
    Defaults, HasChecksumField, HasFileVersionField, HasHeaderField, IsDefault, OctatrackFileIO,
    OtToolsIoError,
};
use ot_tools_io_derive::{ArrayDefaults, BoxedBigArrayDefaults, IntegrityChecks};
use serde::Serialize;
use serde_big_array::Array;
use std::array::from_fn;

/// Arrangement file header data
/// ```text
/// ASCII: FORM....DPS1ARRA........
/// Hex: 46 4f 52 4d 00 00 00 00 44 50 53 31 41 52 52 41 00 00 00 00 00 06
/// U8: [70, 79, 82, 77, 0, 0, 0, 0, 68, 80, 83, 49, 65, 82, 82, 65, 0, 0, 0, 0, 0, 6]
/// ```
pub const ARRANGEMENT_FILE_HEADER: [u8; 21] = [
    70, 79, 82, 77, 0, 0, 0, 0, 68, 80, 83, 49, 65, 82, 82, 65, 0, 0, 0, 0, 0,
];

/// Current/supported version of arrangements files.
pub const ARRANGEMENT_FILE_VERSION: u8 = 6;

/// `"OT_TOOLS_ARR` -- this is a custom name specifically created for ot-tools.
/// The octatrack will normally copy the name of a previously created arrangement
/// when creating arrangements on project creation. Not sure why, but it means
/// arrangements never have a single default name.
const ARRANGEMENT_DEFAULT_NAME: [u8; 15] =
    [79, 67, 84, 65, 84, 79, 79, 76, 83, 45, 65, 82, 82, 32, 32];

// max length: 11336 bytes
/// Base model for `arr??.*` arrangement binary data files.
#[derive(Debug, Clone, Serialize, PartialEq, IntegrityChecks)]
pub struct ArrangementFile {
    /// Header data
    pub header: [u8; 21],

    /// Patch version of this data type. Is inspected on project load to determine if
    /// the octatrack is able to load a project from current OS, or whether the project
    /// files need to be patched and updated.
    pub datatype_version: u8,

    /// Dunno. Example data:
    /// ```text
    /// [0, 0]
    /// ```
    pub unk1: [u8; 2],

    /// Current arrangement data in active use.
    /// This block is written when saving via Project Menu -> SYNC TO CARD.
    ///
    /// The second block is written when saving the arrangement via Arranger Menu -> SAVE.
    pub arrangement_state_current: ArrangementBlock,

    /// Dunno.
    pub unk2: u8,

    /// Whether the arrangement has been saved within the arrangement menu.
    pub saved_flag: u8,

    /// Arrangement data from the previous saved state.
    /// This block is written when saving the arrangement via Arranger Menu -> SAVE.
    pub arrangement_state_previous: ArrangementBlock,

    /// The current save/unsaved state of **all** loaded arrangements.
    /// 'Saved' means there's been at least one ARRANGER MENU save operation performed,
    /// and [`ArrangementBlock`] data has been written to the `arrangement_state_previous` field.
    ///
    /// Example data:
    /// ```text
    /// Arrangement 1 has been saved: [1, 0, 0, 0, 0, 0, 0, 0]
    /// Arrangement 2 has been saved: [0, 1, 0, 0, 0, 0, 0, 0]
    /// Arrangement 2, 7 & 8 have been saved: [0, 1, 0, 0, 0, 0, 1, 1]
    /// ```
    pub arrangements_saved_state: [u8; 8],

    /// Checksum for the file.
    pub checksum: u16,
}

impl OctatrackFileIO for ArrangementFile {
    fn encode(&self) -> Result<Vec<u8>, OtToolsIoError> {
        // TODO: oh god it looks like i might need to swap byte order on everything,
        //       possibly including bank data swapping bytes is one required when
        //       running on little-endian systems
        let mut swapped = self.clone();
        if cfg!(target_endian = "little") {
            swapped.checksum = self.checksum.swap_bytes();
        }
        let bytes = bincode::serialize(&swapped)?;
        Ok(bytes)
    }
}

impl HasChecksumField for ArrangementFile {
    fn calculate_checksum(&self) -> Result<u16, OtToolsIoError> {
        Ok(0)
    }
    fn check_checksum(&self) -> Result<bool, OtToolsIoError> {
        Ok(self.checksum == self.calculate_checksum()?)
    }
}

impl Default for ArrangementFile {
    fn default() -> Self {
        // TODO: Clean up once checksums are dealt with
        // let init = Self {
        // let init = Self {
        //     header: ARRANGEMENT_FILE_HEADER,
        //     unk1: from_fn(|_| 0),
        //     arrangement_state_current: ArrangementBlock::default(),
        //     // TODO
        //     unk2: 0,
        //     // TODO
        //     saved_flag: 1,
        //     arrangement_state_previous: ArrangementBlock::default(),
        //     // WARN: by default this actually always 0, but generating test data where these things
        //     // are 'inactive' is basically impossible!
        //     // for now, I'm setting this to default a 1-values array, but in reality it depends...
        //     // cba to type this out fully. gonna bite me later i know it. basically i need to check
        //     // if arrangement chaining affects this flag or if the arrangement has been saved.
        //     arrangements_saved_state: from_fn(|_| 1),
        //     checksum: 1973,
        // };

        // let bytes = bincode::serialize(&init).unwrap();
        // let checksum = get_checksum(&bytes);
        // init.checksum = checksum.unwrap();
        // init

        Self {
            header: ARRANGEMENT_FILE_HEADER,
            datatype_version: ARRANGEMENT_FILE_VERSION,
            unk1: from_fn(|_| 0),
            arrangement_state_current: ArrangementBlock::default(),
            // TODO
            unk2: 0,
            // TODO
            saved_flag: 0,
            arrangement_state_previous: ArrangementBlock::default(),
            // WARN: by default this actually always 0, but generating test data where these things
            // are 'inactive' is basically impossible!
            // for now, I'm setting this to default a 1-values array, but in reality it depends...
            // cba to type this out fully. gonna bite me later i know it. basically i need to check
            // if arrangement chaining affects this flag or if the arrangement has been saved.
            arrangements_saved_state: from_fn(|_| 1),
            checksum: 1973,
        }
    }
}

impl HasHeaderField for ArrangementFile {
    fn check_header(&self) -> Result<bool, OtToolsIoError> {
        Ok(self.header == ARRANGEMENT_FILE_HEADER)
    }
}

impl HasFileVersionField for ArrangementFile {
    fn check_file_version(&self) -> Result<bool, OtToolsIoError> {
        Ok(self.datatype_version == ARRANGEMENT_FILE_VERSION)
    }
}

impl IsDefault for ArrangementFile {
    fn is_default(&self) -> bool {
        let default = &ArrangementFile::default();
        // check everything except the arrangement name fields (see
        // ArrangementBlock's IsDefault implementation for more details)
        self.arrangement_state_current.is_default()
            && self.arrangement_state_previous.is_default()
            && default.unk1 == self.unk1
            && default.unk2 == self.unk2
    }
}

/// Base model for an arrangement 'block' within an arrangement binary data file.
/// There are two arrangement 'blocks' in each arrangement file -- enabling the
/// arrangement 'reload ' functionality.
#[derive(Debug, Eq, PartialEq, Clone)]
pub struct ArrangementBlock {
    /// Name of the Arrangement in ASCII values, max length 15 characters
    pub name: [u8; 15], // String,

    /// Unknown data. No idea what this is. Usually `[0, 0]` or `[0, 1]`
    pub unknown_1: [u8; 2],

    /// Number of active rows in the arrangement. Any parsed row data after this number of rows
    /// should be an `ArrangeRow::EmptyRow` variant.
    ///
    /// #### WARNING
    /// Max number of `ArrangeRows` (256) is a zero value here! Zero rows are also possible, so do
    /// not use this field as a guaranteed row count.
    pub n_rows: u8,

    /// Rows of the arrangement.
    pub rows: Box<Array<ArrangeRow, 256>>,
}

impl Default for ArrangementBlock {
    fn default() -> Self {
        Self {
            name: ARRANGEMENT_DEFAULT_NAME,
            unknown_1: from_fn(|_| 0),
            n_rows: 0,
            rows: ArrangeRow::defaults(),
        }
    }
}

impl IsDefault for ArrangementBlock {
    fn is_default(&self) -> bool {
        let default = &Self::default();

        // when the octatrack creates a new arrangement file, it will reuse a
        // name from a previously created arrangement in a different project
        //
        // no idea why it does this (copying the other file?) but it does it
        // reliably when creating a new project from the project menu.
        default.unknown_1 == self.unknown_1
            && default.n_rows == self.n_rows
            && default.rows == self.rows
    }
}

/// Base model for an arranger row within an arrangement block.
#[derive(Debug, PartialEq, Eq, Clone, ArrayDefaults, BoxedBigArrayDefaults)]
pub enum ArrangeRow {
    /// pattern choice and playback
    PatternRow {
        /// Which Pattern should be played at this point. Patterns are indexed from 0 (A01) -> 256 (P16).
        pattern_id: u8,
        /// How many times to play this arrangement row.
        repetitions: u8,
        /// How track muting is applied during this arrangement row.
        mute_mask: u8,
        /// First part of the Tempo mask for this row.
        /// Needs to be combined with `tempo_2` to work out the actual tempo (not sure how it works yet).
        tempo_1: u8,
        /// Second part of the Tempo mask for this row.
        /// Needs to be combined with `tempo_1` to work out the actual tempo (not sure how it works yet).
        tempo_2: u8,
        /// Which scene is assigned to Scene slot A when this arrangement row is playing.
        scene_a: u8,
        /// Which scene is assigned to Scene slot B when this arrangement row is playing.
        scene_b: u8,
        /// Which trig to start Playing the pattern on.
        offset: u8,
        /// How many trigs to play the pattern for.
        /// Note that this value always has `offset` added to it.
        /// So a length on the machine display of 64 when the offset is 32 will result in a value of 96 in the file data.
        length: u8,
        /// MIDI Track transposes for all 8 midi channels.
        /// 1 -> 48 values are positive transpose settings.
        /// 255 (-1) -> 207 (-48) values are negative transpose settings.
        midi_transpose: [u8; 8],
    },
    /// Loop/Jump/Halt rows are all essentially just loops. Example: Jumps are an infinite loop.
    /// So these are bundled into one type.
    ///
    /// Loops are `loop_count = 0 -> 65` and the `row_target` is any row before this one (`loop_count=0` is infinite looping).
    /// Halts are `loop_count = 0` and the `row_target` is this row.
    /// Jumps are `loop_count = 0` and the `row_target` is any row after this one.
    LoopOrJumpOrHaltRow {
        /// How many times to loop to the `row_target`. Only applies to loops.
        loop_count: u8,
        /// The row number to loop back to, jump to, or end at.
        row_target: u8,
    },
    /// A row of ASCII text data with 15 maximum length.
    ReminderRow(String),
    /// Row is not in use. Only used in an `ArrangementBlock` as a placeholder for null basically.
    EmptyRow(),
}

impl Default for ArrangeRow {
    fn default() -> Self {
        Self::EmptyRow()
    }
}

#[cfg(test)]
mod test {
    mod integrity_check {
        use crate::arrangements::ArrangementFile;
        use crate::{HasHeaderField, OtToolsIoError};

        #[test]
        fn true_valid_header() -> Result<(), OtToolsIoError> {
            let arr = ArrangementFile::default();
            assert!(arr.check_header()?);
            Ok(())
        }

        #[test]
        fn false_invalid_header() -> Result<(), OtToolsIoError> {
            let mut arr = ArrangementFile::default();
            arr.header[0] = 0x01;
            arr.header[1] = 0x01;
            arr.header[2] = 0x50;
            assert!(!arr.check_header()?);
            Ok(())
        }
    }

    mod is_default {
        use crate::arrangements::ArrangementFile;
        use crate::test_utils::get_arrange_dirpath;
        use crate::{IsDefault, OctatrackFileIO};

        #[test]
        fn true_not_modified_default() {
            assert!(ArrangementFile::default().is_default())
        }
        #[test]
        fn true_not_modified_file() {
            let arr =
                ArrangementFile::from_data_file(&get_arrange_dirpath().join("blank.work")).unwrap();
            assert!(arr.is_default())
        }
        #[test]
        fn false_modified_file() {
            let arr =
                ArrangementFile::from_data_file(&get_arrange_dirpath().join("full-options.work"))
                    .unwrap();
            assert!(!arr.is_default())
        }
    }
}

#[cfg(test)]
mod checksum {
    use crate::arrangements::ArrangementFile;
    use crate::test_utils::get_arrange_dirpath;
    use crate::{OctatrackFileIO, OtToolsIoError};
    use std::path::Path;

    // make sure arrangements we're testing are equal first, ignoring the checksums
    fn arr_eq_helper(a: &ArrangementFile, b: &ArrangementFile) {
        assert_eq!(a.header, b.header);
        assert_eq!(a.datatype_version, b.datatype_version);
        assert_eq!(a.unk1, b.unk1);
        assert_eq!(a.unk2, b.unk2);
        assert_eq!(a.arrangements_saved_state, b.arrangements_saved_state);
        assert_eq!(a.arrangement_state_current, b.arrangement_state_current);
        assert_eq!(a.arrangement_state_previous, b.arrangement_state_previous);
    }

    // :eyes: https://github.com/beeb/octarranger/blob/master/src/js/stores/OctaStore.js#L150-L174
    // this only seems to work for default / blank patterns (with name changes)
    // and the 4x patterns test case ...?
    //
    // ... which is the same as the get_checksum function below :/
    //
    // yeah it's basically the same fucntion. when you factor all the crud out,
    // it basically turns into an overcomplicated u16 wrapped sum of individual
    // bytes
    #[allow(dead_code)]
    fn get_checksum_octarranger(bytes: &[u8]) -> Result<u16, OtToolsIoError> {
        let bytes_no_header_no_chk = &bytes[16..bytes.len() - 2];

        let mut chk: u16 = 0;
        for byte_pair in &bytes_no_header_no_chk
            .chunks(2)
            .map(|x| x.to_vec())
            .filter(|x| x.len() > 1)
            .collect::<Vec<Vec<u8>>>()
        {
            let first_byte = byte_pair[0] as u16;
            let second_byte = byte_pair[1] as u16;
            chk = second_byte.wrapping_add(first_byte.wrapping_add(chk));
        }

        Ok(chk)
    }

    fn get_checksum_simple(bytes: &[u8]) -> Result<u16, OtToolsIoError> {
        let bytes_no_header_no_chk = &bytes[16..bytes.len() - 2];

        let mut prev;
        let mut chk: u32 = 0;
        for byte in bytes_no_header_no_chk {
            prev = chk;
            chk = chk.wrapping_add((*byte as u32).wrapping_add(0));
            if byte != &0 {
                println!("chk: {chk} diff: {}", chk - prev);
            }
        }
        println!("CHK32: {chk}");
        Ok((chk).wrapping_mul(1) as u16)
    }

    // TODO: Very dirty implementation
    // not working for arrangements -- looks like these have a different checksum implementation
    #[allow(dead_code)]
    fn get_checksum_bank(bytes: &[u8]) -> Result<u16, OtToolsIoError> {
        let bytes_no_header_no_chk = &bytes[16..bytes.len() - 2];
        let default_bytes = &bincode::serialize(&ArrangementFile::default())?;
        let def_important_bytes = &default_bytes[16..bytes.len() - 2];
        let default_checksum: i32 = 1870;
        let mut byte_diffs: i32 = 0;
        for (byte, def_byte) in bytes_no_header_no_chk.iter().zip(def_important_bytes) {
            let byte_diff = (*byte as i32) - (*def_byte as i32);
            if byte_diff != 0 {
                byte_diffs += byte_diff;
            }
        }
        let check = byte_diffs * 256 + default_checksum;
        let modded = check.rem_euclid(65535);
        Ok(modded as u16)
    }

    fn helper_test_chksum(fp: &Path) {
        let valid = ArrangementFile::from_data_file(fp).unwrap();
        let mut test = valid.clone();
        test.checksum = 0;
        arr_eq_helper(&test, &valid);

        let bytes = bincode::serialize(&test).unwrap();
        let r = get_checksum_simple(&bytes);
        assert!(r.is_ok());
        let res = r.unwrap();
        let s_attr_chk: u32 = bytes[16..bytes.len() - 2]
            .iter()
            .map(|x| *x as u32)
            .sum::<u32>()
            .rem_euclid(u16::MAX as u32 + 1);

        let non_zero_bytes = bytes.iter().filter(|b| b > &&0).count();
        let non_zero_sum = bytes
            .iter()
            .cloned()
            .filter(|b| b > &0)
            .map(|x| x as u32)
            .sum::<u32>();

        println!(
            "l: {} r: {} non_zero_bytes {} sum total: {} s-attr {} diff {} (or {})",
            res,
            valid.checksum,
            non_zero_bytes,
            non_zero_sum,
            s_attr_chk,
            res.wrapping_sub(valid.checksum),
            valid.checksum.wrapping_sub(res)
        );
        println!(
            "checksum bytes: {:?} target bytes: {:?}",
            [(res >> 8) as u8, res as u8],
            [(valid.checksum >> 8) as u8, valid.checksum as u8]
        );
        assert_eq!(res, valid.checksum);
    }

    // works with original sample attrs implementation
    #[test]
    fn blank() {
        helper_test_chksum(&get_arrange_dirpath().join("blank.work"));
    }

    // works with original sample attrs implementation
    #[test]
    fn blank_diffname1() {
        helper_test_chksum(&get_arrange_dirpath().join("blank-diffname1.work"));
    }

    // works with original sample attrs implementation
    #[test]
    fn blank_diffname2() {
        helper_test_chksum(&get_arrange_dirpath().join("blank-diffname2.work"));
    }

    // current diff to sample attrs impl = 142 * 8 (1136)
    #[test]
    #[ignore]
    fn one_rem_row_notext() {
        helper_test_chksum(&get_arrange_dirpath().join("1-rem-blank-txt.work"));
    }

    // current difference = 1796
    // so the CHAIN text adds 660
    #[test]
    #[ignore]
    fn one_rem_row_wtext() {
        helper_test_chksum(&get_arrange_dirpath().join("1-rem-CHAIN-txt.work"));
    }

    #[test]
    #[ignore]
    fn two_rem_row_wtext() {
        helper_test_chksum(&get_arrange_dirpath().join("2-rem-CHAIN-txt.work"));
    }

    #[test]
    #[ignore]
    fn four_patterns() {
        helper_test_chksum(&get_arrange_dirpath().join("4-patterns.work"));
    }

    #[test]
    #[ignore]
    fn one_pattern() {
        helper_test_chksum(&get_arrange_dirpath().join("1-pattern.work"));
    }

    #[test]
    #[ignore]
    fn one_halt() {
        helper_test_chksum(&get_arrange_dirpath().join("1-halt.work"));
        // works for this specific case only

        // let bytes = bincode::serialize(&arr).unwrap();
        // let bytes_no_header_no_chk = &bytes[16..bytes.len() - 2];
        //
        // let mut chk: u16 = 0;
        // for byte in bytes_no_header_no_chk {
        //     chk = chk.wrapping_add(*byte as u16);
        // }
        // let checksum = chk.wrapping_mul(2).wrapping_add(254);
        //
        // println!(
        //     "checksum bytes: {:?} target bytes: {:?}",
        //     [(checksum >> 8) as u8, checksum as u8],
        //     [(valid.checksum >> 8) as u8, valid.checksum as u8]
        // );
        // println!(
        //     "diff: {} (or {})",
        //     checksum.wrapping_sub(valid.checksum),
        //     valid.checksum.wrapping_sub(checksum)
        // );
        // assert_eq!(checksum, valid.checksum);
    }

    #[test]
    #[ignore]
    fn one_pattern_1_loop() {
        helper_test_chksum(&get_arrange_dirpath().join("1-patt-1-loop.work"));
    }

    #[test]
    #[ignore]
    fn one_pattern_1_jump_1_loop() {
        helper_test_chksum(&get_arrange_dirpath().join("1-patt-1-jump-1-loop.work"));
    }

    #[test]
    #[ignore]
    fn one_pattern_1_jump_1_loop_1_halt() {
        helper_test_chksum(&get_arrange_dirpath().join("1-patt-1-jump-1-loop-1-halt.work"));
    }

    #[test]
    #[ignore]
    fn full_options() {
        helper_test_chksum(&get_arrange_dirpath().join("full-options.work"));
    }

    // FIXME: Did i not save/copy this properly? this only has empty rows
    #[test]
    #[ignore]
    fn full_options_no_rems() {
        helper_test_chksum(&get_arrange_dirpath().join("full-options-no-rems.work"));
    }

    #[test]
    fn no_saved_flag() {
        helper_test_chksum(&get_arrange_dirpath().join("no-saved-flag.work"));
    }

    #[test]
    fn with_saved_flag() {
        helper_test_chksum(&get_arrange_dirpath().join("with-saved-flag.work"));
    }

    #[test]
    fn blank_samename_saved() {
        helper_test_chksum(&get_arrange_dirpath().join("blank-samename-saved-chktest.work"));
    }

    #[test]
    fn blank_samename_unsaved() {
        helper_test_chksum(&get_arrange_dirpath().join("blank-samename-unsaved-chktest.work"));
    }
}