ot_tools_io/

arrangements.rs

/*
SPDX-License-Identifier: GPL-3.0-or-later
Copyright © 2024 Mike Robeson [dijksterhuis]
*/

//! Types and ser/de for `arr??.*` binary data files.
//!
//! Proper checksum calculations are not yet implemented. But this doesn't seem
//! to have an impact on loading arrangements onto the Octatrack.

mod deserialize;
mod serialize;

use crate::{
    Defaults, HasChecksumField, HasFileVersionField, HasHeaderField, IsDefault, OctatrackFileIO,
    OtToolsIoError,
};
use ot_tools_io_derive::{ArrayDefaults, BoxedBigArrayDefaults, IntegrityChecks};
use serde::Serialize;
use serde_big_array::Array;
use std::array::from_fn;

/// Arrangement file header data
/// ```text
/// ASCII: FORM....DPS1ARRA........
/// Hex: 46 4f 52 4d 00 00 00 00 44 50 53 31 41 52 52 41 00 00 00 00 00 06
/// U8: [70, 79, 82, 77, 0, 0, 0, 0, 68, 80, 83, 49, 65, 82, 82, 65, 0, 0, 0, 0, 0, 6]
/// ```
pub const ARRANGEMENT_FILE_HEADER: [u8; 21] = [
    70, 79, 82, 77, 0, 0, 0, 0, 68, 80, 83, 49, 65, 82, 82, 65, 0, 0, 0, 0, 0,
];

/// Current/supported version of arrangements files.
pub const ARRANGEMENT_FILE_VERSION: u8 = 6;

/// `"OT_TOOLS_ARR` -- this is a custom name specifically created for ot-tools.
/// The octatrack will normally copy the name of a previously created arrangement
/// when creating arrangements on project creation. Not sure why, but it means
/// arrangements never have a single default name.
const ARRANGEMENT_DEFAULT_NAME: [u8; 15] =
    [79, 67, 84, 65, 84, 79, 79, 76, 83, 45, 65, 82, 82, 32, 32];

// max length: 11336 bytes
/// Base model for `arr??.*` arrangement binary data files.
#[derive(Debug, Clone, Serialize, PartialEq, IntegrityChecks)]
pub struct ArrangementFile {
    /// Header data
    pub header: [u8; 21],

    /// Patch version of this data type. Is inspected on project load to determine if
    /// the octatrack is able to load a project from current OS, or whether the project
    /// files need to be patched and updated.
    pub datatype_version: u8,

    /// Dunno. Example data:
    /// ```text
    /// [0, 0]
    /// ```
    pub unk1: [u8; 2],

    /// Current arrangement data in active use.
    /// This block is written when saving via Project Menu -> SYNC TO CARD.
    ///
    /// The second block is written when saving the arrangement via Arranger Menu -> SAVE.
    pub arrangement_state_current: ArrangementBlock,

    /// Dunno.
    pub unk2: u8,

    /// Whether the arrangement has been saved within the arrangement menu.
    pub saved_flag: u8,

    /// Arrangement data from the previous saved state.
    /// This block is written when saving the arrangement via Arranger Menu -> SAVE.
    pub arrangement_state_previous: ArrangementBlock,

    /// The current save/unsaved state of **all** loaded arrangements.
    /// 'Saved' means there's been at least one ARRANGER MENU save operation performed,
    /// and [`ArrangementBlock`] data has been written to the `arrangement_state_previous` field.
    ///
    /// Example data:
    /// ```text
    /// Arrangement 1 has been saved: [1, 0, 0, 0, 0, 0, 0, 0]
    /// Arrangement 2 has been saved: [0, 1, 0, 0, 0, 0, 0, 0]
    /// Arrangement 2, 7 & 8 have been saved: [0, 1, 0, 0, 0, 0, 1, 1]
    /// ```
    pub arrangements_saved_state: [u8; 8],

    /// Checksum for the file.
    pub checksum: u16,
}

impl Default for ArrangementFile {
    fn default() -> Self {
        // TODO: Clean up once checksums are dealt with
        // let init = Self {
        // let init = Self {
        //     header: ARRANGEMENT_FILE_HEADER,
        //     unk1: from_fn(|_| 0),
        //     arrangement_state_current: ArrangementBlock::default(),
        //     // TODO
        //     unk2: 0,
        //     // TODO
        //     saved_flag: 1,
        //     arrangement_state_previous: ArrangementBlock::default(),
        //     // WARN: by default this actually always 0, but generating test data where these things
        //     // are 'inactive' is basically impossible!
        //     // for now, I'm setting this to default a 1-values array, but in reality it depends...
        //     // cba to type this out fully. gonna bite me later i know it. basically i need to check
        //     // if arrangement chaining affects this flag or if the arrangement has been saved.
        //     arrangements_saved_state: from_fn(|_| 1),
        //     checksum: 1973,
        // };

        // let bytes = bincode::serialize(&init).unwrap();
        // let checksum = get_checksum(&bytes);
        // init.checksum = checksum.unwrap();
        // init

        Self {
            header: ARRANGEMENT_FILE_HEADER,
            datatype_version: ARRANGEMENT_FILE_VERSION,
            unk1: from_fn(|_| 0),
            arrangement_state_current: ArrangementBlock::default(),
            // TODO
            unk2: 0,
            // TODO
            saved_flag: 0,
            arrangement_state_previous: ArrangementBlock::default(),
            // WARN: by default this actually always 0, but generating test data where these things
            // are 'inactive' is basically impossible!
            // for now, I'm setting this to default a 1-values array, but in reality it depends...
            // cba to type this out fully. gonna bite me later i know it. basically i need to check
            // if arrangement chaining affects this flag or if the arrangement has been saved.
            arrangements_saved_state: from_fn(|_| 1),
            checksum: 1973,
        }
    }
}

impl IsDefault for ArrangementFile {
    fn is_default(&self) -> bool {
        let default = &ArrangementFile::default();
        // check everything except the arrangement name fields (see
        // ArrangementBlock's IsDefault implementation for more details)
        self.arrangement_state_current.is_default()
            && self.arrangement_state_previous.is_default()
            && default.unk1 == self.unk1
            && default.unk2 == self.unk2
    }
}

#[cfg(test)]
mod is_default {
    use crate::arrangements::ArrangementFile;
    use crate::test_utils::get_arrange_dirpath;
    use crate::{IsDefault, OctatrackFileIO};

    #[test]
    fn true_not_modified_default() {
        assert!(ArrangementFile::default().is_default())
    }
    #[test]
    fn true_not_modified_file() {
        let arr =
            ArrangementFile::from_data_file(&get_arrange_dirpath().join("blank.work")).unwrap();
        assert!(arr.is_default())
    }
    #[test]
    fn false_modified_file() {
        let arr = ArrangementFile::from_data_file(&get_arrange_dirpath().join("full-options.work"))
            .unwrap();
        assert!(!arr.is_default())
    }
}

impl OctatrackFileIO for ArrangementFile {
    fn encode(&self) -> Result<Vec<u8>, OtToolsIoError> {
        // TODO: oh god it looks like i might need to swap byte order on everything,
        //       possibly including bank data swapping bytes is one required when
        //       running on little-endian systems
        let mut swapped = self.clone();
        if cfg!(target_endian = "little") {
            swapped.checksum = self.checksum.swap_bytes();
        }
        let bytes = bincode::serialize(&swapped)?;
        Ok(bytes)
    }
}

#[cfg(test)]
mod decode {
    use crate::{
        read_bin_file, test_utils::get_arrange_dirpath, ArrangementFile, OctatrackFileIO,
        OtToolsIoError,
    };

    const HACKED_ARR_NAME: [u8; 15] = [68, 73, 70, 70, 78, 65, 77, 69, 66, 76, 65, 78, 75, 49, 0];

    #[test]
    fn valid() -> Result<(), OtToolsIoError> {
        let path = get_arrange_dirpath().join("blank.work");
        let bytes = read_bin_file(&path)?;
        let s = ArrangementFile::decode(&bytes)?;

        // TODO: having to hack the default names
        let mut x = ArrangementFile::default();
        x.arrangement_state_current.name = HACKED_ARR_NAME;
        x.arrangement_state_previous.name = HACKED_ARR_NAME;
        // TODO: Another hack
        x.saved_flag = 1;

        assert_eq!(s, x);
        Ok(())
    }
}

#[cfg(test)]
mod encode {
    const HACKED_ARR_NAME: [u8; 15] = [68, 73, 70, 70, 78, 65, 77, 69, 66, 76, 65, 78, 75, 49, 0];
    use crate::{
        read_bin_file, test_utils::get_arrange_dirpath, ArrangementFile, OctatrackFileIO,
        OtToolsIoError,
    };
    #[test]
    fn valid() -> Result<(), OtToolsIoError> {
        let path = get_arrange_dirpath().join("blank.work");
        let bytes = read_bin_file(&path)?;

        // TODO: having to hack the default names
        let mut x = ArrangementFile::default();
        x.arrangement_state_current.name = HACKED_ARR_NAME;
        x.arrangement_state_previous.name = HACKED_ARR_NAME;
        // TODO: Another hack
        x.saved_flag = 1;

        let b = x.encode()?;
        assert_eq!(b, bytes);
        Ok(())
    }
}

impl HasChecksumField for ArrangementFile {
    fn calculate_checksum(&self) -> Result<u16, OtToolsIoError> {
        Ok(0)
    }
    fn check_checksum(&self) -> Result<bool, OtToolsIoError> {
        Ok(self.checksum == self.calculate_checksum()?)
    }
}

// NOTE: This only tests whether the trait methods work or not, not whether they are correct!
#[cfg(test)]
mod checksum_field {
    use crate::{ArrangementFile, HasChecksumField, OtToolsIoError};
    #[test]
    fn valid() -> Result<(), OtToolsIoError> {
        let mut x = ArrangementFile::default();
        x.checksum = x.calculate_checksum()?;
        assert!(x.check_checksum()?);
        Ok(())
    }

    #[test]
    fn invalid() -> Result<(), OtToolsIoError> {
        let x = ArrangementFile {
            checksum: u16::MAX,
            ..Default::default()
        };
        assert!(!x.check_checksum()?);
        Ok(())
    }

    mod files {
        use crate::arrangements::ArrangementFile;
        use crate::test_utils::{get_arrange_dirpath, get_blank_proj_dirpath};
        use crate::{OctatrackFileIO, OtToolsIoError};
        use std::path::Path;

        // make sure arrangements we're testing are equal first, ignoring the checksums
        fn arr_eq_helper(a: &ArrangementFile, b: &ArrangementFile) {
            assert_eq!(a.header, b.header);
            assert_eq!(a.datatype_version, b.datatype_version);
            assert_eq!(a.unk1, b.unk1);
            assert_eq!(a.unk2, b.unk2);
            assert_eq!(a.arrangements_saved_state, b.arrangements_saved_state);
            assert_eq!(a.arrangement_state_current, b.arrangement_state_current);
            assert_eq!(a.arrangement_state_previous, b.arrangement_state_previous);
        }

        // :eyes: https://github.com/beeb/octarranger/blob/master/src/js/stores/OctaStore.js#L150-L174
        // this only seems to work for default / blank patterns (with name changes)
        // and the 4x patterns test case ...?
        //
        // ... which is the same as the get_checksum function below :/
        //
        // yeah it's basically the same fucntion. when you factor all the crud out,
        // it basically turns into an overcomplicated u16 wrapped sum of individual
        // bytes
        #[allow(dead_code)]
        fn get_checksum_octarranger(bytes: &[u8]) -> Result<u16, OtToolsIoError> {
            let bytes_no_header_no_chk = &bytes[16..bytes.len() - 2];

            let mut chk: u16 = 0;
            for byte_pair in &bytes_no_header_no_chk
                .chunks(2)
                .map(|x| x.to_vec())
                .filter(|x| x.len() > 1)
                .collect::<Vec<Vec<u8>>>()
            {
                let first_byte = byte_pair[0] as u16;
                let second_byte = byte_pair[1] as u16;
                chk = second_byte.wrapping_add(first_byte.wrapping_add(chk));
            }

            Ok(chk)
        }

        fn get_checksum_simple(bytes: &[u8]) -> Result<u16, OtToolsIoError> {
            let bytes_no_header_no_chk = &bytes[16..bytes.len() - 2];

            let mut prev;
            let mut chk: u32 = 0;
            for byte in bytes_no_header_no_chk {
                prev = chk;
                chk = chk.wrapping_add((*byte as u32).wrapping_add(0));
                if byte != &0 {
                    println!("chk: {chk} diff: {}", chk - prev);
                }
            }
            println!("CHK32: {chk}");
            Ok((chk).wrapping_mul(1) as u16)
        }

        // TODO: Very dirty implementation
        // not working for arrangements -- looks like these have a different checksum implementation
        #[allow(dead_code)]
        fn get_checksum_bank(bytes: &[u8]) -> Result<u16, OtToolsIoError> {
            let bytes_no_header_no_chk = &bytes[16..bytes.len() - 2];
            let default_bytes = &bincode::serialize(&ArrangementFile::default())?;
            let def_important_bytes = &default_bytes[16..bytes.len() - 2];
            let default_checksum: i32 = 1870;
            let mut byte_diffs: i32 = 0;
            for (byte, def_byte) in bytes_no_header_no_chk.iter().zip(def_important_bytes) {
                let byte_diff = (*byte as i32) - (*def_byte as i32);
                if byte_diff != 0 {
                    byte_diffs += byte_diff;
                }
            }
            let check = byte_diffs * 256 + default_checksum;
            let modded = check.rem_euclid(65535);
            Ok(modded as u16)
        }

        fn helper_test_chksum(fp: &Path) {
            let valid = ArrangementFile::from_data_file(fp).unwrap();
            let mut test = valid.clone();
            test.checksum = 0;
            arr_eq_helper(&test, &valid);

            let bytes = bincode::serialize(&test).unwrap();
            let r = get_checksum_simple(&bytes);
            assert!(r.is_ok());
            let res = r.unwrap();
            let s_attr_chk: u32 = bytes[16..bytes.len() - 2]
                .iter()
                .map(|x| *x as u32)
                .sum::<u32>()
                .rem_euclid(u16::MAX as u32 + 1);

            let non_zero_bytes = bytes.iter().filter(|b| b > &&0).count();
            let non_zero_sum = bytes
                .iter()
                .cloned()
                .filter(|b| b > &0)
                .map(|x| x as u32)
                .sum::<u32>();

            println!(
                "l: {} r: {} non_zero_bytes {} sum total: {} s-attr {} diff {} (or {})",
                res,
                valid.checksum,
                non_zero_bytes,
                non_zero_sum,
                s_attr_chk,
                res.wrapping_sub(valid.checksum),
                valid.checksum.wrapping_sub(res)
            );
            println!(
                "checksum bytes: {:?} target bytes: {:?}",
                [(res >> 8) as u8, res as u8],
                [(valid.checksum >> 8) as u8, valid.checksum as u8]
            );
            assert_eq!(res, valid.checksum);
        }

        #[test]
        fn blank_project_file() {
            helper_test_chksum(&get_blank_proj_dirpath().join("arr01.work"));
        }

        // works with original sample attrs implementation
        #[test]
        fn blank() {
            helper_test_chksum(&get_arrange_dirpath().join("blank.work"));
        }

        // works with original sample attrs implementation
        #[test]
        fn blank_diffname1() {
            helper_test_chksum(&get_arrange_dirpath().join("blank-diffname1.work"));
        }

        // works with original sample attrs implementation
        #[test]
        fn blank_diffname2() {
            helper_test_chksum(&get_arrange_dirpath().join("blank-diffname2.work"));
        }

        // current diff to sample attrs impl = 142 * 8 (1136)
        #[test]
        #[ignore]
        fn one_rem_row_notext() {
            helper_test_chksum(&get_arrange_dirpath().join("1-rem-blank-txt.work"));
        }

        // current difference = 1796
        // so the CHAIN text adds 660
        #[test]
        #[ignore]
        fn one_rem_row_wtext() {
            helper_test_chksum(&get_arrange_dirpath().join("1-rem-CHAIN-txt.work"));
        }

        #[test]
        #[ignore]
        fn two_rem_row_wtext() {
            helper_test_chksum(&get_arrange_dirpath().join("2-rem-CHAIN-txt.work"));
        }

        #[test]
        #[ignore]
        fn four_patterns() {
            helper_test_chksum(&get_arrange_dirpath().join("4-patterns.work"));
        }

        #[test]
        #[ignore]
        fn one_pattern() {
            helper_test_chksum(&get_arrange_dirpath().join("1-pattern.work"));
        }

        #[test]
        #[ignore]
        fn one_halt() {
            helper_test_chksum(&get_arrange_dirpath().join("1-halt.work"));
            // works for this specific case only

            // let bytes = bincode::serialize(&arr).unwrap();
            // let bytes_no_header_no_chk = &bytes[16..bytes.len() - 2];
            //
            // let mut chk: u16 = 0;
            // for byte in bytes_no_header_no_chk {
            //     chk = chk.wrapping_add(*byte as u16);
            // }
            // let checksum = chk.wrapping_mul(2).wrapping_add(254);
            //
            // println!(
            //     "checksum bytes: {:?} target bytes: {:?}",
            //     [(checksum >> 8) as u8, checksum as u8],
            //     [(valid.checksum >> 8) as u8, valid.checksum as u8]
            // );
            // println!(
            //     "diff: {} (or {})",
            //     checksum.wrapping_sub(valid.checksum),
            //     valid.checksum.wrapping_sub(checksum)
            // );
            // assert_eq!(checksum, valid.checksum);
        }

        #[test]
        #[ignore]
        fn one_pattern_1_loop() {
            helper_test_chksum(&get_arrange_dirpath().join("1-patt-1-loop.work"));
        }

        #[test]
        #[ignore]
        fn one_pattern_1_jump_1_loop() {
            helper_test_chksum(&get_arrange_dirpath().join("1-patt-1-jump-1-loop.work"));
        }

        #[test]
        #[ignore]
        fn one_pattern_1_jump_1_loop_1_halt() {
            helper_test_chksum(&get_arrange_dirpath().join("1-patt-1-jump-1-loop-1-halt.work"));
        }

        #[test]
        #[ignore]
        fn full_options() {
            helper_test_chksum(&get_arrange_dirpath().join("full-options.work"));
        }

        // FIXME: Did i not save/copy this properly? this only has empty rows
        #[test]
        #[ignore]
        fn full_options_no_rems() {
            helper_test_chksum(&get_arrange_dirpath().join("full-options-no-rems.work"));
        }

        #[test]
        fn no_saved_flag() {
            helper_test_chksum(&get_arrange_dirpath().join("no-saved-flag.work"));
        }

        #[test]
        fn with_saved_flag() {
            helper_test_chksum(&get_arrange_dirpath().join("with-saved-flag.work"));
        }

        #[test]
        fn blank_samename_saved() {
            helper_test_chksum(&get_arrange_dirpath().join("blank-samename-saved-chktest.work"));
        }

        #[test]
        fn blank_samename_unsaved() {
            helper_test_chksum(&get_arrange_dirpath().join("blank-samename-unsaved-chktest.work"));
        }
    }
}

impl HasHeaderField for ArrangementFile {
    fn check_header(&self) -> Result<bool, OtToolsIoError> {
        Ok(self.header == ARRANGEMENT_FILE_HEADER)
    }
}

#[cfg(test)]
mod header_field {
    use crate::{ArrangementFile, HasHeaderField, OtToolsIoError};
    #[test]
    fn valid() -> Result<(), OtToolsIoError> {
        assert!(ArrangementFile::default().check_header()?);
        Ok(())
    }

    #[test]
    fn invalid() -> Result<(), OtToolsIoError> {
        let mut mutated = ArrangementFile::default();
        mutated.header[0] = 0x00;
        mutated.header[1] = 200;
        mutated.header[2] = 126;
        assert!(!mutated.check_header()?);
        Ok(())
    }
}

impl HasFileVersionField for ArrangementFile {
    fn check_file_version(&self) -> Result<bool, OtToolsIoError> {
        Ok(self.datatype_version == ARRANGEMENT_FILE_VERSION)
    }
}

#[cfg(test)]
mod file_version_field {
    use crate::{ArrangementFile, HasFileVersionField, OtToolsIoError};
    #[test]
    fn valid() -> Result<(), OtToolsIoError> {
        assert!(ArrangementFile::default().check_file_version()?);
        Ok(())
    }

    #[test]
    fn invalid() -> Result<(), OtToolsIoError> {
        let x = ArrangementFile {
            datatype_version: 0,
            ..Default::default()
        };
        assert!(!x.check_file_version()?);
        Ok(())
    }
}

/// Base model for an arrangement 'block' within an arrangement binary data file.
/// There are two arrangement 'blocks' in each arrangement file -- enabling the
/// arrangement 'reload ' functionality.
#[derive(Debug, Eq, PartialEq, Clone)]
pub struct ArrangementBlock {
    /// Name of the Arrangement in ASCII values, max length 15 characters
    pub name: [u8; 15], // String,

    /// Unknown data. No idea what this is. Usually `[0, 0]` or `[0, 1]`
    pub unknown_1: [u8; 2],

    /// Number of active rows in the arrangement. Any parsed row data after this number of rows
    /// should be an `ArrangeRow::EmptyRow` variant.
    ///
    /// #### WARNING
    /// Max number of `ArrangeRows` (256) is a zero value here! Zero rows are also possible, so do
    /// not use this field as a guaranteed row count.
    pub n_rows: u8,

    /// Rows of the arrangement.
    pub rows: Box<Array<ArrangeRow, 256>>,
}

impl Default for ArrangementBlock {
    fn default() -> Self {
        Self {
            name: ARRANGEMENT_DEFAULT_NAME,
            unknown_1: from_fn(|_| 0),
            n_rows: 0,
            rows: ArrangeRow::defaults(),
        }
    }
}

impl IsDefault for ArrangementBlock {
    fn is_default(&self) -> bool {
        let default = &Self::default();

        // when the octatrack creates a new arrangement file, it will reuse a
        // name from a previously created arrangement in a different project
        //
        // no idea why it does this (copying the other file?) but it does it
        // reliably when creating a new project from the project menu.
        default.unknown_1 == self.unknown_1
            && default.n_rows == self.n_rows
            && default.rows == self.rows
    }
}

/// Base model for an arranger row within an arrangement block.
#[derive(Debug, PartialEq, Eq, Clone, ArrayDefaults, BoxedBigArrayDefaults)]
pub enum ArrangeRow {
    /// pattern choice and playback
    PatternRow {
        /// Which Pattern should be played at this point. Patterns are indexed from 0 (A01) -> 256 (P16).
        pattern_id: u8,
        /// How many times to play this arrangement row.
        repetitions: u8,
        /// How track muting is applied during this arrangement row.
        mute_mask: u8,
        /// First part of the Tempo mask for this row.
        /// Needs to be combined with `tempo_2` to work out the actual tempo (not sure how it works yet).
        tempo_1: u8,
        /// Second part of the Tempo mask for this row.
        /// Needs to be combined with `tempo_1` to work out the actual tempo (not sure how it works yet).
        tempo_2: u8,
        /// Which scene is assigned to Scene slot A when this arrangement row is playing.
        scene_a: u8,
        /// Which scene is assigned to Scene slot B when this arrangement row is playing.
        scene_b: u8,
        /// Which trig to start Playing the pattern on.
        offset: u8,
        /// How many trigs to play the pattern for.
        /// Note that this value always has `offset` added to it.
        /// So a length on the machine display of 64 when the offset is 32 will result in a value of 96 in the file data.
        length: u8,
        /// MIDI Track transposes for all 8 midi channels.
        /// 1 -> 48 values are positive transpose settings.
        /// 255 (-1) -> 207 (-48) values are negative transpose settings.
        midi_transpose: [u8; 8],
    },
    /// Loop/Jump/Halt rows are all essentially just loops. Example: Jumps are an infinite loop.
    /// So these are bundled into one type.
    ///
    /// Loops are `loop_count = 0 -> 65` and the `row_target` is any row before this one (`loop_count=0` is infinite looping).
    /// Halts are `loop_count = 0` and the `row_target` is this row.
    /// Jumps are `loop_count = 0` and the `row_target` is any row after this one.
    LoopOrJumpOrHaltRow {
        /// How many times to loop to the `row_target`. Only applies to loops.
        loop_count: u8,
        /// The row number to loop back to, jump to, or end at.
        row_target: u8,
    },
    /// A row of ASCII text data with 15 maximum length.
    ReminderRow(String),
    /// Row is not in use. Only used in an `ArrangementBlock` as a placeholder for null basically.
    EmptyRow(),
}

impl Default for ArrangeRow {
    fn default() -> Self {
        Self::EmptyRow()
    }
}