cu29_export/

lib.rs

mod fsck;
pub mod logstats;

#[cfg(feature = "mcap")]
pub mod mcap_export;

#[cfg(feature = "mcap")]
pub mod serde_to_jsonschema;

use bincode::Decode;
use bincode::config::standard;
use bincode::decode_from_std_read;
use bincode::error::DecodeError;
use clap::{Parser, Subcommand, ValueEnum};
use cu29::UnifiedLogType;
use cu29::prelude::*;
use cu29_intern_strs::read_interned_strings;
use fsck::check;
#[cfg(feature = "mcap")]
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
use logstats::{compute_logstats, write_logstats};
use serde::Serialize;
use std::fmt::{Display, Formatter};
#[cfg(feature = "mcap")]
use std::io::IsTerminal;
use std::io::Read;
use std::path::{Path, PathBuf};

#[cfg(feature = "mcap")]
pub use mcap_export::{
    McapExportStats, PayloadSchemas, export_to_mcap, export_to_mcap_with_schemas, mcap_info,
};

#[cfg(feature = "mcap")]
pub use serde_to_jsonschema::trace_type_to_jsonschema;

#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
pub enum ExportFormat {
    Json,
    Csv,
}

impl Display for ExportFormat {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        match self {
            ExportFormat::Json => write!(f, "json"),
            ExportFormat::Csv => write!(f, "csv"),
        }
    }
}

/// This is a generator for a main function to build a log extractor.
#[derive(Parser)]
#[command(author, version, about)]
pub struct LogReaderCli {
    /// The base path is the name with no _0 _1 et the end.
    /// for example for toto_0.copper, toto_1.copper ... the base name is toto.copper
    pub unifiedlog_base: PathBuf,

    #[command(subcommand)]
    pub command: Command,
}

#[derive(Subcommand)]
pub enum Command {
    /// Extract logs
    ExtractTextLog { log_index: PathBuf },
    /// Extract copperlists
    ExtractCopperlists {
        #[arg(short, long, default_value_t = ExportFormat::Json)]
        export_format: ExportFormat,
    },
    /// Check the log and dump info about it.
    Fsck {
        #[arg(short, long, action = clap::ArgAction::Count)]
        verbose: u8,
    },
    /// Export log statistics to JSON for offline DAG rendering.
    LogStats {
        /// Output JSON file path
        #[arg(short, long, default_value = "cu29_logstats.json")]
        output: PathBuf,
        /// Config file used to map outputs to edges
        #[arg(long, default_value = "copperconfig.ron")]
        config: PathBuf,
        /// Mission id to use when reading the config
        #[arg(long)]
        mission: Option<String>,
    },
    /// Export copperlists to MCAP format (requires 'mcap' feature)
    #[cfg(feature = "mcap")]
    ExportMcap {
        /// Output MCAP file path
        #[arg(short, long)]
        output: PathBuf,
        /// Force progress bar even when stderr is not a TTY
        #[arg(long)]
        progress: bool,
        /// Suppress the progress bar
        #[arg(long)]
        quiet: bool,
    },
    /// Inspect an MCAP file and dump metadata, schemas, and stats (requires 'mcap' feature)
    #[cfg(feature = "mcap")]
    McapInfo {
        /// Path to the MCAP file to inspect
        mcap_file: PathBuf,
        /// Show full schema content
        #[arg(short, long)]
        schemas: bool,
        /// Show sample messages (first N messages per channel)
        #[arg(short = 'n', long, default_value_t = 0)]
        sample_messages: usize,
    },
}

fn write_json_pretty<T: Serialize + ?Sized>(value: &T) -> CuResult<()> {
    serde_json::to_writer_pretty(std::io::stdout(), value)
        .map_err(|e| CuError::new_with_cause("Failed to write JSON output", e))
}

fn write_json<T: Serialize + ?Sized>(value: &T) -> CuResult<()> {
    serde_json::to_writer(std::io::stdout(), value)
        .map_err(|e| CuError::new_with_cause("Failed to write JSON output", e))
}

fn build_read_logger(unifiedlog_base: &Path) -> CuResult<UnifiedLoggerRead> {
    let logger = UnifiedLoggerBuilder::new()
        .file_base_name(unifiedlog_base)
        .build()
        .map_err(|e| CuError::new_with_cause("Failed to create logger", e))?;
    match logger {
        UnifiedLogger::Read(dl) => Ok(dl),
        UnifiedLogger::Write(_) => Err(CuError::from(
            "Expected read-only unified logger in export CLI",
        )),
    }
}

/// This is a generator for a main function to build a log extractor.
/// It depends on the specific type of the CopperList payload that is determined at compile time from the configuration.
///
/// When the `mcap` feature is enabled, P must also implement `PayloadSchemas` for MCAP export support.
#[cfg(feature = "mcap")]
pub fn run_cli<P>() -> CuResult<()>
where
    P: CopperListTuple + CuPayloadRawBytes + mcap_export::PayloadSchemas,
{
    run_cli_inner::<P>()
}

/// This is a generator for a main function to build a log extractor.
/// It depends on the specific type of the CopperList payload that is determined at compile time from the configuration.
#[cfg(not(feature = "mcap"))]
pub fn run_cli<P>() -> CuResult<()>
where
    P: CopperListTuple + CuPayloadRawBytes,
{
    run_cli_inner::<P>()
}

#[cfg(feature = "mcap")]
fn run_cli_inner<P>() -> CuResult<()>
where
    P: CopperListTuple + CuPayloadRawBytes + mcap_export::PayloadSchemas,
{
    let args = LogReaderCli::parse();
    let unifiedlog_base = args.unifiedlog_base;

    let mut dl = build_read_logger(&unifiedlog_base)?;

    match args.command {
        Command::ExtractTextLog { log_index } => {
            let reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::StructuredLogLine);
            textlog_dump(reader, &log_index)?;
        }
        Command::ExtractCopperlists { export_format } => {
            println!("Extracting copperlists with format: {export_format}");
            let mut reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::CopperList);
            let iter = copperlists_reader::<P>(&mut reader);

            match export_format {
                ExportFormat::Json => {
                    for entry in iter {
                        write_json_pretty(&entry)?;
                    }
                }
                ExportFormat::Csv => {
                    let mut first = true;
                    for origin in P::get_all_task_ids() {
                        if !first {
                            print!(", ");
                        } else {
                            print!("id, ");
                        }
                        print!("{origin}_time, {origin}_tov, {origin},");
                        first = false;
                    }
                    println!();
                    for entry in iter {
                        let mut first = true;
                        for msg in entry.cumsgs() {
                            if let Some(payload) = msg.payload() {
                                if !first {
                                    print!(", ");
                                } else {
                                    print!("{}, ", entry.id);
                                }
                                let metadata = msg.metadata();
                                print!("{}, {}, ", metadata.process_time(), msg.tov());
                                write_json(payload)?; // TODO: escape for CSV
                                first = false;
                            }
                        }
                        println!();
                    }
                }
            }
        }
        Command::Fsck { verbose } => {
            if let Some(value) = check::<P>(&mut dl, verbose) {
                return value;
            }
        }
        Command::LogStats {
            output,
            config,
            mission,
        } => {
            run_logstats::<P>(dl, output, config, mission)?;
        }
        #[cfg(feature = "mcap")]
        Command::ExportMcap {
            output,
            progress,
            quiet,
        } => {
            println!("Exporting copperlists to MCAP format: {}", output.display());

            let show_progress = should_show_progress(progress, quiet);
            let total_bytes = if show_progress {
                Some(copperlist_total_bytes(&unifiedlog_base)?)
            } else {
                None
            };

            let reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::CopperList);

            // Export to MCAP with schemas
            // Note: P must implement PayloadSchemas and provide schemas for each task output.
            let stats = if let Some(total_bytes) = total_bytes {
                let progress_bar = make_progress_bar(total_bytes);
                let reader = ProgressReader::new(reader, progress_bar.clone());
                let result = export_to_mcap_impl::<P>(reader, &output);
                progress_bar.finish_and_clear();
                result?
            } else {
                export_to_mcap_impl::<P>(reader, &output)?
            };
            println!("{stats}");
        }
        #[cfg(feature = "mcap")]
        Command::McapInfo {
            mcap_file,
            schemas,
            sample_messages,
        } => {
            mcap_info(&mcap_file, schemas, sample_messages)?;
        }
    }

    Ok(())
}

#[cfg(not(feature = "mcap"))]
fn run_cli_inner<P>() -> CuResult<()>
where
    P: CopperListTuple + CuPayloadRawBytes,
{
    let args = LogReaderCli::parse();
    let unifiedlog_base = args.unifiedlog_base;

    let mut dl = build_read_logger(&unifiedlog_base)?;

    match args.command {
        Command::ExtractTextLog { log_index } => {
            let reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::StructuredLogLine);
            textlog_dump(reader, &log_index)?;
        }
        Command::ExtractCopperlists { export_format } => {
            println!("Extracting copperlists with format: {export_format}");
            let mut reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::CopperList);
            let iter = copperlists_reader::<P>(&mut reader);

            match export_format {
                ExportFormat::Json => {
                    for entry in iter {
                        write_json_pretty(&entry)?;
                    }
                }
                ExportFormat::Csv => {
                    let mut first = true;
                    for origin in P::get_all_task_ids() {
                        if !first {
                            print!(", ");
                        } else {
                            print!("id, ");
                        }
                        print!("{origin}_time, {origin}_tov, {origin},");
                        first = false;
                    }
                    println!();
                    for entry in iter {
                        let mut first = true;
                        for msg in entry.cumsgs() {
                            if let Some(payload) = msg.payload() {
                                if !first {
                                    print!(", ");
                                } else {
                                    print!("{}, ", entry.id);
                                }
                                let metadata = msg.metadata();
                                print!("{}, {}, ", metadata.process_time(), msg.tov());
                                write_json(payload)?;
                                first = false;
                            }
                        }
                        println!();
                    }
                }
            }
        }
        Command::Fsck { verbose } => {
            if let Some(value) = check::<P>(&mut dl, verbose) {
                return value;
            }
        }
        Command::LogStats {
            output,
            config,
            mission,
        } => {
            run_logstats::<P>(dl, output, config, mission)?;
        }
    }

    Ok(())
}

fn run_logstats<P>(
    dl: UnifiedLoggerRead,
    output: PathBuf,
    config: PathBuf,
    mission: Option<String>,
) -> CuResult<()>
where
    P: CopperListTuple + CuPayloadRawBytes,
{
    let config_path = config
        .to_str()
        .ok_or_else(|| CuError::from("Config path is not valid UTF-8"))?;
    let cfg = read_configuration(config_path)
        .map_err(|e| CuError::new_with_cause("Failed to read configuration", e))?;
    let reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::CopperList);
    let stats = compute_logstats::<P>(reader, &cfg, mission.as_deref())?;
    write_logstats(&stats, &output)
}

/// Helper function for MCAP export.
///
/// Uses the PayloadSchemas trait to get task payload schemas.
#[cfg(feature = "mcap")]
fn export_to_mcap_impl<P>(src: impl Read, output: &Path) -> CuResult<McapExportStats>
where
    P: CopperListTuple + mcap_export::PayloadSchemas,
{
    mcap_export::export_to_mcap::<P, _>(src, output)
}

#[cfg(feature = "mcap")]
struct ProgressReader<R> {
    inner: R,
    progress: ProgressBar,
}

#[cfg(feature = "mcap")]
impl<R> ProgressReader<R> {
    fn new(inner: R, progress: ProgressBar) -> Self {
        Self { inner, progress }
    }
}

#[cfg(feature = "mcap")]
impl<R: Read> Read for ProgressReader<R> {
    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
        let read = self.inner.read(buf)?;
        if read > 0 {
            self.progress.inc(read as u64);
        }
        Ok(read)
    }
}

#[cfg(feature = "mcap")]
fn make_progress_bar(total_bytes: u64) -> ProgressBar {
    let progress_bar = ProgressBar::new(total_bytes);
    progress_bar.set_draw_target(ProgressDrawTarget::stderr_with_hz(5));

    let style = ProgressStyle::with_template(
        "[{elapsed_precise}] {bar:40} {bytes}/{total_bytes} ({bytes_per_sec}, ETA {eta})",
    )
    .unwrap_or_else(|_| ProgressStyle::default_bar());

    progress_bar.set_style(style.progress_chars("=>-"));
    progress_bar
}

#[cfg(feature = "mcap")]
fn should_show_progress(force_progress: bool, quiet: bool) -> bool {
    !quiet && (force_progress || std::io::stderr().is_terminal())
}

#[cfg(feature = "mcap")]
fn copperlist_total_bytes(log_base: &Path) -> CuResult<u64> {
    let mut reader = UnifiedLoggerRead::new(log_base)
        .map_err(|e| CuError::new_with_cause("Failed to open log for progress estimation", e))?;
    reader
        .scan_section_bytes(UnifiedLogType::CopperList)
        .map_err(|e| CuError::new_with_cause("Failed to scan log for progress estimation", e))
}

fn read_next_entry<T: Decode<()>>(src: &mut impl Read) -> Option<T> {
    let entry = decode_from_std_read::<T, _, _>(src, standard());
    match entry {
        Ok(entry) => Some(entry),
        Err(DecodeError::UnexpectedEnd { .. }) => None,
        Err(DecodeError::Io { inner, additional }) => {
            if inner.kind() == std::io::ErrorKind::UnexpectedEof {
                None
            } else {
                println!("Error {inner:?} additional:{additional}");
                None
            }
        }
        Err(e) => {
            println!("Error {e:?}");
            None
        }
    }
}

/// Extracts the copper lists from a binary representation.
/// P is the Payload determined by the configuration of the application.
pub fn copperlists_reader<P: CopperListTuple>(
    mut src: impl Read,
) -> impl Iterator<Item = CopperList<P>> {
    std::iter::from_fn(move || read_next_entry::<CopperList<P>>(&mut src))
}

/// Extracts the keyframes from the log.
pub fn keyframes_reader(mut src: impl Read) -> impl Iterator<Item = KeyFrame> {
    std::iter::from_fn(move || read_next_entry::<KeyFrame>(&mut src))
}

pub fn structlog_reader(mut src: impl Read) -> impl Iterator<Item = CuResult<CuLogEntry>> {
    std::iter::from_fn(move || {
        let entry = decode_from_std_read::<CuLogEntry, _, _>(&mut src, standard());

        match entry {
            Err(DecodeError::UnexpectedEnd { .. }) => None,
            Err(DecodeError::Io {
                inner,
                additional: _,
            }) => {
                if inner.kind() == std::io::ErrorKind::UnexpectedEof {
                    None
                } else {
                    Some(Err(CuError::new_with_cause("Error reading log", inner)))
                }
            }
            Err(e) => Some(Err(CuError::new_with_cause("Error reading log", e))),
            Ok(entry) => {
                if entry.msg_index == 0 {
                    None
                } else {
                    Some(Ok(entry))
                }
            }
        }
    })
}

/// Full dump of the copper structured log from its binary representation.
/// This rebuilds a textual log.
/// src: the source of the log data
/// index: the path to the index file (containing the interned strings constructed at build time)
pub fn textlog_dump(src: impl Read, index: &Path) -> CuResult<()> {
    let all_strings = read_interned_strings(index).map_err(|e| {
        CuError::new_with_cause(
            "Failed to read interned strings from index",
            std::io::Error::other(e),
        )
    })?;

    for result in structlog_reader(src) {
        match result {
            Ok(entry) => match rebuild_logline(&all_strings, &entry) {
                Ok(line) => println!("{line}"),
                Err(e) => println!("Failed to rebuild log line: {e:?}"),
            },
            Err(e) => return Err(e),
        }
    }

    Ok(())
}

// only for users opting into python interface, not supported on macOS at the moment
#[cfg(all(feature = "python", not(target_os = "macos")))]
mod python {
    use bincode::config::standard;
    use bincode::decode_from_std_read;
    use bincode::error::DecodeError;
    use cu29::prelude::*;
    use cu29_intern_strs::read_interned_strings;
    use pyo3::exceptions::PyIOError;
    use pyo3::prelude::*;
    use pyo3::types::{PyDelta, PyDict, PyList};
    use std::io::Read;
    use std::path::Path;

    #[pyclass]
    pub struct PyLogIterator {
        reader: Box<dyn Read + Send + Sync>,
    }

    #[pymethods]
    impl PyLogIterator {
        fn __iter__(slf: PyRefMut<Self>) -> PyRefMut<Self> {
            slf
        }

        fn __next__(mut slf: PyRefMut<Self>) -> Option<PyResult<PyCuLogEntry>> {
            match decode_from_std_read::<CuLogEntry, _, _>(&mut slf.reader, standard()) {
                Ok(entry) => {
                    if entry.msg_index == 0 {
                        None
                    } else {
                        Some(Ok(PyCuLogEntry { inner: entry }))
                    }
                }
                Err(DecodeError::UnexpectedEnd { .. }) => None,
                Err(DecodeError::Io { inner, .. })
                    if inner.kind() == std::io::ErrorKind::UnexpectedEof =>
                {
                    None
                }
                Err(e) => Some(Err(PyIOError::new_err(e.to_string()))),
            }
        }
    }

    /// Creates an iterator of CuLogEntries from a bare binary structured log file (ie. not within a unified log).
    /// This is mainly used for using the structured logging out of the Copper framework.
    /// it returns a tuple with the iterator of log entries and the list of interned strings.
    #[pyfunction]
    pub fn struct_log_iterator_bare(
        bare_struct_src_path: &str,
        index_path: &str,
    ) -> PyResult<(PyLogIterator, Vec<String>)> {
        let file = std::fs::File::open(bare_struct_src_path)
            .map_err(|e| PyIOError::new_err(e.to_string()))?;
        let all_strings = read_interned_strings(Path::new(index_path))
            .map_err(|e| PyIOError::new_err(e.to_string()))?;
        Ok((
            PyLogIterator {
                reader: Box::new(file),
            },
            all_strings,
        ))
    }
    /// Creates an iterator of CuLogEntries from a unified log file.
    /// This function allows you to easily use python to datamind Copper's structured text logs.
    /// it returns a tuple with the iterator of log entries and the list of interned strings.
    #[pyfunction]
    pub fn struct_log_iterator_unified(
        unified_src_path: &str,
        index_path: &str,
    ) -> PyResult<(PyLogIterator, Vec<String>)> {
        let all_strings = read_interned_strings(Path::new(index_path))
            .map_err(|e| PyIOError::new_err(e.to_string()))?;

        let logger = UnifiedLoggerBuilder::new()
            .file_base_name(Path::new(unified_src_path))
            .build()
            .map_err(|e| PyIOError::new_err(e.to_string()))?;
        let dl = match logger {
            UnifiedLogger::Read(dl) => dl,
            UnifiedLogger::Write(_) => {
                return Err(PyIOError::new_err(
                    "Expected read-only unified logger for Python export",
                ));
            }
        };

        let reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::StructuredLogLine);
        Ok((
            PyLogIterator {
                reader: Box::new(reader),
            },
            all_strings,
        ))
    }

    /// This is a python wrapper for CuLogEntries.
    #[pyclass]
    pub struct PyCuLogEntry {
        pub inner: CuLogEntry,
    }

    #[pymethods]
    impl PyCuLogEntry {
        /// Returns the timestamp of the log entry.
        pub fn ts<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, PyDelta>> {
            let nanoseconds: u64 = self.inner.time.into();

            // Convert nanoseconds to seconds and microseconds
            let days = (nanoseconds / 86_400_000_000_000) as i32;
            let seconds = (nanoseconds / 1_000_000_000) as i32;
            let microseconds = ((nanoseconds % 1_000_000_000) / 1_000) as i32;

            PyDelta::new(py, days, seconds, microseconds, false)
        }

        /// Returns the index of the message in the vector of interned strings.
        pub fn msg_index(&self) -> u32 {
            self.inner.msg_index
        }

        /// Returns the index of the parameter names in the vector of interned strings.
        pub fn paramname_indexes(&self) -> Vec<u32> {
            self.inner.paramname_indexes.iter().copied().collect()
        }

        /// Returns the parameters of this log line
        pub fn params(&self, py: Python<'_>) -> PyResult<Vec<Py<PyAny>>> {
            self.inner
                .params
                .iter()
                .map(|value| value_to_py(value, py))
                .collect()
        }
    }

    /// This needs to match the name of the generated '.so'
    #[pymodule(name = "libcu29_export")]
    fn cu29_export(m: &Bound<'_, PyModule>) -> PyResult<()> {
        m.add_class::<PyCuLogEntry>()?;
        m.add_class::<PyLogIterator>()?;
        m.add_function(wrap_pyfunction!(struct_log_iterator_bare, m)?)?;
        m.add_function(wrap_pyfunction!(struct_log_iterator_unified, m)?)?;
        Ok(())
    }

    fn value_to_py(value: &cu29::prelude::Value, py: Python<'_>) -> PyResult<Py<PyAny>> {
        match value {
            Value::String(s) => Ok(s.into_pyobject(py)?.into()),
            Value::U64(u) => Ok(u.into_pyobject(py)?.into()),
            Value::I64(i) => Ok(i.into_pyobject(py)?.into()),
            Value::F64(f) => Ok(f.into_pyobject(py)?.into()),
            Value::Bool(b) => Ok(b.into_pyobject(py)?.to_owned().into()),
            Value::CuTime(t) => Ok(t.0.into_pyobject(py)?.into()),
            Value::Bytes(b) => Ok(b.into_pyobject(py)?.into()),
            Value::Char(c) => Ok(c.into_pyobject(py)?.into()),
            Value::I8(i) => Ok(i.into_pyobject(py)?.into()),
            Value::U8(u) => Ok(u.into_pyobject(py)?.into()),
            Value::I16(i) => Ok(i.into_pyobject(py)?.into()),
            Value::U16(u) => Ok(u.into_pyobject(py)?.into()),
            Value::I32(i) => Ok(i.into_pyobject(py)?.into()),
            Value::U32(u) => Ok(u.into_pyobject(py)?.into()),
            Value::Map(m) => {
                let dict = PyDict::new(py);
                for (k, v) in m.iter() {
                    dict.set_item(value_to_py(k, py)?, value_to_py(v, py)?)?;
                }
                Ok(dict.into_pyobject(py)?.into())
            }
            Value::F32(f) => Ok(f.into_pyobject(py)?.into()),
            Value::Option(o) => match o.as_ref() {
                Some(value) => value_to_py(value, py),
                None => Ok(py.None()),
            },
            Value::Unit => Ok(py.None()),
            Value::Newtype(v) => value_to_py(v, py),
            Value::Seq(s) => {
                let items: Vec<Py<PyAny>> = s
                    .iter()
                    .map(|value| value_to_py(value, py))
                    .collect::<PyResult<_>>()?;
                let list = PyList::new(py, items)?;
                Ok(list.into_pyobject(py)?.into())
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use bincode::{Decode, Encode, encode_into_slice};
    use serde::Deserialize;
    use std::env;
    use std::fs;
    use std::io::Cursor;
    use std::path::PathBuf;
    use std::sync::{Arc, Mutex};
    use tempfile::{TempDir, tempdir};

    fn copy_stringindex_to_temp(tmpdir: &TempDir) -> PathBuf {
        // Build a minimal index on the fly so tests don't depend on build-time artifacts.
        let fake_out_dir = tmpdir.path().join("build").join("out").join("dir");
        fs::create_dir_all(&fake_out_dir).unwrap();
        // SAFETY: Tests run single-threaded here and we only read the variable after setting it.
        unsafe {
            env::set_var("LOG_INDEX_DIR", &fake_out_dir);
        }

        // Provide entries for the message indexes used in this test module.
        let _ = cu29_intern_strs::intern_string("unused to start counter");
        let _ = cu29_intern_strs::intern_string("Just a String {}");
        let _ = cu29_intern_strs::intern_string("Just a String (low level) {}");
        let _ = cu29_intern_strs::intern_string("Just a String (end to end) {}");

        let index_dir = cu29_intern_strs::default_log_index_dir();
        cu29_intern_strs::read_interned_strings(&index_dir).unwrap();
        index_dir
    }

    #[test]
    fn test_extract_low_level_cu29_log() {
        let temp_dir = TempDir::new().unwrap();
        let temp_path = copy_stringindex_to_temp(&temp_dir);
        let entry = CuLogEntry::new(3, CuLogLevel::Info);
        let bytes = bincode::encode_to_vec(&entry, standard()).unwrap();
        let reader = Cursor::new(bytes.as_slice());
        textlog_dump(reader, temp_path.as_path()).unwrap();
    }

    #[test]
    fn end_to_end_datalogger_and_structlog_test() {
        let dir = tempdir().expect("Failed to create temp dir");
        let path = dir
            .path()
            .join("end_to_end_datalogger_and_structlog_test.copper");
        {
            // Write a couple log entries
            let UnifiedLogger::Write(logger) = UnifiedLoggerBuilder::new()
                .write(true)
                .create(true)
                .file_base_name(&path)
                .preallocated_size(100000)
                .build()
                .expect("Failed to create logger")
            else {
                panic!("Failed to create logger")
            };
            let data_logger = Arc::new(Mutex::new(logger));
            let stream = stream_write(data_logger.clone(), UnifiedLogType::StructuredLogLine, 1024)
                .expect("Failed to create stream");
            let rt = LoggerRuntime::init(RobotClock::default(), stream, None::<NullLog>);

            let mut entry = CuLogEntry::new(4, CuLogLevel::Info); // this is a "Just a String {}" log line
            entry.add_param(0, Value::String("Parameter for the log line".into()));
            log(&mut entry).expect("Failed to log");
            let mut entry = CuLogEntry::new(2, CuLogLevel::Info); // this is a "Just a String {}" log line
            entry.add_param(0, Value::String("Parameter for the log line".into()));
            log(&mut entry).expect("Failed to log");

            // everything is dropped here
            drop(rt);
        }
        // Read back the log
        let UnifiedLogger::Read(logger) = UnifiedLoggerBuilder::new()
            .file_base_name(
                &dir.path()
                    .join("end_to_end_datalogger_and_structlog_test.copper"),
            )
            .build()
            .expect("Failed to create logger")
        else {
            panic!("Failed to create logger")
        };
        let reader = UnifiedLoggerIOReader::new(logger, UnifiedLogType::StructuredLogLine);
        let temp_dir = TempDir::new().unwrap();
        textlog_dump(
            reader,
            Path::new(copy_stringindex_to_temp(&temp_dir).as_path()),
        )
        .expect("Failed to dump log");
    }

    // This is normally generated at compile time in CuPayload.
    #[derive(Debug, PartialEq, Clone, Copy, Serialize, Deserialize, Encode, Decode, Default)]
    struct MyMsgs((u8, i32, f32));

    impl ErasedCuStampedDataSet for MyMsgs {
        fn cumsgs(&self) -> Vec<&dyn ErasedCuStampedData> {
            Vec::new()
        }
    }

    impl MatchingTasks for MyMsgs {
        fn get_all_task_ids() -> &'static [&'static str] {
            &[]
        }
    }

    /// Checks if we can recover the copper lists from a binary representation.
    #[test]
    fn test_copperlists_dump() {
        let mut data = vec![0u8; 10000];
        let mypls: [MyMsgs; 4] = [
            MyMsgs((1, 2, 3.0)),
            MyMsgs((2, 3, 4.0)),
            MyMsgs((3, 4, 5.0)),
            MyMsgs((4, 5, 6.0)),
        ];

        let mut offset: usize = 0;
        for pl in mypls.iter() {
            let cl = CopperList::<MyMsgs>::new(1, *pl);
            offset +=
                encode_into_slice(&cl, &mut data.as_mut_slice()[offset..], standard()).unwrap();
        }

        let reader = Cursor::new(data);

        let mut iter = copperlists_reader::<MyMsgs>(reader);
        assert_eq!(iter.next().unwrap().msgs, MyMsgs((1, 2, 3.0)));
        assert_eq!(iter.next().unwrap().msgs, MyMsgs((2, 3, 4.0)));
        assert_eq!(iter.next().unwrap().msgs, MyMsgs((3, 4, 5.0)));
        assert_eq!(iter.next().unwrap().msgs, MyMsgs((4, 5, 6.0)));
    }
}