sekirei-train 0.3.71

//! Statistical opening book built from real game records, via `lineprior`
//! (<https://github.com/kent-tokyo/lineprior>) -- a domain-agnostic action-prior
//! library. `lineprior` never sees a shogi concept: this module's only job
//! is walking CSA games into `lineprior::Observation`s (state = SFEN, action
//! = USI move) and handing them to `lineprior::build_prior_book`, which
//! does the actual smoothing/confidence-scoring/ranking.
//!
//! Unlike the NNUE eval (which never saw an opening-phase position until
//! this session's fix — see the ply<20 gap documented in the training
//! pipeline), this is built directly from what strong players actually
//! played, weighted by how often and how successfully. It's a short-term
//! complement to opening-phase training, not a replacement: a book only
//! helps for known positions and falls back to search the moment the
//! opponent deviates, so the eval still needs to carry its own weight past
//! the book's coverage.

use std::io::{self, Write};

use lineprior::{BuildConfig, Observation, Outcome};
use sekirei_core::color::Color;
use sekirei_core::sfen::{board_to_sfen, move_to_usi};

use crate::csa::{CsaGame, GameResult};

/// Replays every game up to `max_ply`, emitting one `Observation` per ply,
/// then hands the whole set to `lineprior::build_prior_book` for smoothing/
/// ranking. Games are assumed already rating-filtered by the caller (reuses
/// `--min-rate`, same as `--export` mode, rather than adding a separate
/// book-specific flag).
#[derive(Debug, Clone)]
pub struct BookBuildReport {
    pub observation_count: usize,
    pub retained_state_count: usize,
    pub config: BuildConfig,
    pub config_fingerprint: u64,
}

pub fn build_book(
    games: &[CsaGame],
    max_ply: usize,
    min_count: u64,
    out: &mut impl Write,
) -> io::Result<BookBuildReport> {
    let mut observations = Vec::new();

    for (game_idx, game) in games.iter().enumerate() {
        let mut board = game.initial_board.clone();
        for (ply, &mv) in game.moves.iter().enumerate() {
            if ply >= max_ply {
                break;
            }
            let Ok(step) = u32::try_from(ply) else {
                break;
            };
            let mover = if ply % 2 == 0 {
                Color::Black
            } else {
                Color::White
            };
            let (outcome, score) = match (mover, game.result) {
                (Color::Black, GameResult::BlackWin) | (Color::White, GameResult::WhiteWin) => {
                    (Outcome::Success, Some(1.0))
                }
                (Color::Black, GameResult::WhiteWin) | (Color::White, GameResult::BlackWin) => {
                    (Outcome::Failure, Some(0.0))
                }
                (_, GameResult::Draw) => (Outcome::Draw, Some(0.5)),
                (_, GameResult::Unknown) => (Outcome::Unknown, None),
            };

            observations.push(Observation {
                sequence_id: format!("game{game_idx:06}"),
                step,
                state: board_to_sfen(&board),
                action: move_to_usi(mv),
                outcome,
                score,
                weight: 1.0,
                tags: vec!["opening".to_string()],
                observed_at_unix_seconds: None,
                source: None,
            });

            board.do_move(mv);
        }
    }

    let seen = observations.len();
    let config = BuildConfig {
        min_count,
        max_actions_per_state: Some(6),
        smoothing_alpha: 5.0,
        ..Default::default()
    };
    let book = match lineprior::build_prior_book(&observations, &config) {
        Ok(b) => b,
        Err(e) => {
            eprintln!("book: lineprior build failed: {e}");
            return Err(io::Error::other(e.to_string()));
        }
    };

    let kept = book.entries.len();
    if let Err(e) = lineprior::save_prior_book_with_config(&book, &config, &mut *out) {
        eprintln!("book: failed to write prior book: {e}");
        return Err(io::Error::other(e.to_string()));
    }
    eprintln!(
        "book: {kept} positions kept (of {seen} observations, min_count={min_count}, max_ply={max_ply})"
    );
    Ok(BookBuildReport {
        observation_count: seen,
        retained_state_count: kept,
        config_fingerprint: lineprior::build_config_fingerprint(&config),
        config,
    })
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::csa::parse_csa;

    struct FailingWriter;

    impl Write for FailingWriter {
        fn write(&mut self, _buf: &[u8]) -> io::Result<usize> {
            Err(io::Error::other("intentional writer failure"))
        }

        fn flush(&mut self) -> io::Result<()> {
            Ok(())
        }
    }

    #[test]
    fn generated_book_records_lineprior_schema_and_build_config() {
        let game = parse_csa("V2.2\nPI\n+7776FU\n%TORYO\n").expect("valid CSA");
        let mut output = Vec::new();
        let report = build_book(&[game], 1, 1, &mut output).expect("build book");

        let text = String::from_utf8(output).expect("UTF-8 JSONL");
        let header: serde_json::Value =
            serde_json::from_str(text.lines().next().expect("metadata header"))
                .expect("valid metadata JSON");
        assert_eq!(header["prior_book_schema_version"], 1);
        assert_eq!(header["producer_version"], "0.12.4");
        assert_eq!(header["build_config"]["min_count"], 1);

        let loaded = lineprior::load_prior_book(text.as_bytes()).expect("reload generated book");
        assert_eq!(loaded.entries.len(), 1);
        assert_eq!(report.observation_count, 1);
        assert_eq!(report.retained_state_count, 1);
        assert_eq!(
            report.config_fingerprint,
            lineprior::build_config_fingerprint(&report.config)
        );
    }

    #[test]
    fn every_game_result_and_mover_perspective_builds_observations() {
        let parsed = parse_csa("V2.2\nPI\n+7776FU\n-3334FU\n%CHUDAN\n").unwrap();
        let games: Vec<_> = [
            GameResult::BlackWin,
            GameResult::WhiteWin,
            GameResult::Draw,
            GameResult::Unknown,
        ]
        .into_iter()
        .map(|result| CsaGame {
            initial_board: parsed.initial_board.clone(),
            moves: parsed.moves.clone(),
            result,
            black_rate: None,
            white_rate: None,
        })
        .collect();

        let mut output = Vec::new();
        let report = build_book(&games, 2, 1, &mut output).unwrap();
        assert_eq!(report.observation_count, 8);
        assert!(!output.is_empty());
    }

    #[test]
    fn output_failure_is_reported_without_a_partial_success() {
        let game = parse_csa("V2.2\nPI\n+7776FU\n%TORYO\n").unwrap();
        let error = build_book(&[game], 1, 1, &mut FailingWriter).unwrap_err();
        assert_eq!(error.kind(), io::ErrorKind::Other);
    }
}