#![deny(missing_docs)]
#![deny(clippy::allow_attributes_without_reason)]
#![deny(clippy::as_ptr_cast_mut)]
#![deny(clippy::as_underscore)]
#![deny(clippy::assertions_on_result_states)]
#![deny(clippy::borrow_as_ptr)]
#![deny(clippy::branches_sharing_code)]
#![deny(clippy::case_sensitive_file_extension_comparisons)]
#![deny(clippy::cast_ptr_alignment)]
#![deny(clippy::checked_conversions)]
#![deny(clippy::clear_with_drain)]
#![deny(clippy::clone_on_ref_ptr)]
#![deny(clippy::cloned_instead_of_copied)]
#![deny(clippy::cognitive_complexity)]
#![deny(clippy::collection_is_never_read)]
#![deny(clippy::copy_iterator)]
#![deny(clippy::create_dir)]
#![deny(clippy::dbg_macro)]
#![deny(clippy::debug_assert_with_mut_call)]
#![deny(clippy::decimal_literal_representation)]
#![deny(clippy::default_trait_access)]
#![deny(clippy::default_union_representation)]
#![deny(clippy::derive_partial_eq_without_eq)]
#![deny(clippy::doc_link_with_quotes)]
#![deny(clippy::doc_markdown)]
#![deny(clippy::explicit_into_iter_loop)]
#![deny(clippy::explicit_iter_loop)]
#![deny(clippy::missing_safety_doc)]
#![deny(clippy::undocumented_unsafe_blocks)]
use std::{
cmp::Reverse,
collections::{BTreeMap, HashMap, HashSet},
ffi::{OsStr, OsString},
fs::{self, File},
io::{self, stderr, BufRead, BufWriter, Read, Write},
path::{Path, PathBuf},
str,
sync::{
atomic::{AtomicBool, AtomicUsize, Ordering},
Arc, Mutex,
},
time::Instant,
};
use anyhow::{anyhow, bail, Context, Result};
use human_panic::setup_panic;
use indicatif::ProgressBar;
use is_terminal::IsTerminal;
use itertools::Itertools;
use jja::lieval::LichessEval;
#[macro_use]
extern crate prettytable;
use clap::{builder::PathBufValueParser, Arg, ArgAction, Command};
use console::style;
#[cfg(feature = "i18n")]
use i18n_embed::{gettext::gettext_language_loader, DesktopLanguageRequester};
use jja::{
abk::{
format_abk_entries, traverse_tree, ColorPriority, CompactSBookMoveEntry, NagPriority,
ABK_EDIT_COMMENT, ABK_ENTRY_LENGTH,
},
abkbook::AbkBook,
brainlearn::{
self, exp_entry_to_file, format_exp_entries, ExperienceEntry, BRAINLEARN_EDIT_COMMENT,
EXPERIENCE_ENTRY_SIZE,
},
brainlearnfile::BrainLearnFile,
built_info,
chess::{deserialize_chess, edit_comment, get_board_lines, lines_from_tree, ChessMove, ROOT},
ctg::colored_uci,
ctgbook::{sort_ctg_moves, CtgBook},
eco::{ECO, ECO_MAX},
error::HashCollision,
hash::{
zobrist128_hash, zobrist16_hash, zobrist32_hash, zobrist8_hash, zobrist_hash,
ZobristHashSet, ZobristHasherBuilder,
},
merge::{
average_weight, dynamic_midpoint_merge, entropy_merge, geometric_scaling_merge,
harmonic_merge, logarithmic_merge, quadratic_mean_merge, sigmoid_merge,
weighted_average_weight, weighted_distance_merge, weighted_median_merge, MergeStrategy,
},
obk::ObkMoveEntry,
obkbook::ObkBook,
pgn::{self, pgn_dump, DumpElement, OutputFormat, PositionTracker},
pgnbook::{create_opening_book, deserialize_game_entries, GameEntry},
pgnfilt::{parse_filter_expression, FilterComponent},
polyglot::{
self, bin_entry_to_file, format_bin_entries, from_uci, is_king_on_start_square, BookEntry,
ColorWeight, NagWeight, BOOK_ENTRY_SIZE, POLYGLOT_EDIT_COMMENT,
},
polyglotbook::PolyGlotBook,
quote::{print_quote, search_quote, wrap_text, QUOTE_MAX},
random::{play_random_book_game, play_random_game, MoveSelection},
reexec::jja_call,
stockfish::stockfish_hash,
system::{
edit_tempfile, get_progress_bar, get_progress_spinner, get_username, DISPLAY_PROGRESS,
NPROC, NPROC_STR, STDOUT_IS_TTY,
},
};
use pgn_reader::BufferedReader;
use prettytable::{Attr, Cell, Row, Table};
use quick_csv::Csv;
#[cfg(feature = "i18n")]
use rust_embed::RustEmbed;
use sha2::{Digest, Sha256};
use shakmaty::{
fen::{Epd, Fen},
san::SanPlus,
uci::Uci,
zobrist::{Zobrist128, Zobrist16, Zobrist32, Zobrist64, Zobrist8, ZobristHash},
CastlingMode, Chess, Color, EnPassantMode, Move, Outcome, Position, Rank, Role,
};
use shakmaty_syzygy::{AmbiguousWdl, Material, MaybeRounded, Tablebase};
#[cfg(feature = "i18n")]
#[derive(RustEmbed)]
#[folder = "i18n/mo"]
struct Translations;
use jja::tr;
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]
struct QuoteIndex(usize);
#[derive(Clone, Debug)]
struct QuoteIndexValueParser;
impl clap::builder::ValueParserFactory for QuoteIndex {
type Parser = QuoteIndexValueParser;
fn value_parser() -> Self::Parser {
QuoteIndexValueParser
}
}
impl clap::builder::TypedValueParser for QuoteIndexValueParser {
type Value = QuoteIndex;
fn parse_ref(
&self,
cmd: &clap::Command,
_arg: Option<&clap::Arg>,
value: &OsStr,
) -> Result<Self::Value, clap::Error> {
let value_str = value.to_str().ok_or_else(|| {
let mut cmd = cmd.clone();
cmd.error(
clap::error::ErrorKind::InvalidValue,
tr!("Unable to parse quote index"),
)
})?;
if value_str.chars().next().map_or(false, |c| c.is_numeric()) {
match value_str.parse::<usize>() {
Ok(idx) => {
if idx >= QUOTE_MAX {
let mut cmd = cmd.clone();
Err(cmd.error(
clap::error::ErrorKind::InvalidValue,
tr!(
"Invalid value '{}' for quote index: '{} is not in 0..{}'",
idx,
idx,
QUOTE_MAX
),
))
} else {
Ok(QuoteIndex(idx))
}
}
Err(err) => {
let mut cmd = cmd.clone();
Err(cmd.error(
clap::error::ErrorKind::InvalidValue,
tr!("Unable to parse quote index: {}", err),
))
}
}
} else {
match search_quote(value_str) {
Ok(Some(idx)) => Ok(QuoteIndex(idx)),
Ok(None) => {
let mut cmd = cmd.clone();
Err(cmd.error(
clap::error::ErrorKind::InvalidValue,
tr!(
"Unable to find quote with the given regular expression `{}'",
value_str
),
))
}
Err(err) => {
let mut cmd = cmd.clone();
Err(cmd.error(
clap::error::ErrorKind::InvalidValue,
tr!("Invalid regular expression `{}': {}", value_str, err),
))
}
}
}
}
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]
struct HumanBytes(usize);
#[derive(Clone, Debug)]
struct HumanBytesValueParser;
impl clap::builder::ValueParserFactory for HumanBytes {
type Parser = HumanBytesValueParser;
fn value_parser() -> Self::Parser {
HumanBytesValueParser
}
}
impl clap::builder::TypedValueParser for HumanBytesValueParser {
type Value = HumanBytes;
fn parse_ref(
&self,
cmd: &clap::Command,
_arg: Option<&clap::Arg>,
value: &OsStr,
) -> Result<Self::Value, clap::Error> {
let value_str = value.to_str().ok_or_else(|| {
let mut cmd = cmd.clone();
cmd.error(
clap::error::ErrorKind::InvalidValue,
tr!("Unable to parse formatted bytes"),
)
})?;
match bytefmt::parse(value_str).and_then(|value| {
value
.try_into()
.map_err(|_| "u64 -> usize conversion error")
}) {
Ok(value) => Ok(HumanBytes(value)),
Err(err) => {
let mut cmd = cmd.clone();
Err(cmd.error(
clap::error::ErrorKind::InvalidValue,
tr!("Unable to parse formatted bytes `{}': {}", value_str, err),
))
}
}
}
}
#[allow(clippy::cognitive_complexity)]
fn main() -> Result<()> {
setup_panic!();
#[cfg(unix)]
unsafe {
nix::sys::signal::signal(
nix::sys::signal::Signal::SIGPIPE,
nix::sys::signal::SigHandler::SigDfl,
)
}
.context(tr!("Failed to set SIGPIPE signal handler to default."))?;
jja::stockfish::zobrist::init();
#[cfg(feature = "i18n")]
{
let translations = Translations {};
let language_loader = gettext_language_loader!();
let requested_languages = DesktopLanguageRequester::requested_languages();
let _ = i18n_embed::select(&language_loader, &translations, &requested_languages);
}
let version_s = style(built_info::GIT_VERSION.unwrap_or(built_info::PKG_VERSION))
.bold()
.magenta()
.to_string();
let version_l = format!(
"{} by {}\nFeatures: {}\nGit: {}\nHost: {}\nTarget: {}\nRustc: {}",
version_s,
style(built_info::PKG_AUTHORS).bold().red(),
style(built_info::FEATURES_LOWERCASE_STR).bold().yellow(),
style(built_info::GIT_COMMIT_HASH.unwrap_or("?"))
.bold()
.green(),
style(built_info::HOST).bold().blue(),
style(built_info::TARGET).bold().blue(),
style(built_info::RUSTC_VERSION).bold().cyan(),
);
let version_s = Box::leak(version_s.into_boxed_str());
let version_l = Box::leak(version_l.into_boxed_str());
let mut cmd = Command::new(built_info::PKG_NAME)
.about(tr!("jja: swiss army knife for chess file formats"))
.author(built_info::PKG_AUTHORS)
.version(&*version_s)
.long_version(&*version_l)
.help_template(r#"
{before-help}{name} {version}
{about}
Copyright (c) 2023 {author}
SPDX-License-Identifier: GPL-3.0-or-later
{usage-heading} {usage}
{all-args}{after-help}
"#)
.arg_required_else_help(true)
.help_expected(true)
.next_line_help(false)
.infer_long_args(true)
.infer_subcommands(true)
.propagate_version(true)
.subcommand_required(true)
.max_term_width(80)
.arg(
Arg::new("porcelain")
.long("porcelain")
.help(tr!("Give the output in an easy-to-parse format for scripts"))
.global(true)
.num_args(0..=1)
.require_equals(true)
.value_name("FORMAT")
.value_parser(["auto", "csv"])
.default_value("auto")
.default_missing_value("csv"),
)
.arg(
Arg::new("verbose")
.short('v')
.long("verbose")
.global(true)
.env("JJA_VERBOSITY")
.action(clap::ArgAction::Count)
.help(tr!("Run verbosely, may be specified multiple times"))
)
.arg(
Arg::new("threads")
.short('T')
.long("threads")
.env("JJA_NPROC")
.help(tr!("Specify maximum number of threads"))
.global(true)
.default_value(NPROC_STR.as_str())
.value_parser(clap::value_parser!(u64).range(0..=512))
)
.subcommand(
Command::new("info")
.about(tr!("Show brief information about the given file"))
.arg(
Arg::new("input")
.required(true)
.help("Path to the chess file")
.value_name("input-file")
.num_args(1),
),
)
.subcommand(
Command::new("dump")
.about(tr!("Dump all entries in an opening book"))
.arg(
Arg::new("dump-input")
.required(true)
.num_args(1)
.value_name("input-file")
.help(tr!("Path to the chess file")),
)
.arg(
Arg::new("dump-format")
.short('f')
.long("format")
.default_value("json")
.value_name("format")
.value_parser(["binary", "csv", "json"])
.help(tr!("Choose dump format (for PGN only)"))
)
.arg(
Arg::new("dump-elements")
.short('e')
.long("elements")
.action(ArgAction::Append)
.default_values(["id", "pos"])
.value_parser(["id", "sf", "p", "pos", "position", "z8", "z16", "z32", "z64", "z64sf", "zobrist8", "zobrist16", "zobrist32", "zobrist64", "zobrist64sf"])
.help(tr!("Choose dump elements (for PGN only)"))
)
)
.subcommand(
Command::new("restore")
.about(tr!("Restore entries from a previous dump into a new opening book"))
.arg(
Arg::new("restore-output")
.required(true)
.num_args(1)
.value_name("output-file")
.help(tr!("Path to the chess file")),
)
.arg(
Arg::new("restore-json")
.short('j')
.long("json")
.action(ArgAction::SetTrue)
.help(tr!("Restore from Lichess evaluations export")),
)
)
.subcommand(
Command::new("edit")
.about(tr!("Edit the given chess file"))
.arg(
Arg::new("edit-input")
.required(true)
.num_args(1)
.value_name("input-file")
.help(tr!("Path to the input chess file")),
)
.arg(
Arg::new("edit-output")
.short('o')
.long("output")
.required(true)
.value_name("output-file")
.help(tr!("Path to the output chess file")),
)
.arg(
Arg::new("edit-fen")
.short('f')
.long("fen")
.num_args(1)
.value_name("EPD")
.help(tr!("Query the given position in EPD format")),
)
.arg(
Arg::new("edit-pgn")
.short('p')
.long("pgn")
.num_args(1)
.value_name("PGN")
.help(tr!("Query the leaf node of the given PGN line")),
)
.arg(
Arg::new("edit-in-place")
.short('i')
.long("in-place")
.required(true)
.require_equals(true)
.conflicts_with("edit-output")
.num_args(0..=1)
.value_name("SUFFIX")
.help(tr!("edit files in place (makes backup if SUFFIX supplied)")),
)
.arg(
Arg::new("edit-preserve-null")
.short('0')
.long("null")
.action(ArgAction::SetTrue)
.help(tr!("Preserve moves with zero weights in the output book")),
)
.arg(
Arg::new("edit-learn")
.long("no-learn")
.action(ArgAction::SetFalse)
.help(tr!("Avoid preserving CTG nags and ABK priorities in the learn field")),
)
.arg(
Arg::new("edit-scale")
.long("no-scale")
.action(ArgAction::SetFalse)
.help(tr!("Avoid scaling weights globally to fit into 16 bits")),
)
.arg(
Arg::new("edit-rescale")
.long("rescale")
.action(ArgAction::SetTrue)
.help(tr!("Rescale weights of all entries in the PolyGlot book"))
.conflicts_with("edit-scale") )
.arg(
Arg::new("edit-look-ahead")
.long("look-ahead")
.default_value("0")
.value_parser(clap::value_parser!(usize))
.help(tr!("Look ahead this number of plies on book lookup misses during PGN generation (useful for books generated with --only-black|white)"))
)
.arg(
Arg::new("edit-max-ply")
.long("max-ply")
.default_value("1024")
.value_parser(clap::value_parser!(usize))
.help(tr!("Specify the maximum ply limit during PGN generation"))
)
.arg(
Arg::new("edit-no-colors")
.short('C')
.long("no-colors")
.action(ArgAction::SetTrue)
.help(tr!("Avoid assigning weights/priorities based on CTG move colors"))
)
.arg(
Arg::new("edit-no-nags")
.short('N')
.long("no-nags")
.action(ArgAction::SetTrue)
.help(tr!("Avoid assigning weights/priorities based on CTG move NAGs"))
)
.arg(
Arg::new("edit-color-weight-green")
.long("color-weight-green")
.default_value("65520")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for green (recommended) moves.")),
)
.arg(
Arg::new("edit-color-weight-blue")
.long("color-weight-blue")
.default_value("65280")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for blue (neutral) moves.")),
)
.arg(
Arg::new("edit-color-weight-red")
.long("color-weight-red")
.default_value("1")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for red (unrecommended) moves.")),
)
.arg(
Arg::new("edit-nag-weight-good")
.long("nag-weight-good")
.default_value("65280")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for good moves (aka \"!\" or $1)")),
)
.arg(
Arg::new("edit-nag-weight-mistake")
.long("nag-weight-mistake")
.default_value("500")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for bad moves (aka \"?\" or $2)")),
)
.arg(
Arg::new("edit-nag-weight-hard")
.long("nag-weight-hard")
.default_value("65520")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for hard moves (aka \"!!\" or $3)")),
)
.arg(
Arg::new("edit-nag-weight-blunder")
.long("nag-weight-blunder")
.default_value("1")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for very bad moves (aka \"??\" or $4)")),
)
.arg(
Arg::new("edit-nag-weight-interesting")
.long("nag-weight-interesting")
.default_value("61440")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for interesting moves (aka \"!?\" or $5)")),
)
.arg(
Arg::new("edit-nag-weight-dubious")
.long("nag-weight-dubious")
.default_value("1000")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for dubious moves (aka \"?!\" or $6)")),
)
.arg(
Arg::new("edit-nag-weight-forced")
.long("nag-weight-forced")
.default_value("65520")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for forced moves (aka \"â–¡\" or $7)")),
)
.arg(
Arg::new("edit-nag-weight-only")
.long("nag-weight-only")
.default_value("65520")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u16::MAX)))
.help(tr!("Specify weight for only moves (aka \"â–¡\" or $8)")),
)
.arg(
Arg::new("edit-color-priority-green")
.long("color-priority-green")
.default_value("9")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for green (recommended) moves.")),
)
.arg(
Arg::new("edit-color-priority-blue")
.long("color-priority-blue")
.default_value("5")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for blue (neutral) moves.")),
)
.arg(
Arg::new("edit-color-priority-red")
.long("color-priority-red")
.default_value("1")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for red (unrecommended) moves.")),
)
.arg(
Arg::new("edit-nag-priority-good")
.long("nag-priority-good")
.default_value("8")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for good moves (aka \"!\" or $1)")),
)
.arg(
Arg::new("edit-nag-priority-mistake")
.long("nag-priority-mistake")
.default_value("2")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for bad moves (aka \"?\" or $2)")),
)
.arg(
Arg::new("edit-nag-priority-hard")
.long("nag-priority-hard")
.default_value("9")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for hard moves (aka \"!!\" or $3)")),
)
.arg(
Arg::new("edit-nag-priority-blunder")
.long("nag-priority-blunder")
.default_value("1")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for very bad moves (aka \"??\" or $4)")),
)
.arg(
Arg::new("edit-nag-priority-interesting")
.long("nag-priority-interesting")
.default_value("7")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for interesting moves (aka \"!?\" or $5)")),
)
.arg(
Arg::new("edit-nag-priority-dubious")
.long("nag-priority-dubious")
.default_value("5")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for dubious moves (aka \"?!\" or $6)")),
)
.arg(
Arg::new("edit-nag-priority-forced")
.long("nag-priority-forced")
.default_value("9")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for forced moves (aka \"â–¡\" or $7)")),
)
.arg(
Arg::new("edit-nag-priority-only")
.long("nag-priority-only")
.default_value("9")
.value_parser(clap::value_parser!(u64).range(0..=u64::from(u8::MAX)))
.help(tr!("Specify priority for only moves (aka \"â–¡\" or $8)")),
)
.arg(
Arg::new("edit-author")
.long("author")
.num_args(1)
.help(tr!("Specify book author for ABK books")),
)
.arg(
Arg::new("edit-comment")
.long("comment")
.num_args(1)
.help(tr!("Specify book comment for ABK books")),
)
.arg(
Arg::new("edit-probability-priority")
.long("probability-priority")
.value_parser(clap::value_parser!(u32).range(0..=i64::from(u32::MAX)))
.help(tr!("Specify the priority probability used for move selection for ABK books"))
)
.arg(
Arg::new("edit-probability-games")
.long("probability-games")
.value_parser(clap::value_parser!(u32).range(0..=i64::from(u32::MAX)))
.help(tr!("Specify the number of games probability used for move selection for ABK books"))
)
.arg(
Arg::new("edit-probability-win-percent")
.long("probability-win-percent")
.value_parser(clap::value_parser!(u32).range(0..=i64::from(u32::MAX)))
.help(tr!("Specify the win percentage probability used for move selection for ABK books"))
)
.arg(
Arg::new("edit-event")
.long("event")
.value_name("pgn-event")
.help(tr!("Specify the Event tag for PGN output, defaults to book name"))
)
.arg(
Arg::new("edit-site")
.long("site")
.value_name("pgn-site")
.help(tr!("Specify the Site tag for PGN output, defaults to host name"))
)
.arg(
Arg::new("edit-date")
.long("date")
.value_name("pgn-date")
.help(tr!("Specify the Date tag for PGN output, defaults to last modification date"))
)
.arg(
Arg::new("edit-black")
.long("black")
.default_value("?")
.value_name("pgn-black")
.help(tr!("Specify the Black tag for PGN output"))
)
.arg(
Arg::new("edit-white")
.long("white")
.default_value("?")
.value_name("pgn-white")
.help(tr!("Specify the White tag for PGN output"))
)
.arg(
Arg::new("edit-result")
.long("result")
.default_value("*")
.value_name("pgn-result")
.value_parser(["*", "1/2-1/2", "0-1", "1-0"])
.help(tr!("Specify the Result tag for PGN output"))
)
)
.subcommand(
Command::new("find")
.about(tr!("Query the given file"))
.arg(
Arg::new("find-input")
.required(true)
.num_args(1)
.value_name("input-file")
.help(tr!("Path to the chess file")),
)
.arg(
Arg::new("find-fen")
.short('f')
.long("fen")
.num_args(1)
.value_name("EPD")
.help(tr!("Query the given position in EPD format")),
)
.arg(
Arg::new("find-pgn")
.short('p')
.long("pgn")
.num_args(1)
.value_name("PGN")
.help(tr!("Query the leaf node of the given PGN line")),
)
.arg(
Arg::new("find-hash")
.short('z')
.long("hash")
.num_args(1)
.conflicts_with("find-fen")
.conflicts_with("find-pgn")
.conflicts_with("find-line")
.conflicts_with("find-tree")
.value_name("ZOBRIST-HASH")
.value_parser(clap::value_parser!(u64))
.help(tr!("Query the given position using its Zobrist hash \
(supported for bin, and exp books only)"))
)
.arg(
Arg::new("find-line")
.short('l')
.long("line")
.conflicts_with("find-tree")
.num_args(0..=1)
.value_name("max-ply")
.value_parser(clap::value_parser!(u64).range(1..=1024))
.help(tr!("Display the variations as a table of opening lines"))
)
.arg(
Arg::new("find-tree")
.short('t')
.long("tree")
.conflicts_with("find-line")
.num_args(0..=1)
.value_name("max-ply")
.value_parser(clap::value_parser!(u64).range(1..=1024))
.help(tr!("Display the variations as an opening tree"))
)
)
.subcommand(
Command::new("make")
.about(tr!("Compile PGN files into an opening book"))
.after_help(tr!("## FILTER EXPRESSIONS
The filter expression string should contain filter conditions, which consist of
a tag name, a comparison operator, and a value. The following operators are supported:
- '>' (greater than)
- '>=' (greater than or equal to)
- '<' (less than)
- '<=' (less than or equal to)
- '=' (equal to)
- '!=' (not equal to)
- '=~' (regex match, case insensitive)
- '!~' (negated regex match, case insensitive)
Filter conditions can be combined using the following logical operators:
- 'AND' (logical AND)
- 'OR' (logical OR)
Example:
--filter=\"Event =~ World AND White =~ Carlsen AND ( Result = 1-0 OR ECO = B33 )\"
Supported tags are Event, Site, Date, UTCDate, Round, Black, White, Result,
BlackElo, WhiteElo, BlackRatingDiff, WhiteRatingDiff, BlackTitle, WhiteTitle,
ECO, Opening, TimeControl, Termination, TotalPlyCount, and ScidFlags.
In addition to these are four special variables, namely, Player, Elo, Title, and
RatingDiff. These variables may be used to match the relevant header from either
one of the sides. E.g the filter:
--filter=\"Player =~ Carlsen\"
is functionally equivalent to
--filter=\"( White =~ Carlsen OR Black =~ Carlsen )\"
Note: The filtering is designed to be simple and fast. The tokens, including
parantheses are split by whitespace. Quoting values is not allowed. For more
sophisticated filtering needs, use pgn-extract.
## Scid Flags
Scid uses one character flags, DWBMENPTKQ!?U123456, for each field where:
D - Deleted
W - White opening
B - Black opening
M - Middlegame
E - Endgame
N - Novelty
P - Pawn structure
T - Tactics
K - Kingside play
Q - Queenside play
! - Brilliancy
? - Blunder
U - User-defined
1..6 - Custom flags
It is ill-advised to rely on the order of the characters flags.
Use a regex match if/when you can.
"))
.arg(
Arg::new("make-pgn")
.num_args(1..)
.required(true)
.value_name("input-pgn")
.help(tr!("Input PGN file, .pgn or compressed .pgn.{bz2,gz,lz4,zst}")),
)
.arg(
Arg::new("make-output")
.short('o')
.long("output")
.required(true)
.value_name("output-file")
.help(tr!("Path to the output chess opening book file (.bin)")),
)
.arg(
Arg::new("make-filter")
.short('f')
.long("filter")
.value_name("expression")
.num_args(1)
.help(tr!("Filter games by combining tag criteria with operators"))
)
.arg(
Arg::new("make-max-ply")
.short('M')
.long("max-ply")
.num_args(1)
.default_value("1024")
.value_parser(clap::value_parser!(u64).range(1..=u64::MAX))
.value_name("ply")
.help(tr!("Specify the maximum ply-depth of lines included in the book")),
)
.arg(
Arg::new("make-min-games")
.short('m')
.long("min-games")
.num_args(1)
.default_value("3")
.value_parser(clap::value_parser!(u64).range(0..=u64::MAX))
.value_name("games")
.help(tr!("Specifies the minimum number of games that have to contain \
this move for it to be included in the book")),
)
.arg(
Arg::new("make-min-score")
.short('s')
.long("min-score")
.num_args(1)
.allow_hyphen_values(true)
.default_value("0.0")
.value_parser(clap::value_parser!(f64))
.value_name("score")
.help(tr!("Specifies the minimum score (or weight) this move should have \
received for it to be included in the book. The score is 2*(wins)+(draws), \
globally scaled to fit into 16 bits"))
)
.arg(
Arg::new("make-min-wins")
.long("min-wins")
.num_args(1)
.default_value("0")
.value_parser(clap::value_parser!(u64))
.value_name("wins")
.help(tr!("Specifies the minimum number of wins this move should have won for it to be included in the book"))
)
.arg(
Arg::new("make-min-pieces")
.short('p')
.long("min-pieces")
.num_args(1)
.default_value("8")
.value_parser(clap::value_parser!(u64).range(0..=32))
.value_name("piece-count")
.help(tr!("Specifies minimum number of pieces for a position to be include in the book"))
)
.arg(
Arg::new("make-only-white")
.short('w')
.long("only-white")
.conflicts_with("make-only-black")
.action(ArgAction::SetTrue)
.help(tr!("Include only moves for white in the book")),
)
.arg(
Arg::new("make-only-black")
.short('b')
.long("only-black")
.conflicts_with("make-only-white")
.action(ArgAction::SetTrue)
.help(tr!("Include only moves for black in the book")),
)
.arg(
Arg::new("make-hashcode")
.short('H')
.long("hashcode")
.action(ArgAction::SetTrue)
.help(tr!("Honour HashCode tag to skip duplicate games"))
)
.arg(
Arg::new("make-uniform")
.short('u')
.long("uniform")
.action(ArgAction::SetTrue)
.help(tr!("Set all weights to 1. In other words, all moves will be selected with equal probability")),
)
.arg(
Arg::new("make-preserve-null")
.short('0')
.long("null")
.action(ArgAction::SetTrue)
.help(tr!("Preserve moves with zero weights in the output book")),
)
.arg(
Arg::new("make-scale")
.long("no-scale")
.action(ArgAction::SetFalse)
.help(tr!("Avoid scaling weights globally to fit into 16 bits")),
)
.arg(
Arg::new("make-win-factor")
.long("win-factor")
.allow_hyphen_values(true)
.default_value("2.0")
.value_parser(clap::value_parser!(f64))
.help(tr!("Win factor in score calculation"))
)
.arg(
Arg::new("make-draw-factor")
.long("draw-factor")
.allow_hyphen_values(true)
.default_value("1.0")
.value_parser(clap::value_parser!(f64))
.help(tr!("Draw factor in score calculation"))
)
.arg(
Arg::new("make-loss-factor")
.long("loss-factor")
.allow_hyphen_values(true)
.default_value("0.0")
.value_parser(clap::value_parser!(f64))
.help(tr!("Loss factor in score calculation"))
)
.arg(
Arg::new("make-batch-size")
.short('B')
.long("batch-size")
.default_value("96000")
.value_name("games")
.value_parser(clap::value_parser!(u64).range(1..=u64::MAX))
.help(tr!("Write batch size per thread in number of games"))
)
.arg(
Arg::new("make-compression")
.short('C')
.long("compression")
.value_parser(["none", "bzip2", "lz4", "lz4hc", "snappy", "zlib", "zstd"])
.default_value("lz4")
.value_name("type")
.env("JJA_COMPRESSION_TYPE")
.help(tr!("Choose compression type for the temporary database")),
)
.arg(
Arg::new("make-compression-level")
.short('L')
.long("compression-level")
.default_value("4")
.value_name("level")
.value_parser(clap::value_parser!(u64).range(1..=u64::MAX))
.env("JJA_COMPRESSION_LEVEL")
.help(tr!("The compression factor to use with compression"))
)
.arg(
Arg::new("make-max-open-files")
.short('O')
.long("max-open-files")
.allow_hyphen_values(true)
.default_value("1024")
.value_name("level")
.value_parser(clap::value_parser!(i32).range(-1i64..=i32::MAX as i64))
.help(tr!("Sets the number of open files per thread that can be used by the temporary database. \
Value -1 means files opened are always kept open \
(may result in too many files errors)."))
)
.arg(
Arg::new("make-read-ahead")
.short('R')
.long("read-ahead")
.num_args(1)
.env("JJA_READ_AHEAD")
.default_value("4 MB")
.value_name("size")
.value_parser(clap::value_parser!(HumanBytes))
.help(tr!("Specify read ahead size for the temporary database, 0 to disable.")),
)
.arg(
Arg::new("make-sync")
.short('S')
.long("sync")
.action(ArgAction::SetTrue)
.env("JJA_SYNC")
.help(tr!("Use synchronous I/O during temporary database scans.")),
)
)
.subcommand(
Command::new("merge")
.about(tr!("Merge two opening books"))
.after_help(tr!("## MERGE STRATEGIES
Merge subcommand offers various merge strategies to customize how the
weight of equivalent move entries in both books are merged together.
Below you may find brief descriptions of different merge strategies
available, for more information read the code documentation of
`jja::merge::MergeStrategy':
* avg
Calculates the average weight of the moves in both books.
* lavg
Merges using a logarithmic averaging approach. Given that
logarithmic functions compress large values and expand small values,
we can use them to get a merge strategy that's sensitive to
differences in smaller weights while being more resistant to
disparities in larger weights.
* pavg (default)
Merges by computing the weighted average of percentage weights,
taking into account the total number of entries in each book. This
approach gives higher importance to moves from larger books versus
smaller ones, ensuring that the resultant weights reflect the
relative contributions of each book based on its size.
* wavg
Calculates the weighted average weight of the moves in both books,
considering the given weights for each book. This strategy requires
the user to specify respective book weights using -w, --weight1, and
-W, --weight2 command line options.
* wdist
Merges based on the weighted distance each weight has to its maximum
value.
The `WeightedDistance` strategy works by first normalizing the
weights to a range between 0 and 1. For each weight, it calculates
the \"distance\" from the maximum possible value in the normalized
scale (which is 1.0). The rationale behind this approach is to give
more influence to the weight that is closer to its maximum
potential, thus the one with a smaller distance to 1.0.
The resultant merged weight is a blend of the two original weights,
with the weight closer to its maximum having a slightly greater
influence. This strategy ensures that both weights are considered,
but it gives a nod to the more dominant weight.
* wmedian
Merges using a Weighted Median approach.
The Weighted Median Merge strategy is inspired by the statistical
concept of a median, which is the value separating the higher half
from the lower half of a data set. In the realm of this strategy,
the weights are treated as a data set of two values.
The principle behind this approach is to give prominence to the
weight that lies closer to the median of the set. This ensures that
if one weight is substantially larger or smaller than the other, its
influence on the merged result will be correspondingly more
significant.
The steps involved are:
1. Sort the weights.
2. Calculate the median of the set.
3. Determine the distance of each weight from the median.
4. Compute the weighted median by considering each weight's distance
from the median as its weight.
The result is a merged weight that respects the relative importance
of each original weight based on its proximity to the median. This
strategy is particularly effective when the weights have significant
disparity, ensuring that the merged weight reflects the relative
importance of the higher or more significant weight in the set.
* sum
Calculates the sum of the weights of the moves in both books.
* max
Takes the maximum weight of the moves in both books.
* min
Takes the minimum weight of the moves in both books.
* ours
Prioritizes the moves of the first book, ignoring moves from the
second book.
* dynmid
Merges weights based on a dynamic midpoint determined by their
difference.
The `DynamicMidpoint` strategy calculates the difference between the
two weights and defines a dynamic factor based on this difference.
The merged weight is determined as a weighted position between the
two original weights, leaning closer to the larger weight the more
distinct they are.
* entropy
This strategy merges using an entropy-based approach. The Entropy
Merge strategy is inspired by the concept of entropy from
Information Theory. Entropy, in this context, represents the amount
of uncertainty or randomness in data.
When merging two weights, each weight is first normalized to a
probability value, ranging from 0 to 1, by dividing the weight by
the maximum possible value (u16::MAX). The entropy for each
probability is then computed. The formula for entropy of a
probability p is given by:
H(p) = -p log2(p)
If p is either 0 or 1, the entropy is 0, indicating no uncertainty.
The final merged weight is derived by averaging the entropies of the
two weights and then scaling the result back to the range of u16.
This method ensures that the resultant merged weight encapsulates
the combined uncertainty or randomness from both original weights,
providing a unique and mathematically rigorous way to combine
values.
* geometric
Geometric scaling strategy focuses on multiplying numbers together
rather than adding, which can help in equalizing disparities.
* harmonic
Calculates the harmonic mean weight of the moves in both books. This
approach tends to favor more balanced weights and is less influenced
by extreme values.
* quadratic
Merges using the Quadratic Mean (Root Mean Square) approach. The
Quadratic Mean (also known as the Root Mean Square) is a statistical
measure of the magnitude of a set of numbers. It offers a more
balanced view, especially when dealing with numbers of varying
magnitudes.
* sigmoid
Remap the weights in a non-linear fashion using the sigmoid
function. The idea here is to diminish the influence of extreme
values, which might be causing the dissatisfaction in previous
strategies.
* sort
Calculates weight based on relative position in sorted move entries.
"))
.arg(
Arg::new("merge-input1")
.short('1')
.long("input1")
.num_args(1)
.required(true)
.value_name("input-file-1")
.help(tr!("First opening book")),
)
.arg(
Arg::new("merge-input2")
.short('2')
.long("input2")
.num_args(1)
.required(true)
.value_name("input-file-2")
.help(tr!("Second opening book")),
)
.arg(
Arg::new("merge-output")
.short('o')
.long("output")
.required(true)
.value_name("output-file")
.help(tr!("Path to the output chess file")),
)
.arg(
Arg::new("merge-strategy")
.short('s')
.long("strategy")
.default_value("pavg")
.value_parser(["avg", "lavg", "pavg", "wavg",
"wdist", "wmedian", "sum", "max", "min",
"ours", "dynmid", "entropy", "geometric",
"harmonic", "quadratic", "sigmoid", "sort"])
.help(tr!("Merge strategy"))
)
.arg(
Arg::new("merge-book1-weight")
.short('w')
.long("weight1")
.value_parser(clap::value_parser!(f64))
.help(tr!("Book 1 weight for merge strategy weighted average, aka `wavg'"))
)
.arg(
Arg::new("merge-book2-weight")
.short('W')
.long("weight2")
.value_parser(clap::value_parser!(f64))
.help(tr!("Book 2 weight for merge strategy weighted average, aka `wavg'"))
)
.arg(
Arg::new("merge-cutoff")
.short('c')
.long("cutoff")
.value_parser(clap::value_parser!(u64).range(0..=u16::MAX as u64))
.help(tr!("Moves less than this weight/depth will not be included in the book"))
)
.arg(
Arg::new("merge-outlier-threshold")
.short('O')
.long("outlier-threshold")
.default_value("0")
.value_parser(clap::value_parser!(u64).range(0..=u16::MAX as u64))
.help(tr!("Same moves with weights higher than this threshold are filtered out, use 0 to disable."))
)
.arg(
Arg::new("merge-rescale")
.long("rescale")
.action(ArgAction::SetTrue)
.help(tr!("Rescale weights of merged entries in the output opening book (Polyglot only)"))
)
)
.subcommand(
Command::new("match")
.about(tr!("Arrange book matches using random playouts"))
.arg(
Arg::new("match-book1")
.short('1')
.long("book1")
.num_args(1)
.required(true)
.help(tr!("First opening book")),
)
.arg(
Arg::new("match-book2")
.short('2')
.long("book2")
.num_args(1)
.required(true)
.help(tr!("Second opening book")),
)
.arg(
Arg::new("match-count")
.short('c')
.long("count")
.default_value("0")
.value_parser(clap::value_parser!(u64).range(0..=u64::MAX))
.help(tr!("Count of playouts, 0 to play until interrupted")),
)
.arg(
Arg::new("match-fen")
.short('f')
.long("fen")
.num_args(1)
.value_name("EPD")
.help(tr!("Start from this position in EPD format")),
)
.arg(
Arg::new("match-pgn")
.short('p')
.long("pgn")
.num_args(1)
.value_name("PGN")
.help(tr!("Start from the leaf node of the given PGN line")),
)
.arg(
Arg::new("match-chess960")
.short('9')
.long("chess960")
.action(ArgAction::SetTrue)
.help(tr!("Activate Chess960 (aka \"Fischer Random\") mode")),
)
.arg(
Arg::new("match-move-selection")
.short('m')
.long("move-selection")
.num_args(1)
.value_name("ALGO")
.default_value("weighted_random")
.value_parser(["best_move", "uniform_random", "weighted_random"])
.help(tr!("Specify book move selection algoritm")),
)
.arg(
Arg::new("match-irreversible")
.short('i')
.long("irreversible")
.action(ArgAction::SetTrue)
.help(tr!("Prefer irreversible moves during random move selection"))
),
)
.subcommand(
Command::new("play")
.about(tr!("Make random playouts, optionally using books"))
.arg(
Arg::new("play-count")
.short('c')
.long("count")
.default_value("0")
.value_parser(clap::value_parser!(u64).range(0..=u64::MAX))
.help(tr!("Count of playouts, 0 to play until interrupted")),
)
.arg(
Arg::new("play-fen")
.short('f')
.long("fen")
.num_args(1)
.value_name("EPD")
.help(tr!("Start from this position in EPD format")),
)
.arg(
Arg::new("play-pgn")
.short('p')
.long("pgn")
.num_args(1)
.value_name("PGN")
.help(tr!("Start from the leaf node of the given PGN line")),
)
.arg(
Arg::new("play-chess960")
.short('9')
.long("chess960")
.action(ArgAction::SetTrue)
.help(tr!("Activate Chess960 (aka \"Fischer Random\") mode")),
)
.arg(
Arg::new("play-book")
.short('b')
.long("book")
.num_args(1)
.conflicts_with("play-move")
.help(tr!("Specify an opening book to pick the candidates moves")),
)
.arg(
Arg::new("play-move")
.short('m')
.long("move")
.value_parser(clap::value_parser!(ChessMove))
.action(ArgAction::Append)
.help(
tr!("Specify candidate move in UCI format, may be specified multiple times")
),
),
)
.subcommand(
Command::new("digest")
.alias("hash")
.about(tr!("Calculate Zobrist of the given chess position"))
.arg(
Arg::new("hash-stockfish")
.short('S')
.long("stockfish")
.action(ArgAction::SetTrue)
.conflicts_with("hash-bits")
.conflicts_with("hash-chess960")
.conflicts_with("hash-enpassant-mode")
.help(tr!("Calculate Stockfish compatible Zobrist hash"))
)
.arg(
Arg::new("hash-fen")
.short('f')
.long("fen")
.num_args(1)
.value_name("EPD")
.help(tr!("Calculate hash for the given position in EPD format")),
)
.arg(
Arg::new("hash-pgn")
.short('p')
.long("pgn")
.num_args(1)
.value_name("PGN")
.help(tr!("Calculate hash for the leaf node of the given PGN line")),
)
.arg(
Arg::new("hash-bits")
.short('b')
.long("bits")
.default_value("64")
.value_parser(["8", "16", "32", "64", "128"])
.help(tr!("Number of bits in the Zobrist hash"))
)
.arg(
Arg::new("hash-chess960")
.short('9')
.long("chess960")
.action(ArgAction::SetTrue)
.help(tr!("Activate Chess960 (aka \"Fischer Random\") mode")),
)
.arg(
Arg::new("hash-enpassant-mode")
.short('e')
.long("enpassant-mode")
.value_parser(["legal", "pseudolegal", "always"])
.default_value("pseudolegal")
.help(tr!("Set when to include the en passant square in hash calculation"))
)
.arg(
Arg::new("hash-signed")
.short('s')
.long("signed")
.action(ArgAction::SetTrue)
.conflicts_with("hash-hex")
.help(tr!("Print hash as a signed decimal number"))
)
.arg(
Arg::new("hash-hex")
.short('x')
.long("hex")
.action(ArgAction::SetTrue)
.conflicts_with("hash-signed")
.help(tr!("Print hash as hexadecimal, rather than decimal"))
)
.arg(
Arg::new("hash-benchmark")
.short('B')
.long("benchmark")
.action(ArgAction::SetTrue)
.help(tr!("Benchmark different chess hash implementations"))
)
.arg(
Arg::new("hash-benchmark-iterations")
.short('I')
.long("benchmark-iterations")
.default_value("1000")
.value_parser(clap::value_parser!(u64).range(1..=u64::from(u32::MAX)))
.help(tr!("Number of iterations to use during benchmarking"))
)
)
.subcommand(
Command::new("perft").about(tr!("Count legal move paths of a given length"))
.arg(
Arg::new("perft-depth")
.required(true)
.num_args(1)
.value_name("DEPTH")
.value_parser(clap::value_parser!(u64).range(1..=u64::from(u32::MAX)))
.help(tr!("Specify perft depth")),
)
.arg(
Arg::new("perft-fen")
.short('f')
.long("fen")
.num_args(1)
.value_name("EPD")
.help(tr!("Calculate perft for the given position in EPD format")),
)
.arg(
Arg::new("perft-pgn")
.short('p')
.long("pgn")
.num_args(1)
.value_name("PGN")
.help(tr!("Calculate perft for the leaf node of the given PGN line")),
)
)
.subcommand(
Command::new("probe").about(tr!("Probe Syzygy tablebases"))
.after_help(tr!("
This subcommand is a stand-alone Syzygy tablebase probe tool. The subcommand
takes as input a FEN string representation of a chess position and outputs a
PGN representation of the probe result.
In addition to the standard fields, the output PGN represents the
following information:
- Result: \"1-0\" (white wins), \"1/2-1/2\" (draw), or \"0-1\" (black wins)
- The Win-Draw-Loss (WDL) value for the next move: \"Win\", \"Draw\",
\"Loss\", \"CursedWin\" (win but 50-move draw) or \"BlessedLoss\" (loss
but 50-move draw)
- The Distance-To-Zero (DTZ) value (in plys) for the next move
The PGN contains a pseudo \"principle variation\" of Syzygy vs. Syzygy for the
input position. Each PV move is rational with respect to preserving the WDL
value. The PV does not represent the shortest mate nor the most natural human
moves.
This subcommand is based on the awesome Fathom tool which is:
Copyright (c) 2013-2018 Ronald de Man
Copyright (c) 2015 basil00
Copyright (c) 2016-2020 by Jon Dart
"))
.arg(
Arg::new("probe-fen")
.required(true)
.num_args(1)
.value_name("FEN")
.help(tr!("Probe the given position in FEN format")),
)
.arg(
Arg::new("probe-path")
.short('p')
.long("path")
.required(true)
.env("SYZYGY_PATH")
.value_name("SYZYGY-PATH")
.value_parser(PathBufValueParser::new())
.action(ArgAction::Append)
.help(tr!("Specify path to Syzygy tablebases, may be specified multiple times"))
)
.arg(
Arg::new("probe-header")
.long("no-header")
.action(ArgAction::SetFalse)
.help(tr!("Avoid printing PGN headers")),
)
.arg(
Arg::new("probe-test")
.short('t')
.long("test")
.action(ArgAction::SetTrue)
.help(tr!("Print the results only, useful for scripts"))
)
.arg(
Arg::new("probe-fast")
.short('f')
.long("fast")
.action(ArgAction::SetTrue)
.help(tr!("When used with --test, report WDL without disambiguation"))
)
)
.subcommand(
Command::new("open").about(tr!("Browse ECO classification")).arg(
Arg::new("open-eco")
.short('e')
.long("eco")
.required(true)
.num_args(1)
.value_name("eco-code")
.help(tr!("Specify (partial) ECO code")),
),
)
.subcommand(
Command::new("quote")
.about(tr!("Print a chess quote"))
.arg(
Arg::new("quote-index")
.num_args(1)
.value_name("index|pattern")
.value_parser(clap::value_parser!(QuoteIndex))
.help(tr!("Specify the index of the quote to display, range: 0 ≤ idx < {}, or a search term as a case-insensitive regular expression", QUOTE_MAX))
)
);
let matches = cmd.get_matches_mut();
let verbose = matches.get_count("verbose");
let nthreads = matches.get_one::<u64>("threads").expect("threads");
NPROC.store(*nthreads as usize, Ordering::SeqCst);
let porcelain = matches.get_one::<String>("porcelain").expect("porcelain");
let porcelain = if porcelain == "auto" {
!*STDOUT_IS_TTY
} else {
true
};
if !stderr().is_terminal() {
DISPLAY_PROGRESS.store(false, Ordering::SeqCst);
}
let mut epd: Option<String> = None;
let (subcommand, submatches) = matches.subcommand().expect("missing subcommand");
if let "edit" | "find" | "digest" | "perft" | "match" | "play" = subcommand {
let (fen_id, pgn_id) = match subcommand {
"edit" => ("edit-fen", "edit-pgn"),
"find" => ("find-fen", "find-pgn"),
"digest" => ("hash-fen", "hash-pgn"),
"perft" => ("perft-fen", "perft-pgn"),
"match" => ("match-fen", "match-pgn"),
_ => ("play-fen", "play-pgn"), };
let has_fen = submatches.contains_id(fen_id);
let has_pgn = submatches.contains_id(pgn_id);
epd = Some(if has_fen {
submatches
.get_one::<String>(fen_id)
.expect("missing FEN")
.split_whitespace()
.take(4)
.collect::<Vec<&str>>()
.join(" ")
} else if has_pgn {
pgn::pgn2epd(submatches.get_one::<String>(pgn_id).expect("missing PGN"))
} else {
String::from(ROOT)
});
}
match subcommand {
"quote" => {
let idx = if submatches.contains_id("quote-index") {
Some(
submatches
.get_one::<QuoteIndex>("quote-index")
.expect("quote index")
.0,
)
} else {
None
};
print_quote(idx, porcelain);
return Ok(());
}
"digest" => {
let chess960 = submatches.get_flag("hash-chess960");
let stockfish = submatches.get_flag("hash-stockfish");
let hex = submatches.get_flag("hash-hex");
let signed = submatches.get_flag("hash-signed");
let bits = submatches
.get_one::<String>("hash-bits")
.expect("hash bits")
.parse::<u8>()
.expect("hash bits convert");
let enpassant_mode = match submatches
.get_one::<String>("hash-enpassant-mode")
.expect("enpassant mode")
.as_str()
{
"legal" => EnPassantMode::Legal,
"pseudolegal" => EnPassantMode::PseudoLegal,
"always" => EnPassantMode::Always,
_ => unreachable!(),
};
let benchmark = submatches.get_flag("hash-benchmark");
let mut iteration = *submatches
.get_one::<u64>("hash-benchmark-iterations")
.expect("benchmark iterations") as u32;
if !benchmark {
iteration = 0;
}
command_digest(
porcelain,
stockfish,
hex,
signed,
bits,
enpassant_mode,
chess960,
iteration,
&epd.unwrap(),
)
.context(tr!("digest subcommand failed"))?;
}
"perft" => {
let depth = *submatches
.get_one::<u64>("perft-depth")
.expect("perft depth") as u32;
command_perft(porcelain, depth, &epd.unwrap())?;
}
"info" => {
let file = submatches.get_one::<String>("input").expect("input file");
let path = Path::new(file);
command_info(porcelain, path)
.with_context(|| tr!("info subcommand failed for file `{}'.", path.display()))?;
}
"probe" => {
let fen = submatches.get_one::<String>("probe-fen").expect("FEN");
let header = submatches.get_flag("probe-header");
let test = submatches.get_flag("probe-test");
let fast = submatches.get_flag("probe-fast");
let mut tablebase = Tablebase::new();
for path in submatches
.get_many::<PathBuf>("probe-path")
.unwrap_or_default()
{
tablebase.add_directory(path).with_context(|| {
tr!("Failed to add tablebase directory `{}'.", path.display())
})?;
}
command_probe(porcelain, fen, &tablebase, header, test, fast)
.with_context(|| tr!("probe subcommand failed for FEN `{}'.", fen))?;
}
"open" => {
let eco = submatches.get_one::<String>("open-eco").expect("ECO code");
command_open(porcelain, eco)
.with_context(|| tr!("open subcommand failed for ECO `{}'.", eco))?;
}
"dump" => {
let file = submatches
.get_one::<String>("dump-input")
.expect("input file");
let path = Path::new(file);
let format = match submatches
.get_one::<String>("dump-format")
.expect("dump format")
.as_str()
{
"binary" => OutputFormat::Binary,
"csv" => OutputFormat::Csv,
"json" => OutputFormat::Json,
_ => unreachable!(),
};
let elements = submatches
.get_many::<String>("dump-elements")
.expect("dump elements")
.map(|elem| elem.parse::<DumpElement>().expect("dump element"))
.collect();
command_dump(porcelain, path, format, &elements).with_context(|| {
tr!(
"Failed to dump file `{}' with format `{}'.",
path.display(),
format
)
})?;
}
"restore" => {
let file = submatches
.get_one::<String>("restore-output")
.expect("output file");
let path = Path::new(file);
let json = submatches.get_flag("restore-json");
command_restore(porcelain, path, json)
.with_context(|| tr!("Failed to restore file `{}'.", path.display()))?;
}
"edit" => {
let file = submatches
.get_one::<String>("edit-input")
.expect("input file");
let input = Path::new(file);
let in_place = submatches.contains_id("edit-in-place");
let suffix = if in_place {
match submatches.try_get_one::<String>("edit-in-place")? {
None => Some("".to_string()),
suffix => suffix.cloned(),
}
} else {
None
};
let output = if in_place {
None
} else {
Some(Path::new(
submatches
.get_one::<String>("edit-output")
.expect("output file"),
))
};
let edit_learn = submatches.get_flag("edit-learn");
let scale_weights = submatches.get_flag("edit-scale");
let rescale_weights = submatches.get_flag("edit-rescale");
let preserve_null = submatches.get_flag("edit-preserve-null");
let look_ahead = submatches
.get_one::<usize>("edit-look-ahead")
.expect("look ahead");
let max_ply = submatches
.get_one::<usize>("edit-max-ply")
.expect("max ply");
let book_author = submatches
.contains_id("edit-author")
.then(|| submatches.get_one::<String>("edit-author"))
.flatten();
let book_comment = submatches
.contains_id("edit-comment")
.then(|| submatches.get_one::<String>("edit-comment"))
.flatten();
let probability_priority = submatches
.contains_id("edit-probability-priority")
.then(|| submatches.get_one::<u32>("edit-probability-priority"))
.flatten();
let probability_games = submatches
.contains_id("edit-probability-games")
.then(|| submatches.get_one::<u32>("edit-probability-games"))
.flatten();
let probability_win_percent = submatches
.contains_id("edit-probability-win-percent")
.then(|| submatches.get_one::<u32>("edit-probability-win-percent"))
.flatten();
let no_colors = submatches.get_flag("edit-no-colors");
let no_nags = submatches.get_flag("edit-no-nags");
let color_priorities = ColorPriority {
green: *submatches
.get_one::<u64>("edit-color-priority-green")
.expect("color priority green") as u8,
blue: *submatches
.get_one::<u64>("edit-color-priority-blue")
.expect("color priority blue") as u8,
red: *submatches
.get_one::<u64>("edit-color-priority-red")
.expect("color priority red") as u8,
};
let nag_priorities = NagPriority {
good: *submatches
.get_one::<u64>("edit-nag-priority-good")
.expect("nag priority good") as u8,
mistake: *submatches
.get_one::<u64>("edit-nag-priority-mistake")
.expect("nag priority mistake") as u8,
hard: *submatches
.get_one::<u64>("edit-nag-priority-hard")
.expect("nag priority hard") as u8,
blunder: *submatches
.get_one::<u64>("edit-nag-priority-blunder")
.expect("nag priority blunder") as u8,
interesting: *submatches
.get_one::<u64>("edit-nag-priority-interesting")
.expect("nag priority interesting") as u8,
dubious: *submatches
.get_one::<u64>("edit-nag-priority-dubious")
.expect("nag priority dubious") as u8,
forced: *submatches
.get_one::<u64>("edit-nag-priority-forced")
.expect("nag priority forced") as u8,
only: *submatches
.get_one::<u64>("edit-nag-priority-only")
.expect("nag priority only") as u8,
};
let color_weights = ColorWeight {
green: *submatches
.get_one::<u64>("edit-color-weight-green")
.expect("color weight green") as u16,
blue: *submatches
.get_one::<u64>("edit-color-weight-blue")
.expect("color weight blue") as u16,
red: *submatches
.get_one::<u64>("edit-color-weight-red")
.expect("color weight red") as u16,
};
let nag_weights = NagWeight {
good: *submatches
.get_one::<u64>("edit-nag-weight-good")
.expect("nag weight good") as u16,
mistake: *submatches
.get_one::<u64>("edit-nag-weight-mistake")
.expect("nag weight mistake") as u16,
hard: *submatches
.get_one::<u64>("edit-nag-weight-hard")
.expect("nag weight hard") as u16,
blunder: *submatches
.get_one::<u64>("edit-nag-weight-blunder")
.expect("nag weight blunder") as u16,
interesting: *submatches
.get_one::<u64>("edit-nag-weight-interesting")
.expect("nag weight interesting") as u16,
dubious: *submatches
.get_one::<u64>("edit-nag-weight-dubious")
.expect("nag weight dubious") as u16,
forced: *submatches
.get_one::<u64>("edit-nag-weight-forced")
.expect("nag weight forced") as u16,
only: *submatches
.get_one::<u64>("edit-nag-weight-only")
.expect("nag weight only") as u16,
};
let pgn_result = submatches.get_one::<String>("edit-result").expect("result");
let pgn_black = submatches.get_one::<String>("edit-black").expect("black");
let pgn_white = submatches.get_one::<String>("edit-white").expect("white");
let pgn_event = if submatches.contains_id("edit-event") {
Some(
submatches
.get_one::<String>("edit-event")
.expect("event")
.as_str(),
)
} else {
None
};
let pgn_site = if submatches.contains_id("edit-site") {
Some(
submatches
.get_one::<String>("edit-site")
.expect("site")
.as_str(),
)
} else {
None
};
let pgn_date = if submatches.contains_id("edit-date") {
Some(
submatches
.get_one::<String>("edit-date")
.expect("date")
.as_str(),
)
} else {
None
};
command_edit(
porcelain,
preserve_null,
edit_learn,
scale_weights,
rescale_weights,
*look_ahead,
*max_ply,
no_colors,
no_nags,
color_priorities,
nag_priorities,
color_weights,
nag_weights,
(
probability_priority,
probability_games,
probability_win_percent,
),
book_author.cloned(),
book_comment.cloned(),
(
pgn_event, pgn_site, pgn_date, pgn_white, pgn_black, pgn_result,
),
suffix,
input,
output,
&epd.unwrap(),
)
.with_context(|| tr!("Editing file `{}' failed.", input.display()))?;
}
"find" => {
let file = submatches
.get_one::<String>("find-input")
.expect("input file");
let path = Path::new(file);
let hash = submatches.get_one::<u64>("find-hash");
let line_ply = if submatches.contains_id("find-line") {
Some(
if let Ok(Some(value)) = submatches.try_get_one::<u64>("find-line") {
*value.min(&1024) as u16
} else {
6
},
)
} else {
None
};
let tree_ply = if submatches.contains_id("find-tree") {
Some(
if let Ok(Some(value)) = submatches.try_get_one::<u64>("find-tree") {
*value.min(&1024) as u16
} else {
6
},
)
} else {
None
};
command_find(
porcelain,
path,
&epd.unwrap(),
hash.copied(),
line_ply,
tree_ply,
)
.with_context(|| tr!("Finding in file `{}' failed.", path.display()))?;
}
"make" => {
let input_files = submatches
.get_many::<String>("make-pgn")
.unwrap_or_default()
.collect::<Vec<_>>();
let input_files: Vec<String> = input_files.into_iter().map(|s| s.to_owned()).collect();
let output_file = Path::new(
submatches
.get_one::<String>("make-output")
.expect("output file"),
);
let max_ply = submatches.get_one::<u64>("make-max-ply").expect("max ply");
let min_games = submatches
.get_one::<u64>("make-min-games")
.expect("min games");
let min_wins = submatches
.get_one::<u64>("make-min-wins")
.expect("min wins");
let min_score = submatches
.get_one::<f64>("make-min-score")
.expect("min score");
let min_pieces = submatches
.get_one::<u64>("make-min-pieces")
.expect("min pieces");
let win_factor = submatches
.get_one::<f64>("make-win-factor")
.expect("win factor");
let draw_factor = submatches
.get_one::<f64>("make-draw-factor")
.expect("draw factor");
let loss_factor = submatches
.get_one::<f64>("make-loss-factor")
.expect("loss factor");
let only_black = submatches.get_flag("make-only-black");
let only_white = submatches.get_flag("make-only-white");
let hashcode = submatches.get_flag("make-hashcode");
let uniform = submatches.get_flag("make-uniform");
let preserve_null = submatches.get_flag("make-preserve-null");
let scale_weights = submatches.get_flag("make-scale");
let sync_io = submatches.get_flag("make-sync");
let compression_which = match submatches
.get_one::<String>("make-compression")
.expect("compression type")
.as_str()
{
"none" => rocksdb::DBCompressionType::None,
"bzip2" => rocksdb::DBCompressionType::Bz2,
"lz4" => rocksdb::DBCompressionType::Lz4,
"lz4hc" => rocksdb::DBCompressionType::Lz4hc,
"snappy" => rocksdb::DBCompressionType::Snappy,
"zlib" => rocksdb::DBCompressionType::Zlib,
"zstd" => rocksdb::DBCompressionType::Zstd,
_ => panic!("{}", tr!("unexpected value for compression type")),
};
let compression_level = submatches
.get_one::<u64>("make-compression-level")
.expect("compression level");
let max_open_files = submatches
.get_one::<i32>("make-max-open-files")
.expect("max open files");
let batch_size = submatches
.get_one::<u64>("make-batch-size")
.expect("batch size");
let read_ahead = submatches
.get_one::<HumanBytes>("make-read-ahead")
.expect("read ahead")
.0;
let filter_side = if only_white {
Some(Color::White)
} else if only_black {
Some(Color::Black)
} else {
None
};
let parsed_filter_expr = match submatches.try_get_one::<String>("make-filter")? {
Some(filter_expr) => match parse_filter_expression(filter_expr) {
Ok(expr) => Some(expr),
Err(e) => {
bail!("{}", tr!("Error parsing filter expression: {}", e));
}
},
None => None,
};
command_make(
porcelain,
verbose,
*nthreads as usize,
*batch_size as usize,
compression_which,
*compression_level as i32,
*max_open_files,
read_ahead,
input_files.as_slice(),
output_file,
*max_ply,
*min_games,
*min_wins,
*min_score,
*min_pieces as usize,
filter_side,
hashcode,
uniform,
preserve_null,
scale_weights,
sync_io,
(*win_factor, *draw_factor, *loss_factor),
parsed_filter_expr,
)
.context(tr!("Failed to make opening book."))?;
}
"merge" => {
let input_file1 = Path::new(
submatches
.get_one::<String>("merge-input1")
.expect("input file 1"),
);
let input_file2 = Path::new(
submatches
.get_one::<String>("merge-input2")
.expect("input file 2"),
);
let output_file = Path::new(
submatches
.get_one::<String>("merge-output")
.expect("output file"),
);
let strategy = match submatches
.get_one::<String>("merge-strategy")
.expect("merge strategy")
.as_str()
{
"avg" => MergeStrategy::AvgWeight,
"max" => MergeStrategy::MaxWeight,
"min" => MergeStrategy::MinWeight,
"sigmoid" => MergeStrategy::Sigmoid,
"sort" => MergeStrategy::Sort,
"sum" => MergeStrategy::SumWeight,
"ours" => MergeStrategy::Ours,
"dynmid" => MergeStrategy::DynamicMidpoint,
"entropy" => MergeStrategy::Entropy,
"geometric" => MergeStrategy::GeometricScaling,
"harmonic" => MergeStrategy::HarmonicMean,
"quadratic" => MergeStrategy::QuadraticMean,
"wdist" => MergeStrategy::WeightedDistance,
"wmedian" => MergeStrategy::WeightedMedian,
"lavg" => MergeStrategy::LogarithmicAverage,
"pavg" => MergeStrategy::PercentageAverage,
"wavg" => {
if !submatches.contains_id("merge-book1-weight")
|| !submatches.contains_id("merge-book2-weight")
{
cmd.error(clap::error::ErrorKind::InvalidValue,
tr!("Weighted average, aka `wavg', merge strategy requires --weight1, and --weight2")).exit();
}
let weight1 = submatches
.get_one::<f64>("merge-book1-weight")
.expect("merge book1 weight");
let weight2 = submatches
.get_one::<f64>("merge-book1-weight")
.expect("merge book2 weight");
MergeStrategy::WeightedAverageWeight(*weight1, *weight2)
}
_ => unreachable!("{}", tr!("Invalid merge strategy")),
};
let cutoff = if submatches.contains_id("merge-cutoff") {
Some(
*submatches
.get_one::<u64>("merge-cutoff")
.expect("merge cutoff") as u16,
)
} else {
None
};
let outlier_threshold = if submatches.contains_id("merge-outlier-threshold") {
let val = *submatches
.get_one::<u64>("merge-outlier-threshold")
.expect("merge outlier threshold") as u16;
if val == 0 {
None
} else {
Some(val)
}
} else {
None
};
let rescale_weights = submatches.get_flag("merge-rescale");
command_merge(
porcelain,
input_file1,
input_file2,
output_file,
strategy,
cutoff,
outlier_threshold,
rescale_weights,
)
.with_context(|| {
tr!(
"Merging file `{}' into file `{}' failed.",
input_file1.display(),
input_file2.display()
)
})?;
}
"match" => {
let epd = epd.unwrap();
let count = submatches
.get_one::<u64>("match-count")
.expect("match count");
let chess960 = submatches.get_flag("match-chess960");
let prefer_irreversible = submatches.get_flag("match-irreversible");
let move_selection = match submatches
.get_one::<String>("match-move-selection")
.expect("move selection")
.to_lowercase()
.as_str()
{
"best_move" => MoveSelection::BestMove,
"uniform_random" => MoveSelection::UniformRandom,
"weighted_random" => MoveSelection::WeightedRandom,
_ => unreachable!("{}", tr!("Invalid move selection algorithm")),
};
let file1 = submatches
.get_one::<String>("match-book1")
.expect("match book 1");
let file2 = submatches
.get_one::<String>("match-book2")
.expect("match book 2");
let path1_ext = Path::new(&file1)
.extension()
.and_then(|s| s.to_str())
.map(|s| s.to_ascii_lowercase());
let path2_ext = Path::new(&file2)
.extension()
.and_then(|s| s.to_str())
.map(|s| s.to_ascii_lowercase());
let temp_dir =
tempfile::tempdir().context(tr!("Failed to create temporary directory."))?;
let mut file1_str = file1.to_string();
if path1_ext.as_deref() != Some("bin") {
let temp_file_path1 = temp_dir.path().join("book1.bin");
{
let args1 = ["edit", file1, "-o", temp_file_path1.to_str().unwrap()];
let cmd_builder1 = jja_call(&args1);
let status1 = cmd_builder1()
.status()
.with_context(|| tr!("jja-edit exited abnormally for file `{}'.", file1))?;
if !status1.success() {
bail!(
"{}",
tr!("Error converting the first book to Polyglot format.")
);
}
}
file1_str = temp_file_path1.to_str().unwrap().to_string();
}
let mut file2_str = file2.to_string();
if path2_ext.as_deref() != Some("bin") {
let temp_file_path2 = temp_dir.path().join("book2.bin");
{
let args2 = ["edit", file2, "-o", temp_file_path2.to_str().unwrap()];
let cmd_builder2 = jja_call(&args2);
let status2 = cmd_builder2()
.status()
.with_context(|| tr!("jja-edit exited abnormally for file `{}'.", file2))?;
if !status2.success() {
bail!(
"{}",
tr!("Error converting the second book to Polyglot format.")
);
}
}
file2_str = temp_file_path2.to_str().unwrap().to_string();
}
let book1 = PolyGlotBook::open(file1_str)
.with_context(|| tr!("Failed to open PolyGlot file `{}'.", file1))?;
let book2 = PolyGlotBook::open(file2_str)
.with_context(|| tr!("Failed to open PolyGlot file `{}'.", file2))?;
command_play_book_match(
porcelain,
*count as usize,
chess960,
move_selection,
prefer_irreversible,
Arc::new(book1),
Arc::new(book2),
&epd,
*nthreads as usize,
)
.with_context(|| {
tr!(
"Failed to play match with file `{}' versus file `{}'.",
file1,
file2
)
})?;
}
"play" => {
let epd = epd.unwrap();
let count = submatches.get_one::<u64>("play-count").expect("play count");
let chess960 = submatches.get_flag("play-chess960");
let candidate_moves = if submatches.contains_id("play-move") {
Some(
submatches
.get_many::<ChessMove>("play-move")
.unwrap_or_default()
.map(|uci| uci.0.clone())
.collect::<Vec<_>>(),
)
} else if submatches.contains_id("play-book") {
let file = submatches
.get_one::<String>("play-book")
.expect("play book");
let path_ext = Path::new(file)
.extension()
.ok_or(anyhow!("{}", tr!("Path `{}' has no file extension.", file)))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has invalid UTF-8 in file extension.", file)
))?
.to_ascii_lowercase();
if path_ext == "bin" {
let db = PolyGlotBook::open(file)
.with_context(|| tr!("Failed to open PolyGlot file `{}'.", file))?;
let epd = Epd::from_ascii(epd.as_bytes())
.context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN."))?;
let mut candidate_moves = Vec::new();
if let Some(entries) = db.lookup_moves(zobrist_hash(&pos)) {
for entry in &entries {
if let Some(mv) = polyglot::to_move(&pos, entry.mov) {
candidate_moves.push(Uci::from_standard(&mv));
}
}
}
if candidate_moves.is_empty() {
bail!(tr!("Position does not exist in the book!"));
}
Some(candidate_moves)
} else if path_ext == "ctg" {
let mut db = CtgBook::open(file)
.with_context(|| tr!("Failed to open CTG file `{}'.", file))?;
let epd = Epd::from_ascii(epd.as_bytes())
.context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN."))?;
let mut candidate_moves = Vec::new();
if let Some(entries) = db.lookup_moves(&pos) {
for entry in &entries {
candidate_moves.push(entry.uci.clone());
}
}
if candidate_moves.is_empty() {
bail!(tr!("Position does not exist in the book!"));
}
Some(candidate_moves)
} else {
bail!(tr!("Unknown opening book format `{}'", path_ext));
}
} else {
None
};
command_play(porcelain, *count, chess960, candidate_moves, &epd)
.context(tr!("play subcommand failed."))?;
}
_ => unreachable!(
"{}",
tr!("Exhausted list of subcommands and subcommand_required prevents `None`")
),
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn command_digest(
porcelain: bool,
stockfish: bool,
hex: bool,
signed: bool,
bits: u8,
enpassant_mode: EnPassantMode,
chess960: bool,
benchmark_iterations: u32,
epd_str: &str,
) -> Result<()> {
let epd = Epd::from_ascii(epd_str.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let castling_mode = if chess960 {
CastlingMode::Chess960
} else {
CastlingMode::Standard
};
let pos: Chess = epd
.into_position(castling_mode)
.context(tr!("Failed to parse illegal FEN."))?;
if benchmark_iterations > 0 {
if porcelain {
println!("hash,elapsed,speed");
} else {
println!("{}", tr!("# jja Hash Function Benchmark"));
println!("{}", tr!("Position: {}", style(epd_str).bold().green()));
println!(
"{}",
tr!(
"Iterations: {}",
style(benchmark_iterations.to_string()).bold().cyan()
)
);
}
benchmarking::warm_up();
let bench_results = benchmarking::measure_function_n(6, |measurers| {
for _ in 0..benchmark_iterations {
measurers[0].measure(|| {
let _hash = stockfish_hash(&pos);
});
}
for _ in 0..benchmark_iterations {
measurers[1].measure(|| {
let _hash = zobrist8_hash(&pos);
});
}
for _ in 0..benchmark_iterations {
measurers[2].measure(|| {
let _hash = zobrist16_hash(&pos);
});
}
for _ in 0..benchmark_iterations {
measurers[3].measure(|| {
let _hash = zobrist32_hash(&pos);
});
}
for _ in 0..benchmark_iterations {
measurers[4].measure(|| {
let _hash = zobrist_hash(&pos);
});
}
for _ in 0..benchmark_iterations {
measurers[5].measure(|| {
let _hash = zobrist128_hash(&pos);
});
}
})
.context(tr!("Failed to benchmark different hash algorithms."))?;
if porcelain {
println!(
"stockfish,{:?},{:.2}",
bench_results[0].elapsed(),
bench_results[0].speed()
);
println!(
"zobrist8,{:?},{:.2}",
bench_results[1].elapsed(),
bench_results[1].speed()
);
println!(
"zobrist16,{:?},{:.2}",
bench_results[2].elapsed(),
bench_results[2].speed()
);
println!(
"zobrist32,{:?},{:.2}",
bench_results[3].elapsed(),
bench_results[3].speed()
);
println!(
"zobrist64,{:?},{:.2}",
bench_results[4].elapsed(),
bench_results[4].speed()
);
println!(
"zobrist128,{:?},{:.2}",
bench_results[5].elapsed(),
bench_results[5].speed()
);
} else {
println!(
"{}",
tr!(
"{} hash function takes {} to run with speed {}/s.",
style("Stockfish").bold().magenta(),
style(format!("{:?}", bench_results[0].elapsed()))
.bold()
.cyan(),
style(format!("{:.2}", bench_results[0].speed()))
.bold()
.green()
)
);
println!(
"{}",
tr!(
"{} hash function takes {} to run with speed {}/s.",
style("Zobrist8").bold().magenta(),
style(format!("{:?}", bench_results[1].elapsed()))
.bold()
.cyan(),
style(format!("{:.2}", bench_results[1].speed()))
.bold()
.green()
)
);
println!(
"{}",
tr!(
"{} hash function takes {} to run with speed {}/s.",
style("Zobrist16").bold().magenta(),
style(format!("{:?}", bench_results[2].elapsed()))
.bold()
.cyan(),
style(format!("{:.2}", bench_results[2].speed()))
.bold()
.green()
)
);
println!(
"{}",
tr!(
"{} hash function takes {} to run with speed {}/s.",
style("Zobrist32").bold().magenta(),
style(format!("{:?}", bench_results[3].elapsed()))
.bold()
.cyan(),
style(format!("{:.2}", bench_results[3].speed()))
.bold()
.green()
)
);
println!(
"{}",
tr!(
"{} hash function takes {} to run with speed {}/s.",
style("Zobrist64").bold().magenta(),
style(format!("{:?}", bench_results[4].elapsed()))
.bold()
.cyan(),
style(format!("{:.2}", bench_results[4].speed()))
.bold()
.green()
)
);
println!(
"{}",
tr!(
"{} hash function takes {} to run with speed {}/s.",
style("Zobrist128").bold().magenta(),
style(format!("{:?}", bench_results[5].elapsed()))
.bold()
.cyan(),
style(format!("{:.2}", bench_results[5].speed()))
.bold()
.green()
)
);
let speed0 = bench_results[0].speed();
let speed1 = bench_results[4].speed();
let spdiff = if speed0 == speed1 {
0.0
} else {
speed0.max(speed1) / speed0.min(speed1)
};
println!(
"{}",
tr!(
"{} hash function is {} times faster than {} hash function.",
style(if speed0 >= speed1 {
"Stockfish"
} else {
"Zobrist64"
})
.bold()
.green(),
style(format!("{:.2}", spdiff)).bold().cyan(),
style(if speed0 >= speed1 {
"Zobrist64"
} else {
"Stockfish"
})
.bold()
.red()
)
);
}
return Ok(());
}
if stockfish {
let hash = stockfish_hash(&pos);
if hex {
println!("{} id {:#x}", epd_str, hash);
} else if signed {
println!("{} id {}", epd_str, hash as i64);
} else {
println!("{} id {}", epd_str, hash);
}
return Ok(());
}
match bits {
8 => {
let hash = pos.zobrist_hash::<Zobrist8>(enpassant_mode).0;
if hex {
println!("{} id {:#x}", epd_str, hash)
} else if signed {
println!("{} id {}", epd_str, hash as i8)
} else {
println!("{} id {}", epd_str, hash)
}
}
16 => {
let hash = pos.zobrist_hash::<Zobrist16>(enpassant_mode).0;
if hex {
println!("{} id {:#x}", epd_str, hash)
} else if signed {
println!("{} id {}", epd_str, hash as i16)
} else {
println!("{} id {}", epd_str, hash)
}
}
32 => {
let hash = pos.zobrist_hash::<Zobrist32>(enpassant_mode).0;
if hex {
println!("{} id {:#x}", epd_str, hash)
} else if signed {
println!("{} id {}", epd_str, hash as i32)
} else {
println!("{} id {}", epd_str, hash)
}
}
64 => {
let hash = pos.zobrist_hash::<Zobrist64>(enpassant_mode).0;
if hex {
println!("{} id {:#x}", epd_str, hash)
} else if signed {
println!("{} id {}", epd_str, hash as i64)
} else {
println!("{} id {}", epd_str, hash)
}
}
128 => {
let hash = pos.zobrist_hash::<Zobrist128>(enpassant_mode).0;
if hex {
println!("{} id {:#x}", epd_str, hash)
} else if signed {
println!("{} id {}", epd_str, hash as i128)
} else {
println!("{} id {}", epd_str, hash)
}
}
_ => unreachable!("{}", tr!("invalid Zobrist hash bits `{}'", bits)),
};
Ok(())
}
fn command_perft(_porcelain: bool, depth: u32, epd_str: &str) -> Result<()> {
let epd = Epd::from_ascii(epd_str.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.into_position(CastlingMode::Chess960)
.context(tr!("Failed to parse illegal FEN."))?;
let key = zobrist_hash(&pos);
let now = Instant::now();
let acn = shakmaty::perft(&pos, depth);
let acs = now.elapsed().as_secs();
let epd = Epd::from_position(pos, EnPassantMode::PseudoLegal);
println!(
"{} id {:#x} acd {} acn {} acs {}",
epd, key, depth, acn, acs
);
Ok(())
}
fn command_info(_porcelain: bool, path: &Path) -> Result<()> {
let stat = fs::metadata(path)
.with_context(|| tr!("Failed to read metadata for file `{}'.", path.display()))?;
if !stat.is_file() {
bail!(
"{}",
tr!("Path `{}' is not a regular file.", path.display())
);
}
let size = stat.len();
let ext = path
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", path.display())
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
path.display()
)
))?
.to_ascii_lowercase();
let path = path.to_str().ok_or(anyhow!(
"{}",
tr!("Path `{}' has invalid UTF-8 in file name.", path.display())
))?;
println!("{}", tr!("# jja Chess File Information"));
println!("{}", tr!("Path: {}", path));
println!(
"{}",
tr!("Size: {} ({} bytes)", bytefmt::format(size), size)
);
println!("{}", tr!("Type: {}", ext.to_ascii_uppercase()));
let mut file = File::open(path)?;
let mut hasher = Sha256::new();
std::io::copy(&mut file, &mut hasher)?;
let checksum = hasher.finalize();
println!("{}", tr!("Sha256: {}", format!("{:x}", checksum)));
if ext == "abk" {
return abk_info(path);
} else if ext == "obk" {
return obk_info(path);
} else if ext == "bin" {
return polyglot_info(path);
} else if ext == "exp" {
return brainlearn_info(path);
} else if ext == "ctg" {
return ctg_info(path);
}
Err(anyhow!(
"{}",
tr!("Unsupported chess file extension `{}'.", ext)
))
}
fn abk_info(filename: &str) -> Result<()> {
let db =
AbkBook::open(filename).with_context(|| tr!("Failed to open ABK file `{}'.", filename))?;
println!("{}", tr!("## jja ABK File Information"));
println!("{}", tr!("Number of entries: {}", db.total_entries()));
println!(
"{}",
tr!("Size of a single entry: {} bytes", ABK_ENTRY_LENGTH)
);
println!("{}", tr!("### jja ABK File Header"));
println!("{}", tr!("Author: \"{}\"", db.author));
println!("{}", tr!("Comment: \"{}\"", db.comment));
println!("{}", tr!("Book Depth: {}", db.book_depth));
println!("{}", tr!("Book Moves: {}", db.book_moves));
println!("{}", tr!("Minimum number of games: {}", db.min_games));
println!("{}", tr!("Minimum number of wins: {}", db.min_wins));
println!(
"{}",
tr!("Win percentage for white: {}", db.win_percent_white)
);
println!(
"{}",
tr!("Win percentage for black: {}", db.win_percent_black)
);
println!(
"{}",
tr!("Priority probability: {}", db.probability_priority)
);
println!(
"{}",
tr!(
"Number of games for probability move selection: {}",
db.probability_games
)
);
println!(
"{}",
tr!(
"Win percentage for probability move selection: {}",
db.probability_win_percent
)
);
println!(
"{}",
tr!("Maximum half move depth: {}", db.use_book_half_move)
);
Ok(())
}
fn obk_info(filename: &str) -> Result<()> {
let db: ObkBook =
ObkBook::open(filename).with_context(|| tr!("Failed to open OBK file `{}'.", filename))?;
println!("{}", tr!("## jja OBK File Information"));
println!("{}", tr!("Version: {}", db.version));
println!("{}", tr!("Move count: {}", db.move_count));
if let Some(text_notes_count) = db.text_notes_count {
println!("{}", tr!("Text notes count: {}", text_notes_count));
} else {
println!("{}", tr!("Text notes count: N/A"));
}
Ok(())
}
fn ctg_info(filename: &str) -> Result<()> {
let mut db =
CtgBook::open(filename).with_context(|| tr!("Failed to open CTG file `{}'.", filename))?;
println!("{}", tr!("## jja CTG File Information"));
println!("{}", tr!("Total pages: {}", db.total_pages()));
let num = db.total_positions()? as u64;
println!("{}", tr!("Number of positions: {}", num));
Ok(())
}
fn brainlearn_info(filename: &str) -> Result<()> {
let db = BrainLearnFile::open(filename)
.with_context(|| tr!("Failed to open BrainLearn file `{}'.", filename))?;
println!("{}", tr!("## jja BrainLearn File Information"));
println!("{}", tr!("Number of entries: {}", db.num_entries));
println!(
"{}",
tr!("Size of a single entry: {} bytes", EXPERIENCE_ENTRY_SIZE)
);
Ok(())
}
fn polyglot_info(filename: &str) -> Result<()> {
let db = PolyGlotBook::open(filename)
.with_context(|| tr!("Failed to open PolyGlot file `{}'.", filename))?;
println!("{}", tr!("## jja PolyGlot File Information"));
println!("{}", tr!("Number of entries: {}", db.num_entries));
println!(
"{}",
tr!("Size of a single entry: {} bytes", BOOK_ENTRY_SIZE)
);
Ok(())
}
fn command_probe(
_porcelain: bool,
fen: &str,
tablebase: &Tablebase<Chess>,
header: bool,
test: bool,
fast: bool,
) -> Result<()> {
let fen = Fen::from_ascii(fen.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let mut pos: Chess = fen
.clone()
.into_position(CastlingMode::Chess960)
.context(tr!("Failed to parse illegal FEN."))?;
let material = Material::from_board(pos.board());
let wdl = tablebase
.probe_wdl(&pos)
.with_context(|| tr!("Failed to probe WDL for position `{}'", fen))?;
if test && fast {
let stm = pos.turn();
println!(
"{}",
match wdl {
AmbiguousWdl::Draw | AmbiguousWdl::CursedWin | AmbiguousWdl::BlessedLoss =>
"1/2-1/2",
AmbiguousWdl::Win | AmbiguousWdl::MaybeWin => stm.fold_wb("1-0", "0-1"),
AmbiguousWdl::Loss | AmbiguousWdl::MaybeLoss => stm.fold_wb("0-1", "1-0"),
}
);
return Ok(());
}
let dtz = tablebase
.probe_dtz(&pos)
.with_context(|| tr!("Failed to probe DTZ for position `{}'", fen))?;
let pb = get_progress_spinner();
pb.println(tr!(
"Enumerating the principled variation for position `{}'",
fen
));
pb.set_message(tr!("Probing:"));
let mut movetext = Vec::new();
let mut force_movenumber = true;
let mut posmap: ZobristHashSet = HashSet::with_hasher(ZobristHasherBuilder);
loop {
if pos.is_checkmate() {
movetext.push("{ Checkmate }".to_owned());
break;
}
if pos.is_stalemate() {
movetext.push("{ Stalemate }".to_owned());
break;
}
if pos.is_insufficient_material() {
movetext.push("{ Insufficient material }".to_owned());
break;
}
if pos.is_variant_end() {
movetext.push("{ Variant end }".to_owned());
break;
}
if pos.halfmoves() == 100 {
movetext.push("{ Draw claimed }".to_owned());
force_movenumber = true;
} else if pos.halfmoves() == 0 {
movetext.push(match tablebase.probe_dtz(&pos)? {
MaybeRounded::Precise(dtz) => format!(
"{{ {} with DTZ {} }}",
Material::from_board(pos.board()),
i32::from(dtz)
),
MaybeRounded::Rounded(dtz) => format!(
"{{ {} with DTZ {} or {} }}",
Material::from_board(pos.board()),
i32::from(dtz),
i32::from(dtz.add_plies(1))
),
});
force_movenumber = true;
}
let (bestmove, dtz) = tablebase.best_move(&pos)?.expect("has moves");
pb.inc(1);
match pos.turn() {
Color::White => movetext.push(format!("{}.", pos.fullmoves())),
Color::Black if force_movenumber => movetext.push(format!("{}...", pos.fullmoves())),
_ => (),
}
movetext.push(SanPlus::from_move_and_play_unchecked(&mut pos, &bestmove).to_string());
let hash = zobrist_hash(&pos);
if !posmap.insert(hash) && dtz.is_zero() {
movetext.push("{ Tablebase draw }".to_owned());
break;
}
force_movenumber = false;
}
pb.finish_with_message(tr!("Probing done."));
let result = pos.outcome().unwrap_or(Outcome::Draw);
movetext.push(result.to_string());
if test {
println!("{result}");
} else {
if header {
println!("[Event \"{material}\"]");
println!("[Site \"\"]");
println!("[Date \"????.??.??\"]");
println!("[Round \"-\"]");
println!("[White \"Syzygy\"]");
println!("[Black \"Syzygy\"]");
println!("[Result \"{result}\"]");
println!("[FEN \"{fen}\"]");
println!(
"[Annotator \"{} v{}\"]",
jja::built_info::PKG_NAME,
jja::built_info::PKG_VERSION
);
println!("[WDL \"{wdl:?}\"]");
match dtz {
MaybeRounded::Precise(dtz) => println!("[DTZ \"{}\"]", i32::from(dtz)),
MaybeRounded::Rounded(dtz) => println!(
"[DTZ \"{} or {}\"]",
i32::from(dtz),
i32::from(dtz.add_plies(1))
),
}
println!();
}
println!("{}", movetext.join(" "));
if header {
println!();
}
}
Ok(())
}
fn command_open(porcelain: bool, eco: &str) -> Result<()> {
if porcelain {
println!("*,eco,pgn");
for item in ECO.iter().take(ECO_MAX) {
let (eco_, _, pgn) = item;
if !eco_.starts_with(eco) {
continue;
}
println!("{},{}", eco_, pgn);
}
} else {
let mut table = Table::new();
table.add_row(Row::new(vec![
Cell::new("ECO").with_style(Attr::Bold),
Cell::new("PGN").with_style(Attr::Bold),
]));
for item in ECO.iter().take(ECO_MAX) {
let (eco_, _, moves) = item;
if !eco_.starts_with(eco) {
continue;
}
let wrapped_moves = wrap_text(moves, 72)
.into_iter()
.collect::<Vec<_>>()
.join("\n");
table.add_row(Row::new(vec![Cell::new(eco_), Cell::new(&wrapped_moves)]));
}
table.printstd();
}
Ok(())
}
fn command_dump(
_porcelain: bool,
path: &Path,
format: OutputFormat,
elements: &Vec<DumpElement>,
) -> Result<()> {
let ext = path
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", path.display())
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
path.display()
)
))?
.to_ascii_lowercase();
let path = path.to_str().ok_or(anyhow!(
"{}",
tr!("Path `{}' has invalid UTF-8 in file name.", path.display())
))?;
if ext == "bin" {
return polyglot_dump(path);
} else if ext == "exp" {
return brainlearn_dump(path);
} else if ext == "pgn"
|| ext == "zst"
|| ext == "bz2"
|| ext == "xz"
|| ext == "gz"
|| ext == "lz4"
{
return pgn_dump(path, format, elements).context(tr!(
"Failed to dump PGN file `{}' with format `{}'",
path,
format
));
}
Err(anyhow!(
"{}",
tr!("Unsupported chess file extension `{}'.", ext)
))
}
fn brainlearn_dump(filename: &str) -> Result<()> {
let pb = get_progress_bar(0);
let mut db = BrainLearnFile::open(filename)
.with_context(|| tr!("Failed to open BrainLearn file `{}'.", filename))?;
db.load(None, Some(&pb));
for entries in db.data.values() {
for entry in entries {
println!("{}", entry);
}
}
Ok(())
}
fn polyglot_dump(filename: &str) -> Result<()> {
let mut db = PolyGlotBook::open(filename)
.with_context(|| tr!("Failed to open PolyGlot file `{}'.", filename))?;
for entry in &mut db {
println!("{}", entry);
}
Ok(())
}
fn command_restore(_porcelain: bool, path: &Path, json: bool) -> Result<()> {
let ext = path
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", path.display())
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
path.display()
)
))?
.to_ascii_lowercase();
let path = path.to_str().ok_or(anyhow!(
"{}",
tr!("Path `{}' has invalid UTF-8 in file name.", path.display())
))?;
if ext == "bin" {
return polyglot_restore(path);
} else if ext == "exp" {
return if json {
brainlearn_restore_json(path)
} else {
brainlearn_restore(path)
};
} else if ext == "jja-0" {
return jjadb_restore(path);
} else if ext == "epd" {
return epd_restore(path);
}
Err(anyhow!(
"{}",
tr!("Unsupported chess file extension `{}'.", ext)
))
}
fn brainlearn_restore(filename: &str) -> Result<()> {
let pb = get_progress_bar(0);
pb.println(tr!("Creating output BrainLearn experience file..."));
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(filename)
.with_context(|| {
tr!(
"Failed to create output BrainLearn experience file `{}'.",
filename
)
})?,
);
pb.println(tr!("Success creating output BrainLearn experience file."));
pb.println(tr!(
"Parsing BrainLearn file dump from standard input, and writing them to the output book..."
));
pb.set_message(tr!("Writing:"));
let mut count = 0;
let stdin = io::stdin();
for line in stdin.lock().lines() {
let line = line.context(tr!("Failed to read a line from standard input."))?;
let entry: ExperienceEntry = serde_json::from_str(&line).with_context(|| {
tr!(
"Failed to convert line to BrainLearn experience file entry: `{}'.",
line
)
})?;
exp_entry_to_file(&mut output_file, &entry).with_context(|| {
tr!(
"Failed to write to output BrainLearn experience file `{}'.",
filename
)
})?;
count += 1;
pb.inc(1);
}
pb.println(tr!(
"Success restoring {} BrainLearn experience file entries from standard input.",
count
));
Ok(())
}
fn brainlearn_restore_json(filename: &str) -> Result<()> {
let pb = get_progress_bar(0);
pb.println(tr!("Creating output BrainLearn experience file..."));
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(filename)
.with_context(|| {
tr!(
"Failed to create output BrainLearn experience file `{}'.",
filename
)
})?,
);
pb.println(tr!("Success creating output BrainLearn experience file."));
pb.println(tr!(
"Parsing Lichess evaluations export from standard input, and writing them to the output book..."
));
pb.set_message(tr!("Writing:"));
let mut line_count = 1;
let mut entry_count = 0;
let stdin = io::stdin();
for line in stdin.lock().lines() {
let line = line.context(tr!("Failed to read a line from standard input."))?;
let data: LichessEval = serde_json::from_str(&line)
.with_context(|| tr!("Failed to parse Lichess evaluation JSON: `{}'.", line))?;
let epd = match Epd::from_ascii(data.fen.as_bytes()) {
Ok(epd) => epd,
Err(_error) => {
continue;
}
};
let pos: Chess = match epd.into_position(CastlingMode::Standard) {
Ok(pos) => pos,
Err(_error) => {
continue;
}
};
let mut moves: HashMap<i32, (i32, i32)> = HashMap::new();
for entry in data.evals.iter().sorted_by(|a, b| b.depth.cmp(&a.depth)) {
for pv in &entry.pvs {
let uci = pv.line.split_whitespace().next().expect("uci move");
let mov = match Uci::from_ascii(uci.as_bytes()) {
Ok(mov) => mov,
Err(_error) => {
continue;
}
};
let mov = match mov.to_move(&pos) {
Ok(mov) => brainlearn::from_move(mov),
Err(_error) => {
continue;
}
};
if moves.contains_key(&mov) {
continue;
}
let score = match (pv.cp, pv.mate) {
(Some(cp), _) => cp,
(None, Some(mate)) => 32000 - mate,
(None, None) => panic!("no score or mate on line {line_count}."),
};
moves.insert(mov, (entry.depth, score));
}
}
let key = stockfish_hash(&pos);
for (mov, (depth, score)) in moves.into_iter() {
let entry = ExperienceEntry {
key,
depth,
score,
mov,
perf: 100,
};
exp_entry_to_file(&mut output_file, &entry).with_context(|| {
tr!(
"Failed to write to output BrainLearn experience file `{}'.",
filename
)
})?;
entry_count += 1;
pb.inc(1);
}
line_count += 1;
}
pb.println(tr!(
"Success restoring {} BrainLearn experience file entries from standard input.",
entry_count
));
Ok(())
}
fn polyglot_restore(filename: &str) -> Result<()> {
let pb = get_progress_bar(0);
pb.println(tr!("Creating output PolyGlot opening book..."));
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(filename)
.with_context(|| {
tr!(
"Failed to create output PolyGlot opening book `{}'.",
filename
)
})?,
);
pb.println(tr!("Success creating output PolyGlot opening book."));
pb.println(tr!("Reading PolyGlot dump from standard input..."));
pb.set_message(tr!("Reading:"));
let stdin = io::stdin();
let mut entries = BTreeMap::new();
let mut count = 0;
for line in stdin.lock().lines() {
let line = line.context(tr!("Failed to read a line from standard input."))?;
let entry: BookEntry = serde_json::from_str(&line).with_context(|| {
tr!(
"Failed to convert line to PolyGlot opening book entry: `{}'.",
line
)
})?;
entries
.entry(entry.key)
.or_insert_with(Vec::new)
.push(entry);
count += 1;
pb.inc(1);
}
pb.println(tr!(
"Success reading {} PolyGlot book entries from standard input into memory.",
count
));
pb.finish_and_clear();
let pb = get_progress_bar(count);
pb.println(tr!("Writing output PolyGlot opening book..."));
pb.set_message(tr!("Writing:"));
for moves in entries.values_mut() {
if moves.is_empty() {
continue;
}
moves.sort_unstable_by_key(|mov| Reverse(mov.weight));
for entry in &*moves {
bin_entry_to_file(&mut output_file, entry).with_context(|| {
tr!(
"Failed to write to output PolyGlot opening book `{}'.",
filename
)
})?;
pb.inc(1);
}
}
pb.println(tr!("Success writing output PolyGlot opening book."));
Ok(())
}
fn jjadb_restore(filename: &str) -> Result<()> {
let pb = get_progress_bar(0);
pb.println(tr!("Opening connection to output JJA database..."));
pb.set_message(tr!("Opening..."));
let mut conn = rusqlite::Connection::open(filename)
.with_context(|| tr!("Failed to open JJA database `{}'.", filename))?;
conn.set_db_config(rusqlite::config::DbConfig::SQLITE_DBCONFIG_DEFENSIVE, false)
.context(tr!("Failed to set JJA database defensive mode to false."))?;
conn.set_db_config(
rusqlite::config::DbConfig::SQLITE_DBCONFIG_TRUSTED_SCHEMA,
true,
)
.context(tr!("Failed to set JJA database trusted schema to true."))?;
let version = built_info::PKG_VERSION_MAJOR
.parse::<i64>()
.expect("die die die my darling, don't utter a single word!");
let sql = format!("PRAGMA user_version = {}", version);
conn.execute(&sql, [])
.with_context(|| tr!("Failed to set JJA database user version to `{}'.", version))?;
pb.println(tr!("Success connecting to the output JJA database."));
let pb_clone = pb.clone();
conn.progress_handler(
100,
Some(move || {
pb_clone.inc(1);
false
}),
);
pb.println(tr!("Preparing output JJA database..."));
pb.set_message(tr!("Preparing:"));
pb.set_position(0);
conn.execute_batch(
"
CREATE TABLE IF NOT EXISTS p (
id INTEGER NOT NULL,
p0 INTEGER NOT NULL,
p1 INTEGER NOT NULL,
p2 INTEGER NOT NULL,
p3 INTEGER NOT NULL,
p4 INTEGER NOT NULL,
PRIMARY KEY (p0, p1, p2, p3, p4) ON CONFLICT IGNORE) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS p_idx ON p(id);",
)
.context(tr!("Failed to create JJA database tables and indexes."))?;
pb.println(tr!("Success preparing output JJA database."));
pb.println(tr!("Counting position entries in the JJA database..."));
pb.set_message(tr!("Counting:"));
pb.set_position(0);
let count_null: i64 = conn
.query_row("SELECT COUNT(id) FROM p", (), |row| row.get(0))
.context(tr!("Failed to count JJA database position table rows."))?;
pb.println(tr!(
"Found {} position entries in the JJA database.",
count_null
));
pb.println(tr!("Preparing JJA database for batch insert..."));
pb.set_message(tr!("Preparing:"));
pb.set_position(0);
conn.execute_batch(
"
PRAGMA journal_mode = OFF;
PRAGMA synchronous = OFF;
PRAGMA cache_size = 1000000;
PRAGMA locking_mode = EXCLUSIVE;
PRAGMA temp_store = MEMORY;
PRAGMA mmap_size=999999999;",
)
.context(tr!("Failed to prepare JJA database for batch insert."))?;
pb.println(tr!("Success preparing JJA database for batch insert."));
pb.set_message(tr!("Inserting:"));
pb.set_position(0);
conn.progress_handler::<fn() -> bool>(0, None);
pb.println(tr!(
"Reading PGN dump from standard input and inserting into JJA database..."
));
pb.println(tr!(
"Use ^C to interrupt the process and commit the changes so far."
));
const BATCH_SIZE: usize = 100_000;
let mut transaction = conn
.transaction_with_behavior(rusqlite::TransactionBehavior::Exclusive)
.context(tr!(
"Failed to create exclusive transaction in JJA database."
))?;
let stdin = io::stdin();
let mut count_insert = 0;
let interrupted = Arc::new(AtomicBool::new(false));
let interrupted_clone = Arc::clone(&interrupted);
ctrlc::set_handler(move || {
interrupted_clone.store(true, Ordering::SeqCst);
})
.context(tr!("Failed to set up interrupt handler."))?;
let mut loop_interrupted = false;
for line in stdin.lock().lines() {
if interrupted.load(Ordering::SeqCst) {
pb.println(tr!(
"Interrupted by user, committing entries to the database..."
));
loop_interrupted = true;
break;
}
let line = line.context(tr!("Failed to read a line from standard input."))?;
let entry: [u64; 6] = serde_json::from_str(&line).with_context(|| tr!("Failed to convert line to a series of integers representing a chess position: `{}'.", line))?;
transaction
.execute(
"INSERT INTO p (id,p0,p1,p2,p3,p4) VALUES (?1,?2,?3,?4,?5,?6)",
(
entry[0] as i64,
entry[1] as i64,
entry[2] as i64,
entry[3] as i64,
entry[4] as i64,
entry[5] as i64,
),
)
.with_context(|| {
tr!(
"Failed to insert position entry `{}' to JJA database.",
format!("{:?}", entry)
)
})?;
if count_insert > 0 && count_insert % BATCH_SIZE == 0 {
pb.set_message(tr!("Committing batch..."));
transaction
.commit()
.context(tr!("Failed to commit transaction to JJA database."))?;
pb.set_message(tr!("Committed."));
transaction = conn
.transaction_with_behavior(rusqlite::TransactionBehavior::Exclusive)
.context(tr!(
"Failed to create exclusive transaction in JJA database."
))?;
pb.set_message(tr!("Inserting:"));
}
count_insert += 1;
pb.inc(1);
}
pb.set_message(tr!("Committing..."));
transaction
.commit()
.context(tr!("Failed to commit transaction to JJA database."))?;
pb.set_message(tr!("Committed."));
pb.println(tr!(
"Success inserting {} entries to the JJA output database.",
count_insert
));
if loop_interrupted {
return Ok(());
}
let pb_clone = pb.clone();
conn.progress_handler(
100,
Some(move || {
pb_clone.inc(1);
false
}),
);
pb.println(tr!("Counting position entries in the JJA database..."));
pb.set_message(tr!("Counting:"));
pb.set_position(0);
let count_done: i64 = conn
.query_row("SELECT COUNT(id) FROM p", (), |row| row.get(0))
.context(tr!("Failed to count JJA database position table rows."))?;
pb.println(tr!(
"Found {} position entries in the JJA database.",
count_done
));
pb.println(tr!(
"Searching for Zobrist hash collisions in the JJA database..."
));
pb.set_message(tr!("Searching:"));
pb.set_position(0);
let count_uniq: i64 = conn
.query_row("SELECT COUNT(DISTINCT id) FROM p", (), |row| row.get(0))
.context(tr!(
"Failed to count JJA database position table unique rows."
))?;
let mut retval = Ok(());
if count_done != count_uniq {
let count_coll = count_done - count_uniq;
pb.println(tr!(
"WARNING: Detected {} Zobrist hash collisions in the JJA database!",
count_coll
));
retval = Err(Box::new(HashCollision::new(count_coll as usize)));
pb.println(tr!(
"Querying for duplicate Zobrist hashes in the JJA database..."
));
pb.set_message(tr!("Querying:"));
pb.set_position(0);
let mut stmt = conn
.prepare(
"
SELECT id, COUNT(id) as count
FROM p
GROUP BY id
HAVING count > 1
",
)
.context(tr!(
"Failed to query for duplicate Zobrist hashes in the JJA database."
))?;
let rows = stmt
.query_map([], |row| {
let id: i64 = row.get(0)?;
let count: i64 = row.get(1)?;
Ok((id, count))
})
.context(tr!(
"Failed to read duplicate Zobrist hashes from the JJA database."
))?;
for row in rows {
let (id, count) = row.context(tr!("Failed to read row from the JJA database."))?;
pb.println(tr!(
"Duplicate Zobrist hash: {}, count: {}",
format!("{:#x}", id as u64),
count
));
}
} else {
pb.println(tr!(
"Detected no Zobrist hash collisions in the JJA database."
));
}
pb.println(tr!("Optimizing the JJA database..."));
pb.set_message(tr!("Optimizing..."));
pb.set_position(0);
conn.execute("PRAGMA OPTIMIZE", ())
.context(tr!("Failed to optimize the JJA database."))?;
pb.println(tr!("Success optimizing the JJA database."));
pb.set_message("All done.");
Ok(retval?)
}
fn epd_restore(filename: &str) -> anyhow::Result<()> {
let pb = get_progress_bar(0);
pb.println(tr!("Creating output EPD file..."));
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(filename)
.with_context(|| tr!("Failed to create output EPD file `{}'.", filename))?,
);
pb.println(tr!("Success creating output EPD file."));
pb.println(tr!("Reading PGN dump in CSV format from standard input..."));
pb.set_message(tr!("Reading:"));
let reader = Csv::from_reader(io::stdin().lock())
.delimiter(b',')
.has_header(false);
let mut count = 0;
for record_result in reader {
count += 1;
let record = record_result.context(tr!("Error reading CSV entry on line {}.", count))?;
let columns: Vec<&str> = record
.columns()
.context(tr!("Cannot convert CSV record to UTF-8 on line {}.", count))?
.collect();
if columns.len() != 6 {
return Err(anyhow::anyhow!(tr!(
"Error: CSV entry on line {} does not have enough columns.",
count
)));
}
let hash = columns[0].parse::<i64>().context(tr!(
"Error parsing column 1 as integer in CSV entry on line {}.",
count
))? as u64;
let mut position_arr = [0u64; 5];
for index in 1..=5 {
let item = columns[index].parse::<i64>().context(tr!(
"Error parsing column {} as integer in CSV entry on line {}.",
index + 1,
count
))? as u64;
position_arr[index - 1] = item;
}
let pos = deserialize_chess(position_arr).with_context(|| {
tr!(
"Failed to deserialize chess position array: `{}'.",
format!("{:?}", position_arr)
)
})?;
let epd = Epd::from_position(pos, EnPassantMode::PseudoLegal);
writeln!(output_file, "{} id {:#x}", epd, hash)
.context(tr!("Error writing to output EPD file."))?;
pb.inc(1);
}
pb.println(tr!("Success writing {} entries to output EPD file.", count));
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn command_edit(
porcelain: bool,
preserve_null: bool,
edit_learn: bool,
scale_weights: bool,
rescale_weights: bool,
look_ahead: usize,
max_ply: usize,
no_colors: bool,
no_nags: bool,
color_priorities: ColorPriority,
nag_priorities: NagPriority,
color_weights: ColorWeight,
nag_weights: NagWeight,
probabilities: (Option<&u32>, Option<&u32>, Option<&u32>),
author: Option<String>,
comment: Option<String>,
roster: (Option<&str>, Option<&str>, Option<&str>, &str, &str, &str),
suffix: Option<String>,
input: &Path,
output: Option<&Path>,
epd: &str,
) -> Result<()> {
let input_ext = input
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", input.display())
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
input.display()
)
))?
.to_ascii_lowercase();
let input_path = input.to_str().ok_or(anyhow!(
"{}",
tr!("Path `{}' has invalid UTF-8 in file name.", input.display())
))?;
if output.is_none() && input_ext != "abk" && input_ext != "bin" && input_ext != "exp" {
bail!(
"{}",
tr!(
"In place editing is not supported for {} books.",
input_ext.to_ascii_uppercase()
)
);
}
let output_ext;
let output_path;
let output_temp;
let output_ptmp;
if let Some(output) = output {
output_ext = output
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", output.display())
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
output.display()
)
))?;
output_path = output.to_str().ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file name.",
output.display()
)
))?;
} else {
output_temp =
tempfile::Builder::new().tempfile_in(input.parent().unwrap_or(Path::new("./")))?;
let tmp = Arc::new(Mutex::new(
output_temp.path().to_string_lossy().into_owned(),
));
let tmp_clone = Arc::clone(&tmp);
ctrlc::set_handler(move || {
let tmp = tmp_clone.lock().unwrap();
fs::remove_file(&*tmp).unwrap();
std::process::exit(1);
})
.context(tr!("Failed to set up interrupt handler."))?;
output_ptmp = tmp.lock().unwrap().to_string();
output_path = &output_ptmp;
output_ext = &input_ext;
}
match (input_ext.as_str(), output_ext) {
("bin", "bin") => bin2bin(
porcelain,
preserve_null,
scale_weights,
rescale_weights,
suffix,
input_path,
output_path,
epd,
),
("abk", "bin") => abk2bin(
porcelain,
preserve_null,
edit_learn,
scale_weights,
input_path,
output_path,
epd,
),
("abk", "abk") => abk2abk(
porcelain,
author,
comment,
probabilities,
suffix,
input_path,
output_path,
epd,
),
("ctg", "abk") => ctg2abk(
porcelain,
author,
comment,
probabilities,
no_colors,
no_nags,
color_priorities,
nag_priorities,
input_path,
output_path,
epd,
),
("obk", "bin") => obk2bin(
porcelain,
preserve_null,
edit_learn,
scale_weights,
input_path,
output_path,
epd,
),
("ctg", "bin") => ctg2bin(
porcelain,
preserve_null,
edit_learn,
scale_weights,
no_colors,
no_nags,
color_weights,
nag_weights,
input_path,
output_path,
epd,
),
("exp", "exp") => exp2exp(
porcelain,
preserve_null,
suffix,
input_path,
output_path,
epd,
),
("abk" | "bin" | "ctg" | "exp" | "obk", "pgn") => any2pgn(
porcelain,
input_ext.as_str(),
input_path,
output_path,
roster,
epd,
look_ahead,
max_ply,
),
("pgn" | "zst" | "bz2" | "xz" | "gz" | "lz4", "epd") => {
pgn2epd(porcelain, input_ext.as_str(), input_path, output_path)
}
_ => {
bail!(
"{}",
tr!(
"Unsupported chess file conversion `{}' => `{}' for edit.",
input_ext,
output_ext
)
);
}
}
}
#[allow(clippy::too_many_arguments)]
fn command_make(
porcelain: bool,
verbose: u8,
nthreads: usize,
batch_size: usize,
compression_which: rocksdb::DBCompressionType,
compression_level: i32,
max_open_files: i32,
read_ahead: usize,
input_files: &[String],
output_file: &Path,
max_ply: u64,
min_games: u64,
min_wins: u64,
min_score: f64,
min_pieces: usize,
filter_side: Option<Color>,
hashcode: bool,
uniform: bool,
preserve_null: bool,
scale_weights: bool,
sync_io: bool,
wdl_factor: (f64, f64, f64),
parsed_filter_expr: Option<Vec<FilterComponent>>,
) -> Result<()> {
let output_ext = output_file
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", output_file.display())
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
output_file.display()
)
))?
.to_ascii_lowercase();
let output_path = output_file.to_str().ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file name.",
output_file.display()
)
))?;
if output_ext == "bin" {
return polyglot_make(
porcelain,
verbose,
nthreads,
batch_size,
compression_which,
compression_level,
max_open_files,
read_ahead,
input_files,
output_path,
max_ply,
min_games,
min_wins,
min_score,
min_pieces,
filter_side,
hashcode,
uniform,
preserve_null,
scale_weights,
sync_io,
wdl_factor,
parsed_filter_expr,
);
}
let input_ext = Path::new(&input_files[0])
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", input_files[0])
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
input_files[0]
)
))?
.to_ascii_lowercase();
bail!(
"{}",
tr!(
"Unsupported make action `{}' => `{}'.",
input_ext,
output_ext
),
);
}
#[allow(clippy::too_many_arguments)]
fn command_merge(
porcelain: bool,
input1: &Path,
input2: &Path,
output: &Path,
strategy: MergeStrategy,
cutoff: Option<u16>,
outlier_threshold: Option<u16>,
rescale_weights: bool,
) -> Result<()> {
let input1_ext = input1
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", input1.display())
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
input1.display()
)
))?
.to_ascii_lowercase();
let input1_path = input1.to_str().ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file name.",
input1.display()
)
))?;
let input2_ext = input2
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", input2.display())
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
input2.display()
)
))?
.to_ascii_lowercase();
let input2_path = input2.to_str().ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file name.",
input2.display()
)
))?;
let output_ext = output
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", output.display())
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
output.display()
)
))?
.to_ascii_lowercase();
let output_path = output.to_str().ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file name.",
output.display()
)
))?;
if input1_ext == "bin" && input2_ext == "bin" && output_ext == "bin" {
return polyglot_merge(
porcelain,
input1_path,
input2_path,
output_path,
strategy,
cutoff,
outlier_threshold,
rescale_weights,
);
} else if input1_ext == "exp" && input2_ext == "exp" && output_ext == "exp" {
return brainlearn_merge(
porcelain,
input1_path,
input2_path,
output_path,
strategy,
cutoff,
);
}
bail!(
"{}",
tr!(
"Unsupported merge action `{}' + `{}' => `{}'.",
input1_ext,
input2_ext,
output_ext
)
);
}
fn command_find(
porcelain: bool,
path: &Path,
epd: &str,
hash: Option<u64>,
line_ply: Option<u16>,
tree_ply: Option<u16>,
) -> Result<()> {
let ext = path
.extension()
.ok_or(anyhow!(
"{}",
tr!("Path `{}' has no file extension.", path.display())
))?
.to_str()
.ok_or(anyhow!(
"{}",
tr!(
"Path `{}' has invalid UTF-8 in file extension.",
path.display()
)
))?
.to_ascii_lowercase();
let path = path.to_str().ok_or(anyhow!(
"{}",
tr!("Path `{}' has invalid UTF-8 in file name.", path.display())
))?;
if hash.is_some() && ext != "bin" && ext != "exp" {
bail!(
"{}",
tr!("Search by Zobrist hash is only supported for `bin', and `exp' books.")
);
}
if ext == "abk" {
return abk_find(porcelain, path, epd, line_ply, tree_ply);
} else if ext == "obk" {
return obk_find(porcelain, path, epd, line_ply, tree_ply);
} else if ext == "bin" {
return polyglot_find(porcelain, path, epd, hash, line_ply, tree_ply);
} else if ext == "exp" {
return brainlearn_find(porcelain, path, epd, hash, line_ply, tree_ply);
} else if ext == "ctg" {
return ctg_find(porcelain, path, epd, line_ply, tree_ply);
}
Err(anyhow!(
"{}",
tr!("Unsupported chess file extension `{}'.", ext)
))
}
fn abk_find(
porcelain: bool,
filename: &str,
epd: &str,
line_ply: Option<u16>,
tree_ply: Option<u16>,
) -> Result<()> {
let db: AbkBook =
AbkBook::open(filename).with_context(|| tr!("Failed to open ABK file `{}'.", filename))?;
let epd = Epd::from_ascii(epd.as_bytes())?;
let pos: Chess = epd.clone().into_position(CastlingMode::Standard)?;
if let Some(line_ply) = line_ply {
let tree = db.tree(&pos, line_ply, None);
print!("{}", lines_from_tree(&tree, line_ply));
return Ok(());
} else if let Some(tree_ply) = tree_ply {
print!("{}", db.tree(&pos, tree_ply, None));
return Ok(());
}
let moves = db.lookup_moves(&pos, None);
if let Some(mut moves) = moves {
moves.sort_unstable_by_key(|mov| Reverse((mov.priority, mov.nwon, mov.ngames)));
if porcelain {
println!("{}", tr!("*,uci,games,win,loss,priority"));
for (idx, book_move) in moves.iter().enumerate() {
let ngames = book_move.ngames;
let nwon = book_move.nwon;
let nlost = book_move.nlost;
println!(
"{},{},{},{},{},{}",
idx + 1,
Uci::from(*book_move),
ngames,
nwon,
nlost,
book_move.priority
);
}
} else {
let mut table = Table::new();
table.add_row(Row::new(vec![
Cell::new("*").with_style(Attr::Bold),
Cell::new(&tr!("UCI")).with_style(Attr::Bold),
Cell::new(&tr!("Games")).with_style(Attr::Bold),
Cell::new(&tr!("Win")).with_style(Attr::Bold),
Cell::new(&tr!("Loss")).with_style(Attr::Bold),
Cell::new(&tr!("Priority")).with_style(Attr::Bold),
]));
for (idx, book_move) in moves.iter().enumerate() {
let ngames = book_move.ngames;
let nwon = book_move.nwon;
let nlost = book_move.nlost;
table.add_row(row![
(idx + 1).to_string(),
format!("{}", Uci::from(*book_move)),
ngames.to_string(),
nwon.to_string(),
nlost.to_string(),
book_move.priority.to_string(),
]);
}
table.printstd();
}
return Ok(());
}
bail!(
"{}",
tr!("Position `{}' not found in file `{}'.", epd, filename)
);
}
fn obk_find(
porcelain: bool,
filename: &str,
epd: &str,
line_ply: Option<u16>,
tree_ply: Option<u16>,
) -> Result<()> {
let mut db: ObkBook =
ObkBook::open(filename).with_context(|| tr!("Failed to open OBK file `{}'.", filename))?;
db.load(None);
let epd = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.clone()
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN."))?;
if let Some(line_ply) = line_ply {
let tree = db.tree(&pos, line_ply);
print!("{}", lines_from_tree(&tree, line_ply));
return Ok(());
} else if let Some(tree_ply) = tree_ply {
print!("{}", db.tree(&pos, tree_ply));
return Ok(());
}
let key = zobrist_hash(&pos);
let mut entries: Vec<&ObkMoveEntry> =
db.moves.iter().filter(|entry| entry.key == key).collect();
if entries.is_empty() {
bail!(
"{}",
tr!("Position `{}' not found in file `{}'.", epd, filename)
);
}
entries.sort_by(|a, b| b.weight.cmp(&a.weight));
if porcelain {
println!("{}", tr!("*,uci,weight"));
for (idx, entry) in entries.iter().enumerate() {
let mut uci = Uci::from(**entry);
uci = match uci {
Uci::Normal {
from,
to,
promotion: _,
} => {
let piece_at_from = pos
.board()
.piece_at(from)
.expect("No piece on 'from' square");
let promotion = if piece_at_from.role == Role::Pawn
&& ((piece_at_from.color == Color::White && to.rank() == Rank::Eighth)
|| (piece_at_from.color == Color::Black && to.rank() == Rank::First))
{
Some(Role::Queen)
} else {
None
};
Uci::Normal {
from,
to,
promotion,
}
}
_ => unreachable!("{}", tr!("Unexpected null move or put move in book?")),
};
println!("{},{},{}", idx + 1, uci, entry.weight,);
}
} else {
let mut table = Table::new();
table.add_row(Row::new(vec![
Cell::new("*").with_style(Attr::Bold),
Cell::new(&tr!("UCI")).with_style(Attr::Bold),
Cell::new(&tr!("Weight")).with_style(Attr::Bold),
]));
for (idx, entry) in entries.iter().enumerate() {
let mut uci = Uci::from(**entry);
uci = match uci {
Uci::Normal {
from,
to,
promotion: _,
} => {
let piece_at_from = pos
.board()
.piece_at(from)
.expect("No piece on 'from' square");
let promotion = if piece_at_from.role == Role::Pawn
&& ((piece_at_from.color == Color::White && to.rank() == Rank::Eighth)
|| (piece_at_from.color == Color::Black && to.rank() == Rank::First))
{
Some(Role::Queen)
} else {
None
};
Uci::Normal {
from,
to,
promotion,
}
}
_ => unreachable!("{}", tr!("Unexpected null move or put move in book?")),
};
table.add_row(row![
(idx + 1).to_string(),
format!("{}", uci),
entry.weight.to_string(),
]);
}
table.printstd();
}
Ok(())
}
fn ctg_find(
porcelain: bool,
filename: &str,
epd: &str,
line_ply: Option<u16>,
tree_ply: Option<u16>,
) -> Result<()> {
let mut db =
CtgBook::open(filename).with_context(|| tr!("Failed to open CTG file `{}'.", filename))?;
let epd_obj = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd_obj
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN."))?;
if let Some(line_ply) = line_ply {
let tree = db.tree(&pos, line_ply);
print!("{}", lines_from_tree(&tree, line_ply));
return Ok(());
} else if let Some(tree_ply) = tree_ply {
print!("{}", db.tree(&pos, tree_ply));
return Ok(());
}
if let Some(mut moves) = db.lookup_moves(&pos) {
if moves.is_empty() {
bail!(
"{}",
tr!("Position `{}' is a leaf node in file `{}'.", epd, filename)
);
}
if porcelain {
println!("{}", tr!("*,uci,nag,win,draw,loss,avg,perf,recommendation"));
let mut i = 1;
sort_ctg_moves(&mut moves);
for entry in &moves {
let mut display = format!("{},{},", i, entry.uci);
if let Some(nag) = &entry.nag {
display.push_str(&nag.to_string());
}
println!(
"{},{},{},{},{:.2},{:.2},{:#02x}",
display,
entry.win,
entry.draw,
entry.loss,
entry.avg(),
entry.perf(),
entry.recommendation,
);
i += 1;
}
} else {
let mut i = 1;
let mut table = Table::new();
table.add_row(Row::new(vec![
Cell::new("*").with_style(Attr::Bold),
Cell::new(&tr!("UCI")).with_style(Attr::Bold),
Cell::new(&tr!("NAG")).with_style(Attr::Bold),
Cell::new(&tr!("Win")).with_style(Attr::Bold),
Cell::new(&tr!("Draw")).with_style(Attr::Bold),
Cell::new(&tr!("Loss")).with_style(Attr::Bold),
Cell::new(&tr!("Average")).with_style(Attr::Bold),
Cell::new(&tr!("Performance")).with_style(Attr::Bold),
Cell::new(&tr!("Recommendation")).with_style(Attr::Bold),
]));
sort_ctg_moves(&mut moves);
for entry in &moves {
let mut tblent = Vec::new();
tblent.push(Cell::new(&format!("{}", i)));
tblent.push(Cell::new(&colored_uci(
&entry.uci,
&entry.nag,
entry.recommendation,
)));
if let Some(nag) = &entry.nag {
tblent.push(Cell::new(&nag.to_string()));
} else {
tblent.push(Cell::new(""));
}
tblent.push(Cell::new(&format!("{}", entry.win)));
tblent.push(Cell::new(&format!("{}", entry.draw)));
tblent.push(Cell::new(&format!("{}", entry.loss)));
tblent.push(Cell::new(&format!("{:.2}", entry.avg())));
tblent.push(Cell::new(&format!("{:.2}", entry.perf())));
tblent.push(Cell::new(&format!("{:#02x}", entry.recommendation)));
table.add_row(Row::new(tblent));
i += 1;
}
table.printstd();
}
return Ok(());
}
bail!(
"{}",
tr!("Position `{}' not found in file `{}'.", epd, filename)
);
}
fn brainlearn_find(
porcelain: bool,
filename: &str,
epd: &str,
hash: Option<u64>,
line_ply: Option<u16>,
tree_ply: Option<u16>,
) -> Result<()> {
let pb = get_progress_bar(0);
let mut db = BrainLearnFile::open(filename)
.with_context(|| tr!("Failed to open BrainLearn file `{}'.", filename))?;
db.load(None, Some(&pb));
if let Some(hash) = hash {
if let Some(entries) = db.lookup_moves(hash) {
for entry in entries {
println!("{}", entry);
}
return Ok(());
} else {
bail!(
"{}",
tr!(
"Hash `{}' not found in file `{}'.",
format!("{:#x}", hash),
filename
)
);
}
}
let epd = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.clone()
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN."))?;
if let Some(line_ply) = line_ply {
let tree = db.tree(&pos, line_ply);
let table = lines_from_tree(&tree, line_ply);
let mut stdout = io::stdout();
return write!(&mut stdout, "{table}")
.context(tr!("Failed to write table to standard output."));
} else if let Some(tree_ply) = tree_ply {
let tree = db.tree(&pos, tree_ply);
let mut stdout = io::stdout();
return write!(&mut stdout, "{tree}")
.context(tr!("Failed to write table to standard output."));
}
if let Some(mut entries) = db.lookup_moves(stockfish_hash(&pos)) {
entries.sort_unstable_by_key(|entry| Reverse((entry.depth, entry.score, entry.perf)));
if porcelain {
print!("{}", format_exp_entries(&pos, entries));
} else {
let mut table = Table::new();
table.add_row(Row::new(vec![
Cell::new("*").with_style(Attr::Bold),
Cell::new(&tr!("UCI")).with_style(Attr::Bold),
Cell::new(&tr!("Depth")).with_style(Attr::Bold),
Cell::new(&tr!("Score")).with_style(Attr::Bold),
Cell::new(&tr!("Performance")).with_style(Attr::Bold),
]));
let mut i = 1;
for entry in &entries {
if let Some(mv) = brainlearn::to_move(&pos, entry.mov) {
let uci = Uci::from_standard(&mv);
let depth = entry.depth;
let score = entry.score;
let perf = entry.perf;
let mut tblent = Vec::new();
tblent.push(Cell::new(&format!("{}", i)));
tblent.push(Cell::new(&format!("{}", uci)));
tblent.push(Cell::new(&format!("{}", depth)));
tblent.push(Cell::new(&format!("{}", score)));
tblent.push(Cell::new(&format!("{}", perf)));
table.add_row(Row::new(tblent));
} else {
eprintln!(
"{}",
tr!(
"Failed to convert entry `{}' to move in position `{}'.",
format!("{:?}", entry),
epd
)
);
}
i += 1;
}
table.printstd();
}
return Ok(());
}
bail!(
"{}",
tr!("Position `{}' not found in file `{}'.", epd, filename)
);
}
fn polyglot_find(
porcelain: bool,
filename: &str,
epd: &str,
hash: Option<u64>,
line_ply: Option<u16>,
tree_ply: Option<u16>,
) -> Result<()> {
let db = PolyGlotBook::open(filename)
.with_context(|| tr!("Failed to open PolyGlot file `{}'.", filename))?;
if let Some(hash) = hash {
if let Some(entries) = db.lookup_moves(hash) {
for entry in entries {
println!("{}", entry);
}
return Ok(());
} else {
bail!(
"{}",
tr!(
"Hash `{}' not found in file `{}'.",
format!("{:#x}", hash),
filename
)
);
}
}
let epd = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.clone()
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN."))?;
if let Some(line_ply) = line_ply {
let tree = db.tree(&pos, line_ply);
let table = lines_from_tree(&tree, line_ply);
let mut stdout = io::stdout();
return write!(&mut stdout, "{table}")
.context(tr!("Failed to write table to standard output."));
} else if let Some(tree_ply) = tree_ply {
let tree = db.tree(&pos, tree_ply);
let mut stdout = io::stdout();
return write!(&mut stdout, "{tree}")
.context(tr!("Failed to write table to standard output."));
}
if let Some(entries) = db.lookup_moves(zobrist_hash(&pos)) {
if porcelain {
print!("{}", format_bin_entries(&pos, entries));
} else {
let mut table = Table::new();
table.add_row(Row::new(vec![
Cell::new("*").with_style(Attr::Bold),
Cell::new(&tr!("UCI")).with_style(Attr::Bold),
Cell::new(&tr!("Weight")).with_style(Attr::Bold),
Cell::new(&tr!("Learn")).with_style(Attr::Bold),
]));
let mut i = 1;
for entry in &entries {
if let Some(mv) = polyglot::to_move(&pos, entry.mov) {
let uci = Uci::from_standard(&mv);
let weight = entry.weight;
let learn = entry.learn;
let mut tblent = Vec::new();
tblent.push(Cell::new(&format!("{}", i)));
tblent.push(Cell::new(&format!("{}", uci)));
tblent.push(Cell::new(&format!("{}", weight)));
tblent.push(Cell::new(&format!("{}", learn)));
table.add_row(Row::new(tblent));
} else {
eprintln!(
"{}",
tr!(
"Failed to convert entry `{}' to move in position `{}'.",
format!("{:?}", entry),
epd
)
);
}
i += 1;
}
table.printstd();
}
return Ok(());
}
bail!(
"{}",
tr!("Position `{}' not found in file `{}'.", epd, filename)
);
}
fn abk2bin(
_porcelain: bool,
preserve_null: bool,
edit_learn: bool,
scale_weights: bool,
input_filename: &str,
output_filename: &str,
epd: &str,
) -> Result<()> {
let mut db: AbkBook = AbkBook::open(input_filename)
.with_context(|| tr!("Failed to open ABK file `{}'.", input_filename))?;
let pb = get_progress_bar(db.book_moves as u64);
pb.println(tr!("Creating output PolyGlot opening book..."));
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(output_filename)
.with_context(|| {
tr!(
"Failed to create output PolyGlot opening book `{}'.",
output_filename
)
})?,
);
pb.println(tr!("Success creating output PolyGlot opening book."));
let tree = db.traverse_book(Some(&pb));
let epd = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.clone()
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN"))?;
let book = traverse_tree(&tree, pos);
let book_len = book.len() as u64;
pb.set_position(0);
pb.set_length(book_len);
pb.println(tr!(
"Found {} chess positions in the ABK input file.",
style(book_len).bold().cyan()
));
pb.println(tr!(
"Saving entries starting from root `{}' to the output book.",
style(epd).bold().green()
));
pb.set_message(tr!("Saving:"));
for (key, moves) in book {
let max_weight = moves[0].weight;
let scale: f64 = 0xfff0 as f64 / max_weight as f64;
for entry in moves {
if !preserve_null && entry.weight == 0 {
continue;
}
let weight = if scale_weights {
(scale * entry.weight as f64) as u16
} else {
entry.weight
};
let learn = if !edit_learn { 0 } else { entry.learn };
let entry = BookEntry {
key,
weight,
learn,
mov: entry.mov,
};
bin_entry_to_file(&mut output_file, &entry).with_context(|| {
tr!(
"Failed to write to output PolyGlot opening book `{}'.",
output_filename
)
})?;
}
pb.inc(1);
}
pb.finish_with_message(tr!("Saving done."));
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn abk2abk(
_porcelain: bool,
author: Option<String>,
comment: Option<String>,
probabilities: (Option<&u32>, Option<&u32>, Option<&u32>),
suffix: Option<String>,
input_filename: &str,
output_filename: &str,
epd: &str,
) -> Result<()> {
if let Some(ref suffix) = suffix {
if !suffix.is_empty() {
let backup = Path::new(input_filename).with_extension(suffix);
std::fs::copy(input_filename, backup.as_path()).with_context(|| {
tr!(
"Failed to backup Arena opening book `{}' to `{}'.",
input_filename,
backup.display()
)
})?;
}
}
let mut db: AbkBook = AbkBook::open(input_filename)
.with_context(|| tr!("Failed to open ABK file `{}'.", input_filename))?;
let pb = get_progress_bar(db.book_moves as u64);
pb.println(tr!("Creating output Arena opening book..."));
let mut output_opts = fs::OpenOptions::new();
let mut output_file = output_opts.write(true);
if suffix.is_none() {
output_file = output_file.create_new(true);
}
let mut output_file = BufWriter::new(
output_file
.open(output_filename)
.with_context(|| tr!("Failed to open output file `{}'.", output_filename))?,
);
pb.println(tr!("Success creating output Arena opening book."));
if let Some(author) = author {
db.author = author;
};
if let Some(comment) = comment {
db.comment = comment;
};
if let Some(p) = probabilities.0 {
db.probability_priority = *p;
}
if let Some(p) = probabilities.1 {
db.probability_games = *p;
}
if let Some(p) = probabilities.2 {
db.probability_win_percent = *p;
}
pb.println(tr!(
"Searching position `{}' in the opening book.",
style(epd).bold().green()
));
pb.set_message(tr!("Searching:"));
let epd_str = epd;
let epd = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.clone()
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN."))?;
let edit_key = zobrist_hash(&pos);
let entries = match db.lookup_moves(&pos, Some(&pb)) {
None => Vec::new(),
Some(mut entries) => {
entries.sort_unstable_by_key(|mov| {
std::cmp::Reverse((mov.priority, mov.nwon, mov.ngames))
});
entries
}
};
let board_display = get_board_lines(&pos, false)
.iter()
.map(|line| format!("#{}", line))
.collect::<Vec<String>>()
.join("\n");
let mut contents = String::new();
contents.push_str(&tr!("# jja: abk edit screen\n"));
contents.push_str("# vim: set ft=conf :\n#\n");
contents.push_str(&edit_comment(pos.clone(), edit_key));
contents.push('\n');
contents.push_str(&board_display);
contents.push_str(&format!(
"\n\n{}\n\n{}\n",
format_abk_entries(entries),
*ABK_EDIT_COMMENT
));
let contents = edit_tempfile(Some(contents), Some(".csv".to_string()))
.context(tr!("Failed to edit temporary file."))?;
let contents = contents.as_bytes();
let contents: Vec<u8> = contents
.lines()
.filter_map(|line| {
let line = line.ok()?;
if !line.is_empty() && !line.starts_with('#') {
let mut line = line.into_bytes();
line.push(b'\n');
Some(line)
} else {
None
}
})
.flatten()
.collect();
let reader = Csv::from_reader(&contents[..])
.delimiter(b',')
.has_header(true);
pb.println(tr!("Parsing user submitted CSV file..."));
let mut entries_edit = Vec::new();
for (count, record_result) in reader.enumerate() {
let count = count + 1;
let record = record_result.with_context(|| {
tr!(
"Failed to read record `{}' in user submitted CSV file.",
count
)
})?;
let mut columns = record.columns().with_context(|| {
tr!(
"Failed to convert record `{}' to UTF-8 in user submitted CSV file.",
count
)
})?;
let uci = Uci::from_ascii(
columns
.nth(1)
.ok_or(anyhow!(
"{}",
tr!(
"Failed to read column 1 of record `{}' in user submitted CSV file.",
count
)
))?
.as_bytes(),
)
.with_context(|| {
tr!(
"Failed to convert column 1 of record `{}' to UCI in user submitted CSV file.",
count
)
})?;
match uci.to_move(&pos) {
Ok(_) => {}
Err(err) => {
pb.finish_with_message(tr!("Illegal move: `{}'", uci));
bail!(
"{}",
tr!("Illegal move `{}' in position `{}' for record `{}' in user submitted CSV file.: {}", uci, epd, count, err)
);
}
};
let (from, to, promotion) = match uci {
Uci::Normal {
from,
to,
promotion,
} => (from as u8, to as u8, promotion),
_ => unreachable!("unsupported uci"),
};
let promotion = match promotion {
None => 0,
Some(Role::Rook) => 1,
Some(Role::Knight) => 2,
Some(Role::Bishop) => 3,
Some(Role::Queen) => 4,
_ => unreachable!("invalid promotion"),
};
entries_edit.push(CompactSBookMoveEntry {
from,
to,
promotion,
priority: str::parse::<u8>(
columns
.next()
.ok_or(anyhow!("{}", tr!("Failed to read column 2, the column of priority, of record `{}' in user submitted CSV file.", count)))?
).with_context(|| tr!("Failed to convert column 2, the column of priority, of record `{}' to integer in user submitted CSV file.", count))?,
ngames: str::parse::<u32>(
columns
.next()
.ok_or(anyhow!("{}", tr!("Failed to read column 3, the column of number of games, of record `{}' in user submitted CSV file.", count)))?
).with_context(|| tr!("Failed to convert column 3, the column of number of games, of record `{}' to integer in user submitted CSV file.", count))?,
nwon: str::parse::<u32>(
columns
.next()
.ok_or(anyhow!("{}", tr!("Failed to read column 4, the column of number of wins, of record `{}' in user submitted CSV file.", count)))?
).with_context(|| tr!("Failed to convert column 4, the column of number of wins, of record `{}' to integer in user submitted CSV file.", count))?,
nlost: str::parse::<u32>(
columns
.next()
.ok_or(anyhow!("{}", tr!("Failed to read column 5, the column of number of losses, of record `{}' in user submitted CSV file.", count)))?
).with_context(|| tr!("Failed to convert column 5, the column of number of losses, of record `{}' to integer in user submitted CSV file.", count))?,
});
}
db.sort_entries(&mut entries_edit);
pb.println(tr!(
"Success parsing user submitted CSV file with {} entries.",
style(entries_edit.len()).bold().cyan()
));
pb.println(tr!(
"Traversing the input opening book. This may take a while."
));
pb.set_message(tr!("Traversing:"));
pb.set_length(db.book_moves as u64);
pb.set_position(0);
let book = db.traverse_book(Some(&pb));
let mut book: BTreeMap<u64, Vec<CompactSBookMoveEntry>> = book
.into_iter()
.map(|(key, vec)| {
let vec = vec.into_iter().map(CompactSBookMoveEntry::from).collect();
(key, vec)
})
.collect();
let hash = zobrist_hash(&pos);
book.insert(hash, entries_edit);
let len = book.values().map(|vec| vec.len()).sum::<usize>();
pb.set_position(0);
pb.set_length(len as u64);
pb.set_message(tr!("Indexing:"));
pb.println(tr!(
"Indexing {} moves in the memory.",
style(len).bold().cyan()
));
let epd_root =
Epd::from_ascii(ROOT.as_bytes()).expect("ROOT is invalid FEN, please report a bug!");
let pos: Chess = epd_root
.into_position(CastlingMode::Standard)
.expect("ROOT is illegal FEN, please report a bug!");
db.traverse_book_and_merge(&pos, &mut book, Some(&pb));
pb.set_position(0);
pb.set_length(db.book_moves as u64);
pb.set_message(tr!("Copying:"));
pb.println(tr!(
"Copying {} entries from input book to the output book with maximum depth {}.",
style(db.book_moves).bold().cyan(),
style(db.book_depth).bold().cyan()
));
db.write_file(&mut output_file, Some(&pb))
.with_context(|| {
tr!(
"Failed to write output Arena opening book `{}'.",
output_filename
)
})?;
pb.finish_with_message(tr!("Copying done."));
drop(output_file);
let filename = if suffix.is_some() {
std::fs::rename(output_filename, input_filename).with_context(|| {
tr!(
"Failed to rename file `{}' to `{}'.",
output_filename,
input_filename
)
})?;
input_filename
} else {
output_filename
};
command_find(
!*STDOUT_IS_TTY,
Path::new(filename),
epd_str,
None,
None,
None,
)
.context(tr!("find subcommand failed."))?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn ctg2abk(
_porcelain: bool,
author: Option<String>,
comment: Option<String>,
probabilities: (Option<&u32>, Option<&u32>, Option<&u32>),
no_colors: bool,
no_nags: bool,
color_priorities: ColorPriority,
nag_priorities: NagPriority,
input_filename: &str,
output_filename: &str,
epd: &str,
) -> Result<()> {
let mut db = CtgBook::open(input_filename)
.with_context(|| tr!("Failed to open CTG file `{}'.", input_filename))?;
let num = db.total_positions()?;
let pb = get_progress_bar(num as u64);
pb.println(tr!("Creating output Arena opening book..."));
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(output_filename)
.with_context(|| {
tr!(
"Failed to create output Arena opening book `{}'.",
output_filename
)
})?,
);
pb.println(tr!("Success creating output Arena opening book."));
pb.println(tr!(
"Searching for positions in the CTG input file. This may take a while."
));
pb.set_message(tr!("Searching:"));
let mut book = db.extract_abk(
epd,
no_colors,
no_nags,
color_priorities,
nag_priorities,
Some(&pb),
);
let book_len = book.len();
drop(db);
pb.println(tr!(
"Found {} chess positions in the CTG input file.",
style(book_len).bold().cyan()
));
pb.println(tr!("Indexing ABK book entries in the memory..."));
pb.set_message(tr!("Indexing:"));
pb.set_position(0);
pb.set_length(book_len as u64);
let mut outbook = AbkBook::default();
outbook.author = author.unwrap_or(get_username().unwrap_or("".to_string()));
outbook.comment = comment.unwrap_or(format!(
"written by {}-{}",
built_info::PKG_NAME,
built_info::GIT_VERSION.unwrap_or(built_info::PKG_VERSION)
));
outbook.probability_priority = *probabilities.0.unwrap_or(&5);
outbook.probability_games = *probabilities.1.unwrap_or(&1);
outbook.probability_win_percent = *probabilities.2.unwrap_or(&5);
let epd_root =
Epd::from_ascii(ROOT.as_bytes()).expect("ROOT is invalid FEN, please report a bug!");
let pos: Chess = epd_root
.into_position(CastlingMode::Standard)
.expect("ROOT is illegal FEN, please report a bug!");
outbook.traverse_book_and_merge(&pos, &mut book, Some(&pb));
outbook.use_book_half_move = outbook.book_depth + 1;
drop(book);
pb.println(tr!(
"Writing {} ABK book entries to the output book...",
style(outbook.book_moves).bold().cyan()
));
pb.set_message(tr!("Writing:"));
pb.set_position(0);
pb.set_length(outbook.book_moves as u64);
outbook
.write_file(&mut output_file, Some(&pb))
.with_context(|| {
tr!(
"Failed to write output Arena opening book `{}'.",
output_filename
)
})?;
pb.println(tr!("Success writing entries to the output book."));
pb.finish_with_message(tr!("Writing done."));
command_info(!*STDOUT_IS_TTY, Path::new(output_filename))
.context(tr!("info subcommand failed."))?;
Ok(())
}
fn obk2bin(
_porcelain: bool,
preserve_null: bool,
edit_learn: bool,
scale_weights: bool,
input_filename: &str,
output_filename: &str,
epd: &str,
) -> Result<()> {
let mut db: ObkBook = ObkBook::open(input_filename)
.with_context(|| tr!("Failed to open OBK file `{}'.", input_filename))?;
let pb = get_progress_bar(db.move_count as u64);
pb.println(tr!("Creating output PolyGlot opening book..."));
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(output_filename)
.with_context(|| {
tr!(
"Failed to create output PolyGlot opening book `{}'.",
output_filename
)
})?,
);
pb.println(tr!("Success creating output PolyGlot opening book."));
pb.println(tr!(
"Traversing the input opening book. This may take a while."
));
pb.set_message(tr!("Traversing:"));
db.load(Some(&pb));
pb.println(tr!(
"Found {} moves in the input opening book.",
style(db.moves.len()).bold().cyan()
));
let epd = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.clone()
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN."))?;
pb.println(tr!(
"Searching entries starting from root `{}'...",
style(epd.to_string()).bold().green()
));
pb.set_message(tr!("Searching:"));
pb.set_position(0);
pb.set_length(db.moves.len() as u64);
let mut book = db.traverse_tree(pos, Some(&pb));
let mut keys: Vec<u64> = book.keys().copied().collect();
keys.sort();
let mut book: Vec<(u64, Vec<BookEntry>)> = keys
.into_iter()
.filter_map(|key| book.remove(&key).map(|value| (key, value)))
.collect();
pb.set_position(0);
pb.set_length(book.len() as u64);
pb.println(tr!(
"Found {} chess positions in the OBK input file.",
style(book.len()).bold().cyan()
));
pb.println(tr!(
"Saving entries starting from root `{}' to the output book...",
style(epd.to_string()).bold().green()
));
pb.set_message(tr!("Saving:"));
for (_, moves) in &mut book {
if moves.is_empty() {
continue;
}
moves.sort_unstable_by_key(|mov| std::cmp::Reverse(mov.weight));
let max_weight = moves[0].weight;
let scale: f64 = 0xfff0 as f64 / max_weight as f64;
for entry in &mut *moves {
if !preserve_null && entry.weight == 0 {
continue;
}
if scale_weights {
entry.weight = (scale * entry.weight as f64) as u16;
}
if !edit_learn {
entry.learn = 0;
}
bin_entry_to_file(&mut output_file, entry).with_context(|| {
tr!(
"Failed to write to output PolyGlot opening book `{}'.",
output_filename
)
})?;
}
pb.inc(1);
}
pb.finish_with_message(tr!("Saving done."));
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn ctg2bin(
_porcelain: bool,
preserve_null: bool,
edit_learn: bool,
scale_weights: bool,
no_colors: bool,
no_nags: bool,
color_weights: ColorWeight,
nag_weights: NagWeight,
input_filename: &str,
output_filename: &str,
epd: &str,
) -> Result<()> {
let mut db = CtgBook::open(input_filename)
.with_context(|| tr!("Failed to open CTG file `{}'.", input_filename))?;
let num = db.total_positions().with_context(|| {
tr!(
"Failed to count total number of positions in CTG file `{}'.",
input_filename
)
})?;
let pb = get_progress_bar(num as u64);
pb.println(tr!("Creating output PolyGlot opening book..."));
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(output_filename)
.with_context(|| {
tr!(
"Failed to create output PolyGlot opening book `{}'.",
output_filename
)
})?,
);
pb.println(tr!("Success creating output PolyGlot opening book."));
pb.println(tr!(
"Searching for move entries in the CTG input file. This may take a while."
));
pb.set_message(tr!("Searching:"));
let tree = db.extract_bin(
epd,
edit_learn,
no_colors,
no_nags,
color_weights,
nag_weights,
Some(&pb),
);
let tlen = tree.len();
pb.println(tr!(
"Found {} positions in the CTG input file.",
style(tlen).bold().cyan()
));
pb.set_position(0);
pb.set_length(tlen as u64);
pb.println(tr!(
"Saving {} positions from input book to the output book...",
style(tlen).bold().cyan()
));
pb.set_message(tr!("Saving:"));
for (key, moves) in tree {
let max_weight = moves[0].weight;
let scale: f64 = 0xfff0 as f64 / max_weight as f64;
for entry in moves {
if !preserve_null && entry.weight == 0 {
continue;
}
let weight = if scale_weights {
(scale * entry.weight as f64) as u16
} else {
entry.weight
};
let entry = BookEntry {
key,
weight,
mov: entry.mov,
learn: entry.learn,
};
bin_entry_to_file(&mut output_file, &entry).with_context(|| {
tr!(
"Failed to write to output PolyGlot opening book `{}'.",
output_filename
)
})?;
}
}
pb.println(tr!(
"Success copying entries from the input book to the output book."
));
pb.finish_with_message(tr!("Saving done."));
command_info(!*STDOUT_IS_TTY, Path::new(output_filename))
.context(tr!("info subcommand failed."))?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn bin2bin(
_porcelain: bool,
preserve_null: bool,
scale_weights: bool,
rescale_weights: bool,
suffix: Option<String>,
input_filename: &str,
output_filename: &str,
epd: &str,
) -> Result<()> {
if let Some(ref suffix) = suffix {
if !suffix.is_empty() {
let backup = Path::new(input_filename).with_extension(suffix);
std::fs::copy(input_filename, backup.as_path())?;
}
}
let db = PolyGlotBook::open(input_filename)
.with_context(|| tr!("Failed to open PolyGlot file `{}'.", input_filename))?;
let epd_str = epd;
let epd = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.clone()
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN."))?;
let edit_key = zobrist_hash(&pos);
let entries = db.lookup_moves(edit_key).unwrap_or(Vec::new());
let board_display = get_board_lines(&pos, false)
.iter()
.map(|line| format!("#{}", line))
.collect::<Vec<String>>()
.join("\n");
let mut contents = String::new();
contents.push_str(&tr!("# jja: polyglot edit screen\n"));
contents.push_str("# vim: set ft=conf :\n#\n");
contents.push_str(&edit_comment(pos.clone(), edit_key));
contents.push('\n');
contents.push_str(&board_display);
contents.push_str(&format!(
"\n\n{}\n\n{}\n",
format_bin_entries(&pos, entries),
*POLYGLOT_EDIT_COMMENT
));
let contents = edit_tempfile(Some(contents), Some(".csv".to_string()))
.context(tr!("Failed to edit temporary file."))?;
let pb = get_progress_bar(db.num_entries as u64);
let contents = contents.as_bytes();
let contents: Vec<u8> = contents
.lines()
.filter_map(|line| {
let line = line.ok()?;
if !line.is_empty() && !line.starts_with('#') {
let mut line = line.into_bytes();
line.push(b'\n');
Some(line)
} else {
None
}
})
.flatten()
.collect();
let reader = Csv::from_reader(&contents[..])
.delimiter(b',')
.has_header(true);
pb.println(tr!("Parsing user submitted CSV file..."));
let mut entries_edit = Vec::new();
for (count, record_result) in reader.enumerate() {
let count = count + 1;
let record = record_result.with_context(|| {
tr!(
"Failed to read record `{}' in user submitted CSV file.",
count
)
})?;
let mut columns = record.columns().with_context(|| {
tr!(
"Failed to convert record `{}' to UTF-8 in user submitted CSV file.",
count
)
})?;
let uci = Uci::from_ascii(
columns
.nth(1)
.ok_or(anyhow!(
"{}",
tr!(
"Failed to read column 1 of record `{}' in user submitted CSV file.",
count
)
))?
.as_bytes(),
)
.with_context(|| {
tr!(
"Failed to convert column 1 of record `{}' to UCI in user submitted CSV file.",
count
)
})?;
match uci.to_move(&pos) {
Ok(_) => {}
Err(err) => {
pb.finish_with_message(tr!("Illegal move: `{}'", uci));
bail!(
"{}",
tr!("Illegal move `{}' in position `{}' for record `{}' in user submitted CSV file.: {}", uci, epd, count, err)
);
}
};
entries_edit.push(BookEntry {
key: edit_key,
mov: from_uci(uci, is_king_on_start_square(&pos)),
weight: str::parse::<u16>(
columns
.next()
.ok_or(anyhow!("{}", tr!("Failed to read column 2, the column of weight, of record `{}' in user submitted CSV file.", count)))?
).with_context(|| tr!("Failed to convert column 2, the column of weight, of record `{}' to integer in user submitted CSV file.", count))?,
learn: str::parse::<u32>(
columns
.next()
.ok_or(anyhow!("{}", tr!("Failed to read column 3, the column of learn, of record `{}' in user submitted CSV file.", count)))?
).with_context(|| tr!("Failed to convert column 3, the column of learn, of record `{}' to integer in user submitted CSV file.", count))?,
});
}
pb.println(tr!(
"Success parsing user submitted CSV file with {} entries.",
style(entries_edit.len()).bold().cyan()
));
pb.println(tr!("Creating output PolyGlot opening book..."));
let mut output_opts = fs::OpenOptions::new();
let mut output_file = output_opts.write(true);
if suffix.is_none() {
output_file = output_file.create_new(true);
}
let mut output_file = BufWriter::new(output_file.open(output_filename).with_context(|| {
tr!(
"Failed to create output PolyGlot opening book `{}'.",
output_filename
)
})?);
pb.println(tr!("Success creating output PolyGlot opening book."));
pb.println(tr!(
"Copying {} entries from input book to the output book.",
style(db.num_entries).bold().cyan()
));
pb.set_message(tr!("Copying:"));
let mut edit_saved = false;
for result in db.into_iter_grouped() {
let group_entries = match result {
Ok(entries) => entries,
Err(e) => {
bail!(
"{}",
tr!("Failed to read PolyGlot entry: {}.", format!("{:?}", e))
);
}
};
let current_key = group_entries[0].key;
match current_key.cmp(&edit_key) {
std::cmp::Ordering::Equal => {
save_user_submitted_edits(
&mut entries_edit,
preserve_null,
scale_weights,
&mut output_file,
&pb,
)
.context(tr!(
"Failed to save user submitted edits to the output PolyGlot opening book."
))?;
edit_saved = true;
continue; }
std::cmp::Ordering::Greater if !edit_saved => {
save_user_submitted_edits(
&mut entries_edit,
preserve_null,
scale_weights,
&mut output_file,
&pb,
)
.context(tr!(
"Failed to save user submitted edits to the output PolyGlot opening book."
))?;
edit_saved = true;
}
_ => { }
}
let group_max_weight = group_entries
.iter()
.max_by_key(|entry| entry.weight)
.map(|entry| entry.weight)
.unwrap_or(0);
let scale: f64 = if rescale_weights && group_max_weight != 0 {
0xfff0 as f64 / group_max_weight as f64
} else {
1.0 };
for mut entry in group_entries {
if !preserve_null && entry.weight == 0 {
continue;
}
entry.weight = (scale * entry.weight as f64) as u16;
bin_entry_to_file(&mut output_file, &entry).with_context(|| {
tr!(
"Failed to write to output PolyGlot opening book `{}'.",
output_filename
)
})?;
pb.inc(1);
}
}
if !edit_saved {
save_user_submitted_edits(
&mut entries_edit,
preserve_null,
scale_weights,
&mut output_file,
&pb,
)
.context(tr!(
"Failed to save user submitted edits to the output PolyGlot opening book."
))?;
}
pb.println(tr!(
"Success copying entries from the input book to the output book."
));
pb.finish_with_message(tr!("Copying done."));
drop(output_file);
let filename = if suffix.is_some() {
std::fs::rename(output_filename, input_filename).with_context(|| {
tr!(
"Failed to rename file `{}' to `{}'.",
output_filename,
input_filename
)
})?;
input_filename
} else {
output_filename
};
command_find(
!*STDOUT_IS_TTY,
Path::new(filename),
epd_str,
None,
None,
None,
)?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn exp2exp(
_porcelain: bool,
preserve_null: bool,
suffix: Option<String>,
input_filename: &str,
output_filename: &str,
epd: &str,
) -> Result<()> {
if let Some(ref suffix) = suffix {
if !suffix.is_empty() {
let backup = Path::new(input_filename).with_extension(suffix);
std::fs::copy(input_filename, backup.as_path())?;
}
}
let mut db = BrainLearnFile::open(input_filename)
.with_context(|| tr!("Failed to open BrainLearn file `{}'.", input_filename))?;
let pb = get_progress_bar(db.num_entries as u64);
let cutoff = if preserve_null { None } else { Some(0) };
db.load(cutoff, Some(&pb));
let epd_str = epd;
let epd = Epd::from_ascii(epd.as_bytes())?;
let pos: Chess = epd.clone().into_position(CastlingMode::Standard)?;
let edit_key = stockfish_hash(&pos);
let entries = db.lookup_moves(edit_key).unwrap_or(Vec::new());
let board_display = get_board_lines(&pos, false)
.iter()
.map(|line| format!("#{}", line))
.collect::<Vec<String>>()
.join("\n");
let mut contents = String::new();
contents.push_str(&tr!("# jja: brainlearn edit screen\n"));
contents.push_str("# vim: set ft=conf :\n#\n");
contents.push_str(&edit_comment(pos.clone(), edit_key));
contents.push('\n');
contents.push_str(&board_display);
contents.push_str(&format!(
"\n\n{}\n\n{}\n",
format_exp_entries(&pos, entries),
*BRAINLEARN_EDIT_COMMENT
));
let contents = edit_tempfile(Some(contents), Some(".csv".to_string()))?;
let contents = contents.as_bytes();
let contents: Vec<u8> = contents
.lines()
.filter_map(|line| {
let line = line.ok()?;
if !line.is_empty() && !line.starts_with('#') {
let mut line = line.into_bytes();
line.push(b'\n');
Some(line)
} else {
None
}
})
.flatten()
.collect();
let reader = Csv::from_reader(&contents[..])
.delimiter(b',')
.has_header(true);
pb.println(tr!("Parsing user submitted CSV file..."));
let mut entries_edit = Vec::new();
for record_result in reader {
let record = record_result.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
let mut columns = record.columns().expect("cannot convert csv record to utf8");
let uci = Uci::from_ascii(
columns
.nth(1)
.ok_or(io::Error::from(io::ErrorKind::InvalidInput))?
.as_bytes(),
)?;
let mov = match uci.to_move(&pos) {
Ok(mov) => mov,
Err(err) => {
pb.finish_with_message(tr!("Illegal move: `{}'", uci));
bail!(
"{}",
tr!("Illegal move `{}' in position `{}': {}", uci, epd, err)
);
}
};
entries_edit.push(ExperienceEntry {
key: edit_key,
mov: brainlearn::from_move(mov),
depth: str::parse::<i32>(
columns
.next()
.ok_or(io::Error::from(io::ErrorKind::InvalidInput))?,
)?,
score: str::parse::<i32>(
columns
.next()
.ok_or(io::Error::from(io::ErrorKind::InvalidInput))?,
)?,
perf: str::parse::<i32>(
columns
.next()
.ok_or(io::Error::from(io::ErrorKind::InvalidInput))?,
)?,
});
}
let entries_edit_len = entries_edit.len();
pb.println(tr!(
"Success parsing user submitted CSV file with {} entries.",
style(entries_edit_len).bold().cyan()
));
let old_value = db.data.insert(edit_key, entries_edit);
let num_entries =
db.num_entries - old_value.map(|entries| entries.len()).unwrap_or(0) + entries_edit_len;
pb.println(tr!("Creating output BrainLearn experience file..."));
let mut output_opts = fs::OpenOptions::new();
let mut output_file = output_opts.write(true);
if suffix.is_none() {
output_file = output_file.create_new(true);
}
let mut output_file = BufWriter::new(output_file.open(output_filename).with_context(|| {
tr!(
"Failed to create output BrainLearn experience file `{}'.",
output_filename
)
})?);
pb.println(tr!("Success creating output BrainLearn experience file."));
pb.println(tr!(
"Writing {} entries to the output book.",
style(num_entries).bold().cyan()
));
pb.set_message(tr!("Writing:"));
pb.set_position(num_entries as u64);
for entries in db.data.values() {
for entry in entries {
if !preserve_null && entry.depth == 0 {
continue;
}
exp_entry_to_file(&mut output_file, entry)?;
pb.inc(1);
}
}
pb.println(tr!(
"Success copying entries from the input book to the output book."
));
pb.finish_with_message(tr!("Copying done."));
drop(output_file);
let filename = if suffix.is_some() {
std::fs::rename(output_filename, input_filename).with_context(|| {
tr!(
"Failed to rename file `{}' to `{}'.",
output_filename,
input_filename
)
})?;
input_filename
} else {
output_filename
};
command_find(
!*STDOUT_IS_TTY,
Path::new(filename),
epd_str,
None,
None,
None,
)
.context(tr!("find subcommand failed."))?;
Ok(())
}
fn pgn2epd(
_porcelain: bool,
input_ext: &str,
input_filename: &str,
output_filename: &str,
) -> Result<()> {
let pb = get_progress_bar(0);
pb.println(tr!("Creating output EPD file..."));
let output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(output_filename)
.with_context(|| tr!("Failed to create output EPD file `{}'.", output_filename))?,
);
pb.println(tr!("Success creating output EPD file."));
pb.println(tr!("Opening PGN file..."));
let file = match File::open(input_filename) {
Ok(file) => file,
Err(err) => {
bail!(
"{}",
tr!("Failed to open PGN file `{}': {}", input_filename, err)
);
}
};
let uncompressed: Box<dyn Read + Send> = if input_ext.eq_ignore_ascii_case("zst") {
Box::new(match zstd::Decoder::new(file) {
Ok(decoder) => decoder,
Err(err) => {
bail!(
"{}",
tr!("Failed to open PGN file `{}': {}", input_filename, err)
);
}
})
} else if input_ext.eq_ignore_ascii_case("bz2") {
Box::new(bzip2::read::MultiBzDecoder::new(file))
} else if input_ext.eq_ignore_ascii_case("xz") {
Box::new(xz2::read::XzDecoder::new(file))
} else if input_ext.eq_ignore_ascii_case("gz") {
Box::new(flate2::read::GzDecoder::new(file))
} else if input_ext.eq_ignore_ascii_case("lz4") {
Box::new(match lz4::Decoder::new(file) {
Ok(decoder) => decoder,
Err(err) => {
bail!(
"{}",
tr!("Failed to open PGN file `{}': {}", input_filename, err)
);
}
})
} else {
Box::new(file)
};
pb.println(tr!("Success opening PGN file."));
pb.set_message(tr!("Dumping:"));
let mut tracker = PositionTracker::new(
Some(Box::new(output_file)),
Some(OutputFormat::Epd),
None,
None,
Some(&pb),
);
BufferedReader::new(uncompressed).read_all(&mut tracker)?;
let count = pb.position();
pb.println(tr!(
"Successfully dumped {} positions from PGN file `{}' in format EPD to output file `{}'.",
count,
input_filename,
output_filename
));
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn any2pgn(
_porcelain: bool,
input_ext: &str,
input_filename: &str,
output_filename: &str,
roster: (Option<&str>, Option<&str>, Option<&str>, &str, &str, &str),
epd: &str,
look_ahead: usize,
max_ply: usize,
) -> Result<()> {
let epd = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let pos: Chess = epd
.into_position(CastlingMode::Standard)
.context(tr!("Failed to parse illegal FEN."))?;
let event = if let Some(event) = roster.0 {
event
} else {
Path::new(input_filename)
.file_name()
.and_then(|os_str| os_str.to_str())
.unwrap_or("?")
};
let site_string: String;
let site = if let Some(site) = roster.1 {
site
} else {
site_string = hostname::get()
.unwrap_or_else(|_| OsString::from("?"))
.into_string()
.unwrap_or("?".to_string());
site_string.as_str()
};
let date_string: String;
let date = if let Some(date) = roster.2 {
date
} else {
let mtime = std::fs::metadata(input_filename).with_context(|| tr!("Failed to read metadata for file `{}'.", input_filename))?
.modified()
.unwrap_or(std::time::SystemTime::now())
.duration_since(std::time::SystemTime::UNIX_EPOCH).with_context(|| tr!("Failed to calculate last modified time of file `{}', clock may have gone backwards!", input_filename))?
.as_secs();
let (year, month, day) = seconds_to_ymd(mtime);
date_string = format!("{}.{:02}.{:02}", year, month, day);
date_string.as_str()
};
let pb = get_progress_bar(0);
pb.println(tr!("Creating output PGN file..."));
let mut output_file: Box<dyn Write + Send> = Box::new(BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(output_filename)
.with_context(|| tr!("Failed to create output PGN file `{}'", output_filename))?,
));
pb.println(tr!("Success creating output PGN file."));
match input_ext {
"abk" => {
let mut db = AbkBook::open(input_filename)
.with_context(|| tr!("Failed to open ABK file `{}'.", input_filename))?;
db.write_pgn(
&mut output_file,
&pos,
event,
site,
date,
roster.3,
roster.4,
roster.5,
max_ply,
Some(&pb),
);
}
"bin" => {
let db = PolyGlotBook::open(input_filename)
.with_context(|| tr!("Failed to open PolyGlot file `{}'.", input_filename))?;
db.write_pgn(
&mut output_file,
&pos,
event,
site,
date,
roster.3,
roster.4,
roster.5,
look_ahead,
max_ply,
Some(&pb),
);
}
"ctg" => {
let mut db = CtgBook::open(input_filename)
.with_context(|| tr!("Failed to open CTG file `{}'.", input_filename))?;
db.write_pgn(
&mut output_file,
&pos,
event,
site,
date,
roster.3,
roster.4,
roster.5,
max_ply,
Some(&pb),
);
}
"exp" => {
let mut db = BrainLearnFile::open(input_filename)
.with_context(|| tr!("Failed to open BrainLearn file `{}'.", input_filename))?;
pb.set_length(db.num_entries as u64);
db.load(None, Some(&pb));
db.write_pgn(
&mut output_file,
&pos,
event,
site,
date,
roster.3,
roster.4,
roster.5,
max_ply,
Some(&pb),
);
}
"obk" => {
let mut db = ObkBook::open(input_filename)
.with_context(|| tr!("Failed to open OBK file `{}'.", input_filename))?;
pb.set_length(db.move_count as u64);
db.load(Some(&pb));
db.write_pgn(
&mut output_file,
&pos,
event,
site,
date,
roster.3,
roster.4,
roster.5,
max_ply,
Some(&pb),
);
}
_ => unreachable!("{}", tr!("Invalid chess file extension `{}'.", input_ext)),
}
pb.finish_with_message(tr!(
"Successfully converted the chess file to a PGN with {} variations.",
style(pb.position()).bold().cyan()
));
Ok(())
}
fn seconds_to_ymd(seconds: u64) -> (u32, u32, u32) {
let days = seconds / 86400;
let days_per_400_years = 365 * 400 + 97; let years_400 = days / days_per_400_years;
let days = days % days_per_400_years;
let days_per_100_years = 365 * 100 + 24;
let years_100 = days / days_per_100_years;
let days = days % days_per_100_years;
let days_per_4_years = 365 * 4 + 1;
let years_4 = days / days_per_4_years;
let days = days % days_per_4_years;
let years_1 = days / 365;
let days = days % 365;
let year = 400 * years_400 + 100 * years_100 + 4 * years_4 + years_1;
let days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30];
let mut month = 0;
let mut day = days as u32;
while month < 12 && day >= days_in_month[month] {
day -= days_in_month[month];
month += 1;
}
(year as u32 + 1970, month as u32 + 1, day + 1)
}
fn save_user_submitted_edits<W: Write>(
entries_edit: &mut Vec<BookEntry>,
preserve_null: bool,
scale_weights: bool,
output_file: &mut BufWriter<W>,
pb: &ProgressBar,
) -> Result<()> {
pb.println(tr!("Saving user submitted edits to the output file."));
pb.println(tr!("Rescaling user submitted move weights."));
let max_weight = entries_edit
.iter()
.max_by_key(|entry| entry.weight)
.map(|entry| entry.weight)
.unwrap_or(0);
let scale: f64 = 0xfff0 as f64 / max_weight as f64;
for entry in &mut *entries_edit {
if !preserve_null && entry.weight == 0 {
continue;
}
if scale_weights {
entry.weight = (scale * entry.weight as f64) as u16;
}
bin_entry_to_file(&mut *output_file, entry)
.context(tr!("Failed to write to output PolyGlot opening book."))?;
pb.inc(1);
}
pb.println(tr!(
"Success saving {} moves submitted by the user to the output file.",
style(entries_edit.len()).bold().cyan()
));
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn polyglot_make(
_porcelain: bool,
verbose: u8,
nthreads: usize,
batch_size: usize,
compression_which: rocksdb::DBCompressionType,
compression_level: i32,
max_open_files: i32,
read_ahead: usize,
input_files: &[String],
output_filename: &str,
max_ply: u64,
min_games: u64,
min_wins: u64,
min_score: f64,
min_pieces: usize,
filter_side: Option<Color>,
hashcode: bool,
uniform: bool,
preserve_null: bool,
scale_weights: bool,
sync_io: bool,
wdl_factor: (f64, f64, f64),
parsed_filter_expr: Option<Vec<FilterComponent>>,
) -> Result<()> {
let pb = get_progress_spinner();
pb.println(tr!("Creating output PolyGlot opening book..."));
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(output_filename)
.with_context(|| {
tr!(
"Failed to create output PolyGlot opening book `{}'.",
output_filename
)
})?,
);
pb.println(tr!("Success creating output PolyGlot opening book."));
pb.println(tr!(
"Parsing the input PGN files using {} threads with max ply {}, filter_side:{}...",
style(nthreads).bold().cyan(),
style(max_ply).bold().cyan(),
style(format!("{:?}", filter_side)).bold().yellow()
));
pb.set_message(tr!("Parsing:"));
let db_path = Path::new(&input_files[0]).with_extension("jja-rdb");
if db_path.exists() {
pb.finish_and_clear();
eprintln!(
"{}",
tr!(
"Found existing jja rocksdb database located at `{}'.",
db_path.display()
)
);
eprintln!("{}", tr!("Is there another instance of jja running?"));
eprintln!(
"{}",
tr!("If not, please remove the temporary directory and try again.")
);
eprintln!("{}", tr!("Cowardly refusing to continue."));
std::process::exit(1);
}
let base = create_opening_book(
input_files,
&db_path,
verbose,
max_ply,
min_pieces,
filter_side,
hashcode,
nthreads,
batch_size,
compression_which,
compression_level,
max_open_files,
parsed_filter_expr,
Some(pb.clone()),
)
.context(tr!(
"Failed to create PolyGlot opening book out of PGN files."
))?;
pb.finish_with_message(tr!("Parsing done."));
let book_cf = Arc::clone(&base.book.cf_handle("opening_book").unwrap());
let book_len = base
.book
.property_int_value_cf(&book_cf, "rocksdb.estimate-num-keys")?
.unwrap_or(0);
let pb = get_progress_bar(book_len as u64);
pb.println(tr!(
"Found approximately {} unique positions in the input PGN files.",
style(book_len).bold().cyan()
));
pb.println(tr!(
"Filtering the positions based on min_games:{} min_score:{} min_wins:{} preserve_null:{} wdl_factor:({},{},{})",
style(min_games).bold().cyan(),
style(min_score).bold().cyan(),
style(min_wins).bold().cyan(),
style(preserve_null).bold().yellow(),
style(wdl_factor.0).bold().green(),
style(wdl_factor.1).bold().blue(),
style(wdl_factor.2).bold().red()
));
pb.set_message(tr!("Filtering:"));
let mut read_options = rocksdb::ReadOptions::default();
read_options.fill_cache(false);
read_options.set_async_io(!sync_io);
read_options.set_readahead_size(read_ahead);
let iter = base
.book
.iterator_cf_opt(&book_cf, read_options, rocksdb::IteratorMode::Start);
let mut book: BTreeMap<u64, Vec<GameEntry>> = BTreeMap::new();
for (key, entry_vec_bytes) in iter.filter_map(Result::ok) {
let entry_vec = deserialize_game_entries(&entry_vec_bytes);
let key_value = u64::from_le_bytes(key.as_ref().try_into().unwrap());
let filtered_entry_vec: Vec<GameEntry> = entry_vec
.into_iter()
.filter(|entry| {
let this_won = entry.nwon;
let that_won = entry.nlost;
let total_games = entry.ngames;
let draws = total_games - (this_won + that_won);
let this_won_scaled = this_won as f64 * wdl_factor.0;
let draws_scaled = draws as f64 * wdl_factor.1;
let losses_scaled = that_won as f64 * wdl_factor.2;
let score_numerator =
f64::min(this_won_scaled + draws_scaled + losses_scaled, f64::MAX);
let score = score_numerator / total_games as f64 * 100.0;
pb.inc(1);
let score_is_nonzero = score > 0.0 || (score == 0.0 && preserve_null);
score_is_nonzero
&& this_won >= min_wins
&& total_games >= min_games
&& score >= min_score
})
.collect();
if !filtered_entry_vec.is_empty() {
book.insert(key_value, filtered_entry_vec);
}
}
pb.finish_with_message(tr!("Filtering done."));
let book_len = book.values().map(|vec| vec.len()).sum::<usize>();
let pb = get_progress_bar(book_len as u64);
pb.println(tr!(
"Saving {} filtered entries to the output book.",
style(book_len).bold().cyan()
));
pb.set_message(tr!("Saving:"));
for (key, moves) in &mut book {
if moves.is_empty() {
continue;
}
if !uniform {
moves.sort_unstable_by_key(|mov| std::cmp::Reverse(mov.weight(&wdl_factor)));
}
let max_weight = if uniform { 1 } else { moves[0].weight(&wdl_factor) };
let scale: f64 = 0xfff0 as f64 / max_weight as f64;
for game_entry in &mut *moves {
let weight = if uniform { 1 } else { game_entry.weight(&wdl_factor) };
if !preserve_null && weight == 0 {
continue;
}
let entry = BookEntry {
key: *key,
mov: game_entry.mov,
weight: if uniform {
1
} else if !scale_weights {
std::cmp::min(weight, u16::MAX as u64) as u16
} else {
(scale * weight as f64) as u16
},
learn: 0,
};
bin_entry_to_file(&mut output_file, &entry).with_context(|| {
tr!(
"Failed to write to output PolyGlot opening book `{}'.",
output_filename
)
})?;
pb.inc(1);
}
}
pb.finish_with_message(tr!("Saving done."));
command_info(!*STDOUT_IS_TTY, Path::new(output_filename))
.context(tr!("info subcommand failed."))?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn polyglot_merge(
_porcelain: bool,
input1_filename: &str,
input2_filename: &str,
output_filename: &str,
strategy: MergeStrategy,
weight_cutoff: Option<u16>,
outlier_threshold: Option<u16>,
rescale_weights: bool,
) -> Result<()> {
let mut input1 = PolyGlotBook::open(input1_filename)
.with_context(|| tr!("Failed to open PolyGlot file `{}'.", input1_filename))?;
let mut input2 = PolyGlotBook::open(input2_filename)
.with_context(|| tr!("Failed to open PolyGlot file `{}'.", input2_filename))?;
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(output_filename)
.with_context(|| {
tr!(
"Failed to create output PolyGlot opening book `{}'.",
output_filename
)
})?,
);
let mut map1: BTreeMap<u64, Vec<BookEntry>> = BTreeMap::new();
let mut map2: BTreeMap<u64, Vec<BookEntry>> = BTreeMap::new();
let pb = get_progress_bar(input1.num_entries as u64);
pb.set_message(tr!("Reading:"));
pb.println(tr!("Reading entries from the first input book."));
for entry in &mut input1 {
map1.entry(entry.key).or_default().push(entry);
pb.inc(1);
}
pb.set_position(0);
pb.set_length(input2.num_entries as u64);
pb.println(tr!("Reading entries from the second input book."));
for entry in &mut input2 {
if let Some(weight_cutoff) = weight_cutoff {
if entry.weight < weight_cutoff {
pb.inc(1);
continue;
}
}
map2.entry(entry.key).or_default().push(entry);
pb.inc(1);
}
pb.set_position(0);
pb.set_length(input2.num_entries as u64);
pb.println(tr!("Merging entries from both books."));
pb.set_message(tr!("Merging:"));
let total_entries1 = input1.num_entries as f64;
let total_entries2 = input2.num_entries as f64;
for (key, entries2) in map2 {
let mut current_weight: f64 = 65520.0;
if let Some(entries1) = map1.get_mut(&key) {
let decrement_step = if strategy == MergeStrategy::Sort {
Some(65520.0 / (entries1.len() + entries2.len()) as f64)
} else {
None
};
let (total_weight1, total_weight2, max_weight_merged) =
if strategy == MergeStrategy::PercentageAverage {
let total_weight2: u64 = entries2.iter().map(|e| e.weight as u64).sum();
let total_weight1: u64 = entries1.iter().map(|e| e.weight as u64).sum();
let max_weight_merged = entries1
.iter()
.map(|e| e.weight)
.chain(entries2.iter().map(|e| e.weight))
.max()
.unwrap_or(0);
(
Some(total_weight1),
Some(total_weight2),
Some(max_weight_merged),
)
} else {
(None, None, None)
};
for mut entry2 in entries2 {
let mut found = false;
for entry1 in &mut *entries1 {
if entry1.mov == entry2.mov {
if let Some(outlier_threshold) = outlier_threshold {
let diff = if entry1.weight > entry2.weight {
entry1.weight - entry2.weight
} else {
entry2.weight - entry1.weight
};
if diff > outlier_threshold {
entry1.weight = 0;
found = true;
break;
}
}
match strategy {
MergeStrategy::AvgWeight => {
entry1.weight = average_weight(entry1, &entry2);
}
MergeStrategy::MaxWeight => {
entry1.weight = entry1.weight.max(entry2.weight);
}
MergeStrategy::MinWeight => {
entry1.weight = entry1.weight.min(entry2.weight);
}
MergeStrategy::Ours => {
}
MergeStrategy::DynamicMidpoint => {
entry1.weight =
dynamic_midpoint_merge(entry1.weight, entry2.weight);
}
MergeStrategy::Entropy => {
entry1.weight = entropy_merge(entry1.weight, entry2.weight);
}
MergeStrategy::GeometricScaling => {
entry1.weight =
geometric_scaling_merge(entry1.weight, entry2.weight);
}
MergeStrategy::HarmonicMean => {
entry1.weight = harmonic_merge(entry1.weight, entry2.weight);
}
MergeStrategy::LogarithmicAverage => {
entry1.weight = logarithmic_merge(entry1.weight, entry2.weight);
}
MergeStrategy::QuadraticMean => {
entry1.weight = quadratic_mean_merge(entry1.weight, entry2.weight);
}
MergeStrategy::WeightedDistance => {
entry1.weight =
weighted_distance_merge(entry1.weight, entry2.weight);
}
MergeStrategy::WeightedMedian => {
entry1.weight = weighted_median_merge(entry1.weight, entry2.weight);
}
MergeStrategy::Sigmoid => {
entry1.weight = sigmoid_merge(entry1.weight, entry2.weight);
}
MergeStrategy::Sort => {
entry1.weight = current_weight as u16;
current_weight -= decrement_step.unwrap();
}
MergeStrategy::SumWeight => {
entry1.weight = entry1.weight.saturating_add(entry2.weight);
}
MergeStrategy::PercentageAverage => {
let total_weight1 = total_weight1.unwrap();
let total_weight2 = total_weight2.unwrap();
let percentage_weight1 =
(entry1.weight as f64 / total_weight1 as f64) * total_entries1;
let percentage_weight2 =
(entry2.weight as f64 / total_weight2 as f64) * total_entries2;
let average_percentage_weight = (percentage_weight1
+ percentage_weight2)
/ (total_entries1 + total_entries2);
entry1.weight = (average_percentage_weight
* max_weight_merged.unwrap() as f64)
as u16;
}
MergeStrategy::WeightedAverageWeight(weight1, weight2) => {
entry1.weight =
weighted_average_weight(entry1, &entry2, weight1, weight2);
} }
found = true;
break;
}
}
if !found && strategy != MergeStrategy::Ours {
if strategy == MergeStrategy::Sort {
entry2.weight = current_weight as u16;
current_weight -= decrement_step.unwrap();
}
entries1.push(entry2);
}
}
} else {
map1.insert(key, entries2);
}
pb.inc(1);
}
let num_entries: usize = map1.values().map(|vec| vec.len()).sum();
pb.set_position(0);
pb.set_length(num_entries as u64);
pb.println(tr!("Writing entries to the output book."));
pb.set_message(tr!("Writing:"));
for entries in map1.values_mut() {
entries.sort_unstable_by_key(|entry| Reverse(entry.weight));
let max_weight = entries[0].weight;
let scale: f64 = 0xfff0 as f64 / max_weight as f64;
for entry in entries {
if rescale_weights {
entry.weight = (scale * entry.weight as f64) as u16;
}
if let Some(weight_cutoff) = weight_cutoff {
if entry.weight < weight_cutoff {
pb.inc(1);
continue;
}
}
bin_entry_to_file(&mut output_file, entry).with_context(|| {
tr!(
"Failed to write to output PolyGlot opening book `{}'.",
output_filename
)
})?;
pb.inc(1);
}
}
pb.finish_with_message(tr!("Writing done."));
command_info(!*STDOUT_IS_TTY, Path::new(output_filename))
.context(tr!("info subcommand failed."))?;
Ok(())
}
fn brainlearn_merge(
_porcelain: bool,
input1_filename: &str,
input2_filename: &str,
output_filename: &str,
strategy: MergeStrategy,
depth_cutoff: Option<u16>,
) -> Result<()> {
let mut input1 = BrainLearnFile::open(input1_filename).with_context(|| {
tr!(
"Failed to open BrainLearn experience file `{}'.",
input1_filename
)
})?;
let mut input2 = BrainLearnFile::open(input2_filename).with_context(|| {
tr!(
"Failed to open BrainLearn experience file `{}'.",
input2_filename
)
})?;
let mut output_file = BufWriter::new(
fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(output_filename)
.with_context(|| {
tr!(
"Failed to create output BrainLearn experience file `{}'.",
output_filename
)
})?,
);
let pb = get_progress_bar(0);
input1.load(depth_cutoff, Some(&pb));
input2.load(depth_cutoff, Some(&pb));
pb.set_position(0);
pb.set_length(input2.num_entries as u64);
pb.println(tr!("Merging entries from both books."));
pb.set_message(tr!("Merging:"));
for (key, entries2) in input2.data {
if let Some(entries1) = input1.data.get_mut(&key) {
for entry2 in entries2 {
let mut found = false;
for entry1 in &mut *entries1 {
if entry1.mov == entry2.mov {
if strategy != MergeStrategy::Ours && entry2.depth > entry1.depth {
entry1.depth = entry2.depth;
entry1.score = entry2.score;
entry1.perf = entry2.perf;
}
found = true;
break;
}
}
if !found && strategy != MergeStrategy::Ours {
entries1.push(entry2);
}
}
} else {
input1.data.insert(key, entries2);
}
pb.inc(1);
}
let num_entries: usize = input1.data.values().map(|vec| vec.len()).sum();
pb.set_position(0);
pb.set_length(num_entries as u64);
pb.println(tr!("Writing entries to the output book."));
pb.set_message(tr!("Writing:"));
for entries in input1.data.values_mut() {
for entry in entries {
exp_entry_to_file(&mut output_file, entry).with_context(|| {
tr!(
"Failed to write to output BrainLearn experience file `{}'.",
output_filename
)
})?;
pb.inc(1);
}
}
pb.finish_with_message(tr!("Writing done."));
command_info(!*STDOUT_IS_TTY, Path::new(output_filename))
.context(tr!("info subcommand failed."))?;
Ok(())
}
fn command_play(
porcelain: bool,
count: u64,
chess960: bool,
candidate_moves: Option<Vec<Uci>>,
epd: &str,
) -> Result<()> {
let epd = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let castling_mode = if chess960 {
CastlingMode::Chess960
} else {
CastlingMode::Standard
};
let position: Chess = epd
.into_position(castling_mode)
.context(tr!("Failed to parse illegal FEN."))?;
let my_color: Color = position.turn();
let candidate_moves: Vec<Move> = match candidate_moves {
Some(cm) => cm
.iter()
.map(|uci| match uci.to_move(&position) {
Ok(move_) => move_,
Err(err) => {
eprintln!("{}", tr!("Illegal UCI move `{}': {}", uci, err));
std::process::exit(1);
}
})
.collect(),
None => position.legal_moves().as_slice().to_vec(),
};
let interrupted = Arc::new(AtomicBool::new(false));
let interrupted_clone = Arc::clone(&interrupted);
ctrlc::set_handler(move || {
interrupted_clone.store(true, Ordering::SeqCst);
})
.context(tr!("Failed to set up interrupt handler."))?;
let pb = get_progress_bar(count);
pb.println(tr!(
"Starting playout with {} candidate moves.",
candidate_moves.len()
));
pb.set_message(tr!("Performing random playout:"));
let mut n: u64 = 0;
let mut wdl = HashMap::new();
for move_ in candidate_moves.into_iter().cycle() {
if interrupted.load(Ordering::SeqCst) {
break;
}
let outcome = play_random_game(&mut position.clone(), &move_);
let uci_move = move_.to_uci(castling_mode);
let entry = wdl.entry(uci_move).or_insert((0, 0, 0));
match outcome {
Outcome::Draw => entry.1 += 1,
Outcome::Decisive { winner } if winner == my_color => entry.0 += 1,
Outcome::Decisive { .. } => entry.2 += 1,
};
pb.inc(1);
if count != 0 {
n = n.saturating_add(1);
if n >= count {
break;
}
}
}
pb.finish_with_message(tr!("Random playout done."));
let mut sorted_wdl: Vec<(&Uci, &(u64, u64, u64))> = wdl.iter().collect();
sorted_wdl.sort_by(|a, b| {
let a_points = a.1 .0 as f64 + a.1 .1 as f64 * 0.5;
let b_points = b.1 .0 as f64 + b.1 .1 as f64 * 0.5;
b_points.partial_cmp(&a_points).unwrap()
});
let (total_wins, total_draws, total_losses) = wdl.values().fold(
(0, 0, 0),
|(acc_wins, acc_draws, acc_losses), &(wins, draws, losses)| {
(acc_wins + wins, acc_draws + draws, acc_losses + losses)
},
);
if porcelain {
println!("{}", tr!("uci,win,draw,loss"));
println!("0000,{},{},{}", total_wins, total_draws, total_losses);
for (uci, &(win, draw, loss)) in &sorted_wdl {
println!("{},{},{},{}", uci, win, draw, loss);
}
} else {
let mut table = Table::new();
table.add_row(Row::new(vec![
Cell::new(&tr!("UCI")).with_style(Attr::Bold),
Cell::new(&tr!("Win")).with_style(Attr::Bold),
Cell::new(&tr!("Draw")).with_style(Attr::Bold),
Cell::new(&tr!("Loss")).with_style(Attr::Bold),
]));
table.add_row(row![
Cell::new("0000"),
Cell::new(&total_wins.to_string()),
Cell::new(&total_draws.to_string()),
Cell::new(&total_losses.to_string()),
]);
for (uci, &(win, draw, loss)) in &sorted_wdl {
table.add_row(row![
Cell::new(&uci.to_string()),
Cell::new(&win.to_string()),
Cell::new(&draw.to_string()),
Cell::new(&loss.to_string()),
]);
}
table.printstd();
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn command_play_book_match(
porcelain: bool,
count: usize,
chess960: bool,
move_selection: MoveSelection,
prefer_irreversible: bool,
book1: Arc<PolyGlotBook>,
book2: Arc<PolyGlotBook>,
epd: &str,
nthreads: usize,
) -> Result<()> {
let epd = Epd::from_ascii(epd.as_bytes()).context(tr!("Failed to parse invalid FEN."))?;
let castling_mode = if chess960 {
CastlingMode::Chess960
} else {
CastlingMode::Standard
};
let position: Chess = epd
.into_position(castling_mode)
.context(tr!("Failed to parse illegal FEN."))?;
let interrupted = Arc::new(AtomicBool::new(false));
let interrupted_clone = Arc::clone(&interrupted);
ctrlc::set_handler(move || {
interrupted_clone.store(true, Ordering::SeqCst);
})
.context(tr!("Failed to set up interrupt handler."))?;
let n = Arc::new(AtomicUsize::new(0));
let wdl = Arc::new(Mutex::new((0, 0, 0)));
let mut threads = Vec::new();
let n_report = Arc::clone(&n);
let wdl_report = Arc::clone(&wdl);
let interrupted_report = Arc::clone(&interrupted);
let total_games = if count == 0 { usize::MAX } else { count };
let report = std::thread::Builder::new()
.name("jja-report".to_string())
.spawn(move || {
let pb = get_progress_bar(count as u64);
pb.set_message(tr!("Playing book match:"));
let mut ngames = 0;
while !interrupted_report.load(Ordering::SeqCst) {
std::thread::sleep(std::time::Duration::from_secs(1));
let my_n = n_report.load(Ordering::SeqCst);
pb.inc((my_n - ngames) as u64);
ngames = my_n;
let wdl = wdl_report.lock().unwrap();
pb.println(tr!(
"Score: Book1 +{} ={} -{} Book2 => {}",
wdl.0,
wdl.1,
wdl.2,
wdl.0 - wdl.2
));
if ngames >= total_games {
break;
}
}
pb.finish_with_message(tr!("Book match done."));
})
.context(tr!("Failed to spawn jja-report thread."))?;
for i in 0..nthreads {
let n = Arc::clone(&n);
let wdl = Arc::clone(&wdl);
let interrupted = Arc::clone(&interrupted);
let position = position.clone();
let book1 = Arc::clone(&book1);
let book2 = Arc::clone(&book2);
let thread = std::thread::Builder::new()
.name("jja-rand".to_string())
.spawn(move || {
let mut book1_starts_game = i % 2 == 0;
let chunk_size = if count == 0 {
usize::MAX
} else {
count / nthreads + if i < count % nthreads { 1 } else { 0 }
};
for _ in 0..chunk_size {
if interrupted.load(Ordering::SeqCst) {
break;
}
let outcome = if book1_starts_game {
play_random_book_game(
&position,
Arc::clone(&book1),
Arc::clone(&book2),
&move_selection,
prefer_irreversible,
)
.unwrap()
} else {
play_random_book_game(
&position,
Arc::clone(&book2),
Arc::clone(&book1),
&move_selection,
prefer_irreversible,
)
.unwrap()
};
let mut wdl = wdl.lock().unwrap();
if book1_starts_game {
match outcome {
Outcome::Draw => wdl.1 += 1,
Outcome::Decisive { winner } if winner == position.turn() => wdl.0 += 1,
Outcome::Decisive { .. } => wdl.2 += 1,
};
} else {
match outcome {
Outcome::Draw => wdl.1 += 1,
Outcome::Decisive { winner } if winner != position.turn() => wdl.0 += 1,
Outcome::Decisive { .. } => wdl.2 += 1,
};
}
book1_starts_game = !book1_starts_game;
n.fetch_add(1, Ordering::SeqCst);
}
})
.with_context(|| tr!("Failed to spawn jja-rand thread `{}'.", i))?;
threads.push(thread);
}
for thread in threads {
thread.join().expect(&tr!("Error joining jja-rand thread."));
}
report
.join()
.expect(&tr!("Error joining jja-report thread."));
let wdl = Arc::try_unwrap(wdl).unwrap().into_inner().unwrap();
let ngames = wdl.0 + wdl.1 + wdl.2;
if porcelain {
println!("{}", tr!("book,games,win,draw,loss,score"));
println!(
"{}",
tr!(
"book1,{},{},{},{},{}",
ngames,
wdl.0,
wdl.1,
wdl.2,
wdl.0 - wdl.2
)
);
println!(
"{}",
tr!(
"book2,{},{},{},{},{}",
ngames,
wdl.2,
wdl.1,
wdl.0,
wdl.2 - wdl.0
)
);
} else {
let table = table!(
[tr!("Book"), "Games", "Win", "Draw", "Loss", "Score"],
[
tr!("Book 1"),
ngames.to_string(),
wdl.0.to_string(),
wdl.1.to_string(),
wdl.2.to_string(),
(wdl.0 - wdl.2).to_string(),
],
[
tr!("Book 2"),
ngames.to_string(),
wdl.2.to_string(),
wdl.1.to_string(),
wdl.0.to_string(),
(wdl.2 - wdl.0).to_string(),
]
);
table.printstd();
}
Ok(())
}