lindera_dictionary/builder/
unknown_dictionary.rs

1use std::borrow::Cow;
2use std::fs::File;
3use std::io::{self, Write};
4use std::path::Path;
5use std::sync::Arc;
6
7use log::debug;
8
9use crate::LinderaResult;
10use crate::dictionary::character_definition::CharacterDefinition;
11use crate::dictionary::context_id_map::ContextIdMap;
12use crate::dictionary::unknown_dictionary::parse_unk;
13use crate::error::LinderaErrorKind;
14use crate::util::{read_file_with_encoding, write_data};
15
16#[derive(Debug)]
17pub struct UnknownDictionaryBuilder {
18    encoding: Cow<'static, str>,
19    /// Optional connection-cost context-ID remap, applied to each unknown-word
20    /// entry's `left_id`/`right_id` so they match the remapped connection matrix.
21    context_id_remap: Option<Arc<ContextIdMap>>,
22}
23
24/// Options for [`UnknownDictionaryBuilder`]. Every field has a default, so
25/// [`Self::builder`] is infallible.
26#[derive(Debug, Default)]
27pub struct UnknownDictionaryBuilderOptions {
28    encoding: Option<Cow<'static, str>>,
29    context_id_remap: Option<Arc<ContextIdMap>>,
30}
31
32impl UnknownDictionaryBuilderOptions {
33    pub fn encoding(&mut self, value: impl Into<Cow<'static, str>>) -> &mut Self {
34        self.encoding = Some(value.into());
35        self
36    }
37
38    pub fn context_id_remap(&mut self, value: Option<Arc<ContextIdMap>>) -> &mut Self {
39        self.context_id_remap = value;
40        self
41    }
42
43    pub fn builder(&self) -> UnknownDictionaryBuilder {
44        UnknownDictionaryBuilder {
45            encoding: self.encoding.clone().unwrap_or_else(|| "UTF-8".into()),
46            context_id_remap: self.context_id_remap.clone(),
47        }
48    }
49}
50
51impl UnknownDictionaryBuilder {
52    pub fn build(
53        &self,
54        input_dir: &Path,
55        chardef: &CharacterDefinition,
56        output_dir: &Path,
57    ) -> LinderaResult<()> {
58        let unk_data_path = input_dir.join("unk.def");
59        debug!("reading {unk_data_path:?}");
60        let unk_data = read_file_with_encoding(&unk_data_path, &self.encoding)?;
61        let unknown_dictionary = parse_unk(
62            chardef.categories(),
63            &unk_data,
64            self.context_id_remap.as_deref(),
65        )?;
66
67        let mut unk_buffer = Vec::new();
68        let bytes = rkyv::to_bytes::<rkyv::rancor::Error>(&unknown_dictionary).map_err(|err| {
69            LinderaErrorKind::Serialize
70                .with_error(anyhow::anyhow!(err))
71                .add_context("Failed to serialize unknown dictionary data")
72        })?;
73        unk_buffer.write_all(&bytes).map_err(|err| {
74            LinderaErrorKind::Io
75                .with_error(anyhow::anyhow!(err))
76                .add_context("Failed to write unknown dictionary data to buffer")
77        })?;
78
79        let wtr_unk_path = output_dir.join(Path::new("unk.bin"));
80        let mut wtr_unk = io::BufWriter::new(
81            File::create(wtr_unk_path)
82                .map_err(|err| LinderaErrorKind::Io.with_error(anyhow::anyhow!(err)))?,
83        );
84
85        write_data(&unk_buffer, &mut wtr_unk)?;
86
87        wtr_unk
88            .flush()
89            .map_err(|err| LinderaErrorKind::Io.with_error(anyhow::anyhow!(err)))?;
90
91        Ok(())
92    }
93}
lindera_dictionary/builder/unknown_dictionary.rs

lindera_dictionary/builder/
unknown_dictionary.rs
