lindera_dictionary/loader/
prefix_dictionary.rs

1use std::path::Path;
2
3use crate::LinderaResult;
4#[cfg(feature = "compress")]
5use crate::decompress::{CompressedData, decompress};
6use crate::dictionary::prefix_dictionary::PrefixDictionary;
7#[cfg(feature = "compress")]
8use crate::error::LinderaErrorKind;
9#[cfg(feature = "mmap")]
10use crate::util::mmap_file;
11use crate::util::read_file;
12
13pub struct PrefixDictionaryLoader {}
14
15impl PrefixDictionaryLoader {
16    #[allow(unused_mut)]
17    pub fn load(input_dir: &Path) -> LinderaResult<PrefixDictionary> {
18        let mut da_data = read_file(input_dir.join("dict.da").as_path())?;
19        let mut vals_data = read_file(input_dir.join("dict.vals").as_path())?;
20        let mut words_idx_data = read_file(input_dir.join("dict.wordsidx").as_path())?;
21        let mut words_data = read_file(input_dir.join("dict.words").as_path())?;
22
23        #[cfg(feature = "compress")]
24        {
25            let mut aligned = rkyv::util::AlignedVec::<16>::new();
26            aligned.extend_from_slice(&da_data);
27            let compressed_data: CompressedData =
28                rkyv::from_bytes::<CompressedData, rkyv::rancor::Error>(&aligned).map_err(
29                    |err| {
30                        LinderaErrorKind::Deserialize
31                            .with_error(anyhow::anyhow!(err.to_string()))
32                            .add_context("Failed to deserialize dict.da data")
33                    },
34                )?;
35            da_data = decompress(compressed_data).map_err(|err| {
36                LinderaErrorKind::Compression
37                    .with_error(err)
38                    .add_context("Failed to decompress dict.da DoubleArray data")
39            })?;
40        }
41        #[cfg(feature = "compress")]
42        {
43            let mut aligned = rkyv::util::AlignedVec::<16>::new();
44            aligned.extend_from_slice(&vals_data);
45            let compressed_data: CompressedData =
46                rkyv::from_bytes::<CompressedData, rkyv::rancor::Error>(&aligned).map_err(
47                    |err| {
48                        LinderaErrorKind::Deserialize
49                            .with_error(anyhow::anyhow!(err.to_string()))
50                            .add_context("Failed to deserialize dict.vals data")
51                    },
52                )?;
53            vals_data = decompress(compressed_data).map_err(|err| {
54                LinderaErrorKind::Compression
55                    .with_error(err)
56                    .add_context("Failed to decompress dict.vals word values data")
57            })?;
58        }
59        #[cfg(feature = "compress")]
60        {
61            let mut aligned = rkyv::util::AlignedVec::<16>::new();
62            aligned.extend_from_slice(&words_idx_data);
63            let compressed_data: CompressedData =
64                rkyv::from_bytes::<CompressedData, rkyv::rancor::Error>(&aligned).map_err(
65                    |err| {
66                        LinderaErrorKind::Deserialize
67                            .with_error(anyhow::anyhow!(err.to_string()))
68                            .add_context("Failed to deserialize dict.wordsidx data")
69                    },
70                )?;
71            words_idx_data = decompress(compressed_data).map_err(|err| {
72                LinderaErrorKind::Compression
73                    .with_error(err)
74                    .add_context("Failed to decompress dict.wordsidx word index data")
75            })?;
76        }
77        #[cfg(feature = "compress")]
78        {
79            let mut aligned = rkyv::util::AlignedVec::<16>::new();
80            aligned.extend_from_slice(&words_data);
81            let compressed_data: CompressedData =
82                rkyv::from_bytes::<CompressedData, rkyv::rancor::Error>(&aligned).map_err(
83                    |err| {
84                        LinderaErrorKind::Deserialize
85                            .with_error(anyhow::anyhow!(err.to_string()))
86                            .add_context("Failed to deserialize dict.words data")
87                    },
88                )?;
89            words_data = decompress(compressed_data).map_err(|err| {
90                LinderaErrorKind::Compression
91                    .with_error(err)
92                    .add_context("Failed to decompress dict.words word details data")
93            })?;
94        }
95
96        Ok(PrefixDictionary::load(
97            da_data,
98            vals_data,
99            words_idx_data,
100            words_data,
101            true,
102        ))
103    }
104
105    #[cfg(feature = "mmap")]
106    pub fn load_mmap(input_dir: &Path) -> LinderaResult<PrefixDictionary> {
107        let da_data = mmap_file(input_dir.join("dict.da").as_path())?;
108        let vals_data = mmap_file(input_dir.join("dict.vals").as_path())?;
109        let words_idx_data = mmap_file(input_dir.join("dict.wordsidx").as_path())?;
110        let words_data = mmap_file(input_dir.join("dict.words").as_path())?;
111
112        Ok(PrefixDictionary::load(
113            da_data,
114            vals_data,
115            words_idx_data,
116            words_data,
117            true,
118        ))
119    }
120}
lindera_dictionary/loader/prefix_dictionary.rs

lindera_dictionary/loader/
prefix_dictionary.rs
