lindera_analysis/lib.rs
1//! Text analysis chain for Lindera.
2//!
3//! This crate layers Lucene-style text analysis on top of the pure
4//! morphological segmenter provided by the [`lindera`] crate:
5//!
6//! - [`character_filter`]: transforms the input text before segmentation
7//! (with offset correction back to the original text)
8//! - [`token_filter`]: transforms the tokens produced by the segmenter
9//! - [`tokenizer`]: composes character filters, a
10//! [`Segmenter`](lindera::segmenter::Segmenter), and token filters into a
11//! single pipeline, configurable programmatically or via a YAML file
12
13pub mod character_filter;
14pub mod token_filter;
15pub mod tokenizer;
16
17use serde_json::Value;
18
19use lindera::LinderaResult;
20use lindera::error::LinderaErrorKind;
21
22/// Parses a CLI-style filter flag of the form `kind:{"arg": ...}` into the
23/// filter kind and its JSON arguments.
24///
25/// # Arguments
26///
27/// * `cli_flag` - The flag string, e.g. `lowercase` or `length:{"max": 10}`.
28///
29/// # Returns
30///
31/// A tuple of the filter kind and the parsed JSON arguments.
32fn parse_cli_flag(cli_flag: &str) -> LinderaResult<(&str, Value)> {
33 let (kind, json) = cli_flag.split_once(':').unwrap_or((cli_flag, ""));
34
35 let args: Value = serde_json::from_str(json)
36 .map_err(|err| LinderaErrorKind::Content.with_error(anyhow::anyhow!(err)))?;
37
38 Ok((kind, args))
39}