lindera_analysis/lib.rs
1//! Text analysis chain for Lindera.
2//!
3//! This crate layers Lucene-style text analysis on top of the pure
4//! morphological segmenter provided by the [`lindera`] crate:
5//!
6//! - [`character_filter`]: transforms the input text before segmentation
7//! (with offset correction back to the original text)
8//! - [`token_filter`]: transforms the tokens produced by the segmenter
9//! - [`tokenizer`]: composes character filters, a
10//! [`Segmenter`](lindera::segmenter::Segmenter), and token filters into a
11//! single pipeline, configurable programmatically or via a YAML file
12
13pub mod character_filter;
14pub mod token_filter;
15pub mod tokenizer;
16pub mod worker;
17
18use serde_json::Value;
19
20use lindera::LinderaResult;
21use lindera::error::LinderaErrorKind;
22
23/// Parses a CLI-style filter flag of the form `kind:{"arg": ...}` into the
24/// filter kind and its JSON arguments.
25///
26/// # Arguments
27///
28/// * `cli_flag` - The flag string, e.g. `lowercase` or `length:{"max": 10}`.
29///
30/// # Returns
31///
32/// A tuple of the filter kind and the parsed JSON arguments.
33fn parse_cli_flag(cli_flag: &str) -> LinderaResult<(&str, Value)> {
34 let (kind, json) = cli_flag.split_once(':').unwrap_or((cli_flag, ""));
35
36 let args: Value = serde_json::from_str(json)
37 .map_err(|err| LinderaErrorKind::Content.with_error(anyhow::anyhow!(err)))?;
38
39 Ok((kind, args))
40}