logreduce_tokenizer/
lib.rs

1// Copyright (C) 2022 Red Hat
2// SPDX-License-Identifier: Apache-2.0
3
4#![warn(missing_docs)]
5
6//! This library provides a tokenizer function for the [logreduce](https://github.com/logreduce/logreduce) project.
7//!
8//! The goal is to replace varying words with fixed tokens (e.g. `sha256://...` is converted to `%HASH`).
9
10use pyo3::prelude::*;
11
12pub mod tokenizer;
13
14/// The python function
15#[pyfunction]
16fn process(line: &str) -> String {
17    tokenizer::process(line)
18}
19
20/// The python module
21#[pymodule]
22fn logreduce_tokenizer(_py: Python, m: &PyModule) -> PyResult<()> {
23    m.add_function(wrap_pyfunction!(process, m)?)?;
24
25    Ok(())
26}