rustfst 1.3.0

Library for constructing, combining, optimizing, and searching weighted finite-state transducers (FSTs).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

use std::fs::read;
use std::path::Path;
use std::sync::Arc;

use anyhow::{Context, Result};
use nom::bytes::complete::take;
use nom::multi::count;
use nom::number::complete::le_i32;
use nom::IResult;

use crate::fst_impls::const_fst::{
    CONST_ALIGNED_FILE_VERSION, CONST_ARCH_ALIGNMENT, CONST_MIN_FILE_VERSION,
};
use crate::fst_impls::vector_fst::VectorFstState;
use crate::fst_impls::{ConstFst, VectorFst};
use crate::fst_properties::FstProperties;
use crate::fst_traits::SerializableFst;
use crate::parsers::bin_fst::fst_header::FstHeader;
use crate::parsers::bin_fst::utils_parsing::{
    parse_bin_fst_tr, parse_final_weight, parse_start_state,
};
use crate::parsers::nom_utils::NomCustomError;
use crate::semirings::SerializableSemiring;
use crate::{Tr, TrsVec};

impl<W: SerializableSemiring> VectorFst<W> {
    /// Load a VectorFst directly from a ConstFst file.
    pub fn read_from_const<P: AsRef<Path>>(path_bin_fst: P) -> Result<Self> {
        let data = read(path_bin_fst.as_ref()).with_context(|| {
            format!(
                "Can't open ConstFst binary file : {:?}",
                path_bin_fst.as_ref()
            )
        })?;

        let (_, parsed_fst) = parse_const_fst(&data)
            .map_err(|_| format_err!("Error while parsing binary ConstFst file as a VectorFst"))?;

        Ok(parsed_fst)
    }
}

struct TempState<W> {
    final_weight: Option<W>,
    ntrs: usize,
    niepsilons: usize,
    noepsilons: usize,
}

fn parse_const_state<W: SerializableSemiring>(
    i: &[u8],
) -> IResult<&[u8], TempState<W>, NomCustomError<&[u8]>> {
    let (i, final_weight) = W::parse_binary(i)?;
    let (i, _pos) = le_i32(i)?;
    let (i, ntrs) = le_i32(i)?;
    let (i, niepsilons) = le_i32(i)?;
    let (i, noepsilons) = le_i32(i)?;

    Ok((
        i,
        TempState {
            final_weight: parse_final_weight(final_weight),
            ntrs: ntrs as usize,
            niepsilons: niepsilons as usize,
            noepsilons: noepsilons as usize,
        },
    ))
}

fn parse_const_fst<W: SerializableSemiring>(
    i: &[u8],
) -> IResult<&[u8], VectorFst<W>, NomCustomError<&[u8]>> {
    let stream_len = i.len();

    let (mut i, hdr) = FstHeader::parse(
        i,
        CONST_MIN_FILE_VERSION,
        // Intentional as the ConstFst file is being parsed.
        ConstFst::<W>::fst_type(),
        Tr::<W>::tr_type(),
    )?;
    let aligned = hdr.version == CONST_ALIGNED_FILE_VERSION;
    let pos = stream_len - i.len();

    // Align input
    if aligned && hdr.num_states > 0 && !pos.is_multiple_of(CONST_ARCH_ALIGNMENT) {
        i = take(CONST_ARCH_ALIGNMENT - (pos % CONST_ARCH_ALIGNMENT))(i)?.0;
    }
    let (mut i, temp_states) = count(parse_const_state::<W>, hdr.num_states as usize)(i)?;
    let pos = stream_len - i.len();

    // Align input
    if aligned && hdr.num_trs > 0 && !pos.is_multiple_of(CONST_ARCH_ALIGNMENT) {
        i = take(CONST_ARCH_ALIGNMENT - (pos % CONST_ARCH_ALIGNMENT))(i)?.0;
    }

    let mut vector_states = Vec::with_capacity(temp_states.len());
    for temp_state in temp_states {
        let (j, trs) = count(parse_bin_fst_tr, temp_state.ntrs)(i)?;
        i = j;
        vector_states.push(VectorFstState {
            final_weight: temp_state.final_weight,
            trs: TrsVec(Arc::new(trs)),
            niepsilons: temp_state.niepsilons,
            noepsilons: temp_state.noepsilons,
        });
    }

    Ok((
        i,
        VectorFst {
            start_state: parse_start_state(hdr.start),
            states: vector_states,
            isymt: hdr.isymt,
            osymt: hdr.osymt,
            properties: FstProperties::from_bits_truncate(hdr.properties),
        },
    ))
}