encoding_mel/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2#![doc = include_str!("../README.md")]
3
4#[cfg(any(
5    all(feature = "real", feature = "mock"),
6    not(any(feature = "real", feature = "mock"))
7))]
8compile_error!("One of the two features 'real' or 'mock' must be enabled");
9
10use melodium_core::*;
11use melodium_macro::{check, mel_package, mel_treatment};
12
13/// Decodes stream of bytes into string.
14///
15/// The incoming stream of bytes is decoded using the specified encoding.
16/// If some characters cannot be decoded for some reason (i.e. invalid according to encoding),
17/// it is replaced by the `U+FFFD REPLACEMENT CHARACTER` (�).
18///
19/// The supported encodings and possible values for `encoding` are defined in
20/// the [Encoding Standard](https://encoding.spec.whatwg.org/#names-and-labels).
21/// If `encoding` is not recognized, UTF-8 is assumed instead.
22#[mel_treatment(
23    default encoding "utf-8"
24    input data Stream<byte>
25    output text Stream<string>
26)]
27pub async fn decode(encoding: string) {
28    let encoding =
29        encoding_rs::Encoding::for_label(encoding.as_bytes()).unwrap_or(encoding_rs::UTF_8);
30    let mut decoder = encoding.new_decoder();
31
32    let mut finished = false;
33    while !finished {
34        let bytes;
35        if let Ok(data) = data
36            .recv_many()
37            .await
38            .map(|values| TryInto::<Vec<u8>>::try_into(values).unwrap())
39        {
40            bytes = data;
41        } else {
42            bytes = vec![];
43            finished = true;
44        }
45
46        let mut result = String::with_capacity(bytes.len() * 2);
47
48        let _ = decoder.decode_to_string(&bytes, &mut result, finished);
49
50        result.shrink_to_fit();
51
52        check!(text.send_one(result.into()).await);
53    }
54}
55
56/// Encodes streamed text with specified encoding.
57///
58/// The incoming string is encoded and outputted into raw bytes. If some characters cannot
59/// be encoded into the specified `encoding`, the behavior is set by `replace`:
60/// - `false`: the character is dropped;
61/// - `true`: the character is replaced with coded XML character (such as `&#65533;`).
62///
63/// The supported encodings and possible values for `encoding` are defined in
64/// the [Encoding Standard](https://encoding.spec.whatwg.org/#names-and-labels).
65/// If `encoding` is not recognized, UTF-8 is assumed instead.
66#[mel_treatment(
67    default encoding "utf-8"
68    default replace false
69    input text Stream<string>
70    output data Stream<byte>
71)]
72pub async fn encode(encoding: string, replace: bool) {
73    let encoding =
74        encoding_rs::Encoding::for_label(encoding.as_bytes()).unwrap_or(encoding_rs::UTF_8);
75    let mut encoder = encoding.new_encoder();
76
77    'main: while let Ok(text) = text
78        .recv_many()
79        .await
80        .map(|values| TryInto::<Vec<string>>::try_into(values).unwrap())
81    {
82        for text in text {
83            let expected_size = if replace {
84                7 * encoder
85                    .max_buffer_length_from_utf8_if_no_unmappables(text.len())
86                    .unwrap_or(2_usize.pow(20))
87            } else {
88                encoder
89                    .max_buffer_length_from_utf8_without_replacement(text.len())
90                    .unwrap_or(2_usize.pow(20))
91            };
92
93            let mut result = Vec::new();
94            result.reserve(expected_size);
95
96            if replace {
97                let _ = encoder.encode_from_utf8_to_vec(&text, &mut result, false);
98            } else {
99                let _ =
100                    encoder.encode_from_utf8_to_vec_without_replacement(&text, &mut result, false);
101            }
102
103            result.shrink_to_fit();
104            check!('main, data.send_many(TransmissionValue::Byte(result.into())).await);
105        }
106    }
107
108    let expected_size = if replace {
109        7 * encoder
110            .max_buffer_length_from_utf8_if_no_unmappables(0)
111            .unwrap_or(2_usize.pow(6))
112    } else {
113        encoder
114            .max_buffer_length_from_utf8_without_replacement(0)
115            .unwrap_or(2_usize.pow(6))
116    };
117
118    let mut result = Vec::new();
119    result.reserve(expected_size);
120
121    if replace {
122        let _ = encoder.encode_from_utf8_to_vec(&String::new(), &mut result, false);
123    } else {
124        let _ =
125            encoder.encode_from_utf8_to_vec_without_replacement(&String::new(), &mut result, false);
126    }
127
128    result.shrink_to_fit();
129    let _ = data.send_many(TransmissionValue::Byte(result.into())).await;
130}
131
132mel_package!();
encoding_mel/lib.rs

encoding_mel/
lib.rs
