encode_unicode/
utf16_iterators.rs

1/* Copyright 2018-2019 Torbjørn Birch Moltu
2 *
3 * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4 * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5 * http://opensource.org/licenses/MIT>, at your option. This file may not be
6 * copied, modified, or distributed except according to those terms.
7 */
8
9use crate::traits::CharExt;
10use crate::utf16_char::Utf16Char;
11use crate::errors::EmptyStrError;
12extern crate core;
13use core::fmt;
14use core::borrow::Borrow;
15
16// Invalid values that says the field is consumed or empty.
17const FIRST_USED: u16 = 0x_dc_00;
18const SECOND_USED: u16 = 0;
19
20/// Iterate over the units of the UTF-16 representation of a codepoint.
21#[derive(Clone)]
22pub struct Utf16Iterator {
23    first: u16,
24    second: u16,
25}
26impl From<char> for Utf16Iterator {
27    fn from(c: char) -> Self {
28        Self::from(c.to_utf16())
29    }
30}
31impl From<Utf16Char> for Utf16Iterator {
32    fn from(uc: Utf16Char) -> Self {
33        let (first, second) = uc.to_tuple();
34        let second = second.unwrap_or(SECOND_USED);
35        Utf16Iterator{first, second}
36    }
37}
38impl Iterator for Utf16Iterator {
39    type Item=u16;
40    fn next(&mut self) -> Option<u16> {
41        match (self.first, self.second) {
42            (FIRST_USED, SECOND_USED)  =>  {                            None        },
43            (FIRST_USED, second     )  =>  {self.second = SECOND_USED;  Some(second)},
44            (first     ,      _     )  =>  {self.first = FIRST_USED;    Some(first )},
45        }
46    }
47    fn size_hint(&self) -> (usize, Option<usize>) {
48        (self.len(), Some(self.len()))
49    }
50}
51impl ExactSizeIterator for Utf16Iterator {
52    fn len(&self) -> usize {
53        (if self.first == FIRST_USED {0} else {1}) +
54        (if self.second == SECOND_USED {0} else {1})
55    }
56}
57impl fmt::Debug for Utf16Iterator {
58    fn fmt(&self,  fmtr: &mut fmt::Formatter) -> fmt::Result {
59        let mut clone = self.clone();
60        match (clone.next(), clone.next()) {
61            (Some(one), None)  => write!(fmtr, "[{}]", one),
62            (Some(a), Some(b)) => write!(fmtr, "[{}, {}]", a, b),
63            (None,  _)         => write!(fmtr, "[]"),
64        }
65    }
66}
67
68
69
70/// Converts an iterator of `Utf16Char` (or `&Utf16Char`)
71/// to an iterator of `u16`s.
72///
73/// Is equivalent to calling `.flatten()` or `.flat_map()` on the original iterator,
74/// but the returned iterator is about twice as fast.
75///
76/// The exact number of units cannot be known in advance, but `size_hint()`
77/// gives the possible range.
78///
79/// # Examples
80///
81/// From iterator of values:
82///
83/// ```
84/// use encode_unicode::{IterExt, CharExt};
85///
86/// let iterator = "foo".chars().map(|c| c.to_utf16() );
87/// let mut units = [0; 4];
88/// iterator.to_units().zip(&mut units).for_each(|(u,dst)| *dst = u );
89/// assert_eq!(units, ['f' as u16, 'o' as u16, 'o' as u16, 0]);
90/// ```
91///
92/// From iterator of references:
93///
94#[cfg_attr(feature="std", doc=" ```")]
95#[cfg_attr(not(feature="std"), doc=" ```no_compile")]
96/// use encode_unicode::{IterExt, CharExt, Utf16Char};
97///
98/// // (💣 takes two units)
99/// let chars: Vec<Utf16Char> = "💣 bomb 💣".chars().map(|c| c.to_utf16() ).collect();
100/// let units: Vec<u16> = chars.iter().to_units().collect();
101/// let flat_map: Vec<u16> = chars.iter().cloned().flatten().collect();
102/// assert_eq!(units, flat_map);
103/// ```
104#[derive(Clone)]
105pub struct Utf16CharSplitter<U:Borrow<Utf16Char>, I:Iterator<Item=U>> {
106    inner: I,
107    prev_second: u16,
108}
109impl<U:Borrow<Utf16Char>, I:IntoIterator<Item=U>>
110From<I> for Utf16CharSplitter<U, I::IntoIter> {
111    fn from(iterable: I) -> Self {
112        Utf16CharSplitter { inner: iterable.into_iter(),  prev_second: 0 }
113    }
114}
115impl<U:Borrow<Utf16Char>, I:Iterator<Item=U>> Utf16CharSplitter<U,I> {
116    /// Extracts the source iterator.
117    ///
118    /// Note that `iter.into_inner().to_units()` is not a no-op:  
119    /// If the last returned unit from `next()` was a leading surrogate,
120    /// the trailing surrogate is lost.
121    pub fn into_inner(self) -> I {
122        self.inner
123    }
124}
125impl<U:Borrow<Utf16Char>, I:Iterator<Item=U>> Iterator for Utf16CharSplitter<U,I> {
126    type Item = u16;
127    fn next(&mut self) -> Option<Self::Item> {
128        if self.prev_second == 0 {
129            self.inner.next().map(|u16c| {
130                let units = u16c.borrow().to_array();
131                self.prev_second = units[1];
132                units[0]
133            })
134        } else {
135            let prev_second = self.prev_second;
136            self.prev_second = 0;
137            Some(prev_second)
138        }
139    }
140    fn size_hint(&self) -> (usize,Option<usize>) {
141        // Doesn't need to handle unlikely overflows correctly because
142        // size_hint() cannot be relied upon anyway. (the trait isn't unsafe)
143        let (min, max) = self.inner.size_hint();
144        let add = if self.prev_second == 0 {0} else {1};
145        (min.wrapping_add(add), max.map(|max| max.wrapping_mul(2).wrapping_add(add) ))
146    }
147}
148
149
150
151/// An iterator over the codepoints in a `str` represented as `Utf16Char`.
152#[derive(Clone)]
153pub struct Utf16CharIndices<'a>{
154    str: &'a str,
155    index: usize,
156}
157impl<'a> From<&'a str> for Utf16CharIndices<'a> {
158    fn from(s: &str) -> Utf16CharIndices {
159        Utf16CharIndices{str: s, index: 0}
160    }
161}
162impl<'a> Utf16CharIndices<'a> {
163    /// Extract the remainder of the source `str`.
164    ///
165    /// # Examples
166    ///
167    /// ```
168    /// use encode_unicode::{StrExt, Utf16Char};
169    /// let mut iter = "abc".utf16char_indices();
170    /// assert_eq!(iter.next_back(), Some((2, Utf16Char::from('c'))));
171    /// assert_eq!(iter.next(), Some((0, Utf16Char::from('a'))));
172    /// assert_eq!(iter.as_str(), "b");
173    /// ```
174    pub fn as_str(&self) -> &'a str {
175        &self.str[self.index..]
176    }
177}
178impl<'a> Iterator for Utf16CharIndices<'a> {
179    type Item = (usize,Utf16Char);
180    fn next(&mut self) -> Option<(usize,Utf16Char)> {
181        match Utf16Char::from_str_start(&self.str[self.index..]) {
182            Ok((u16c, bytes)) => {
183                let item = (self.index, u16c);
184                self.index += bytes;
185                Some(item)
186            },
187            Err(EmptyStrError) => None
188        }
189    }
190    fn size_hint(&self) -> (usize,Option<usize>) {
191        let len = self.str.len() - self.index;
192        // For len+3 to overflow, the slice must fill all but two bytes of
193        // addressable memory, and size_hint() doesn't need to be correct.
194        (len.wrapping_add(3)/4, Some(len))
195    }
196}
197impl<'a> DoubleEndedIterator for Utf16CharIndices<'a> {
198    fn next_back(&mut self) -> Option<(usize,Utf16Char)> {
199        if self.index < self.str.len() {
200            let rev = self.str.bytes().rev();
201            let len = 1 + rev.take_while(|b| b & 0b1100_0000 == 0b1000_0000 ).count();
202            let starts = self.str.len() - len;
203            let (u16c,_) = Utf16Char::from_str_start(&self.str[starts..]).unwrap();
204            self.str = &self.str[..starts];
205            Some((starts, u16c))
206        } else {
207            None
208        }
209    }
210}
211impl<'a> fmt::Debug for Utf16CharIndices<'a> {
212    fn fmt(&self,  fmtr: &mut fmt::Formatter) -> fmt::Result {
213        fmtr.debug_tuple("Utf16CharIndices")
214            .field(&self.index)
215            .field(&self.as_str())
216            .finish()
217    }
218}
219
220
221/// An iterator over the codepoints in a `str` represented as `Utf16Char`.
222#[derive(Clone)]
223pub struct Utf16Chars<'a>(Utf16CharIndices<'a>);
224impl<'a> From<&'a str> for Utf16Chars<'a> {
225    fn from(s: &str) -> Utf16Chars {
226        Utf16Chars(Utf16CharIndices::from(s))
227    }
228}
229impl<'a> Utf16Chars<'a> {
230    /// Extract the remainder of the source `str`.
231    ///
232    /// # Examples
233    ///
234    /// ```
235    /// use encode_unicode::{StrExt, Utf16Char};
236    /// let mut iter = "abc".utf16chars();
237    /// assert_eq!(iter.next(), Some(Utf16Char::from('a')));
238    /// assert_eq!(iter.next_back(), Some(Utf16Char::from('c')));
239    /// assert_eq!(iter.as_str(), "b");
240    /// ```
241    pub fn as_str(&self) -> &'a str {
242        self.0.as_str()
243    }
244}
245impl<'a> Iterator for Utf16Chars<'a> {
246    type Item = Utf16Char;
247    fn next(&mut self) -> Option<Utf16Char> {
248        self.0.next().map(|(_,u16c)| u16c )
249    }
250    fn size_hint(&self) -> (usize,Option<usize>) {
251        self.0.size_hint()
252    }
253}
254impl<'a> DoubleEndedIterator for Utf16Chars<'a> {
255    fn next_back(&mut self) -> Option<Utf16Char> {
256        self.0.next_back().map(|(_,u16c)| u16c )
257    }
258}
259impl<'a> fmt::Debug for Utf16Chars<'a> {
260    fn fmt(&self,  fmtr: &mut fmt::Formatter) -> fmt::Result {
261        fmtr.debug_tuple("Utf16Chars")
262            .field(&self.as_str())
263            .finish()
264    }
265}
encode_unicode/utf16_iterators.rs

encode_unicode/
utf16_iterators.rs
