winnow/token/
mod.rs

1//! Parsers extracting tokens from the stream
2
3#[cfg(all(test, feature = "ascii"))]
4mod tests;
5
6use crate::combinator::trace;
7use crate::combinator::DisplayDebug;
8use crate::error::Needed;
9use crate::error::ParserError;
10use crate::stream::Range;
11use crate::stream::{Compare, CompareResult, ContainsToken, FindSlice, Stream};
12use crate::stream::{StreamIsPartial, ToUsize};
13use crate::Parser;
14use crate::Result;
15use core::result::Result::Ok;
16
17/// Matches one token
18///
19/// *Complete version*: Will return an error if there's not enough input data.
20///
21/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
22///
23/// # Effective Signature
24///
25/// Assuming you are parsing a `&str` [Stream]:
26/// ```rust
27/// # use winnow::prelude::*;;
28/// pub fn any(input: &mut &str) -> ModalResult<char>
29/// # {
30/// #     winnow::token::any.parse_next(input)
31/// # }
32/// ```
33///
34/// # Example
35///
36/// ```rust
37/// # use winnow::{token::any, error::ErrMode, error::ContextError};
38/// # use winnow::prelude::*;
39/// fn parser(input: &mut &str) -> ModalResult<char> {
40///     any.parse_next(input)
41/// }
42///
43/// assert_eq!(parser.parse_peek("abc"), Ok(("bc",'a')));
44/// assert!(parser.parse_peek("").is_err());
45/// ```
46///
47/// ```rust
48/// # use winnow::{token::any, error::ErrMode, error::ContextError, error::Needed};
49/// # use winnow::prelude::*;
50/// # use winnow::Partial;
51/// assert_eq!(any::<_, ErrMode<ContextError>>.parse_peek(Partial::new("abc")), Ok((Partial::new("bc"),'a')));
52/// assert_eq!(any::<_, ErrMode<ContextError>>.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
53/// ```
54#[inline(always)]
55#[doc(alias = "token")]
56pub fn any<Input, Error>(input: &mut Input) -> Result<<Input as Stream>::Token, Error>
57where
58    Input: StreamIsPartial + Stream,
59    Error: ParserError<Input>,
60{
61    trace("any", move |input: &mut Input| {
62        if <Input as StreamIsPartial>::is_partial_supported() {
63            any_::<_, _, true>(input)
64        } else {
65            any_::<_, _, false>(input)
66        }
67    })
68    .parse_next(input)
69}
70
71fn any_<I, E: ParserError<I>, const PARTIAL: bool>(input: &mut I) -> Result<<I as Stream>::Token, E>
72where
73    I: StreamIsPartial,
74    I: Stream,
75{
76    input.next_token().ok_or_else(|| {
77        if PARTIAL && input.is_partial() {
78            ParserError::incomplete(input, Needed::new(1))
79        } else {
80            ParserError::from_input(input)
81        }
82    })
83}
84
85/// Recognizes a literal
86///
87/// The input data will be compared to the literal combinator's argument and will return the part of
88/// the input that matches the argument
89///
90/// It will return `Err(ErrMode::Backtrack(_))` if the input doesn't match the literal
91///
92/// <div class="warning">
93///
94/// **Note:** [`Parser`] is implemented for strings and byte strings as a convenience (complete
95/// only)
96///
97/// </div>
98///
99/// # Effective Signature
100///
101/// Assuming you are parsing a `&str` [Stream]:
102/// ```rust
103/// # use winnow::prelude::*;;
104/// # use winnow::error::ContextError;
105/// pub fn literal(literal: &str) -> impl Parser<&str, &str, ContextError>
106/// # {
107/// #     winnow::token::literal(literal)
108/// # }
109/// ```
110///
111/// # Example
112/// ```rust
113/// # use winnow::prelude::*;
114/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
115/// #
116/// fn parser<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
117///   "Hello".parse_next(s)
118/// }
119///
120/// assert_eq!(parser.parse_peek("Hello, World!"), Ok((", World!", "Hello")));
121/// assert!(parser.parse_peek("Something").is_err());
122/// assert!(parser.parse_peek("").is_err());
123/// ```
124///
125/// ```rust
126/// # use winnow::prelude::*;
127/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
128/// # use winnow::Partial;
129///
130/// fn parser<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
131///   "Hello".parse_next(s)
132/// }
133///
134/// assert_eq!(parser.parse_peek(Partial::new("Hello, World!")), Ok((Partial::new(", World!"), "Hello")));
135/// assert!(parser.parse_peek(Partial::new("Something")).is_err());
136/// assert!(parser.parse_peek(Partial::new("S")).is_err());
137/// assert_eq!(parser.parse_peek(Partial::new("H")), Err(ErrMode::Incomplete(Needed::Unknown)));
138/// ```
139///
140/// ```rust
141/// # #[cfg(feature = "ascii")] {
142/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
143/// # use winnow::prelude::*;
144/// use winnow::token::literal;
145/// use winnow::ascii::Caseless;
146///
147/// fn parser<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
148///   literal(Caseless("hello")).parse_next(s)
149/// }
150///
151/// assert_eq!(parser.parse_peek("Hello, World!"), Ok((", World!", "Hello")));
152/// assert_eq!(parser.parse_peek("hello, World!"), Ok((", World!", "hello")));
153/// assert_eq!(parser.parse_peek("HeLlO, World!"), Ok((", World!", "HeLlO")));
154/// assert!(parser.parse_peek("Something").is_err());
155/// assert!(parser.parse_peek("").is_err());
156/// # }
157/// ```
158#[inline(always)]
159#[doc(alias = "tag")]
160#[doc(alias = "bytes")]
161#[doc(alias = "just")]
162pub fn literal<Literal, Input, Error>(
163    literal: Literal,
164) -> impl Parser<Input, <Input as Stream>::Slice, Error>
165where
166    Input: StreamIsPartial + Stream + Compare<Literal>,
167    Literal: Clone + core::fmt::Debug,
168    Error: ParserError<Input>,
169{
170    trace(DisplayDebug(literal.clone()), move |i: &mut Input| {
171        let t = literal.clone();
172        if <Input as StreamIsPartial>::is_partial_supported() {
173            literal_::<_, _, _, true>(i, t)
174        } else {
175            literal_::<_, _, _, false>(i, t)
176        }
177    })
178}
179
180fn literal_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
181    i: &mut I,
182    t: T,
183) -> Result<<I as Stream>::Slice, Error>
184where
185    I: StreamIsPartial,
186    I: Stream + Compare<T>,
187    T: core::fmt::Debug,
188{
189    match i.compare(t) {
190        CompareResult::Ok(len) => Ok(i.next_slice(len)),
191        CompareResult::Incomplete if PARTIAL && i.is_partial() => {
192            Err(ParserError::incomplete(i, Needed::Unknown))
193        }
194        CompareResult::Incomplete | CompareResult::Error => Err(ParserError::from_input(i)),
195    }
196}
197
198/// Recognize a token that matches a [set of tokens][ContainsToken]
199///
200/// <div class="warning">
201///
202/// **Note:** [`Parser`] is implemented as a convenience (complete
203/// only) for
204/// - `u8`
205/// - `char`
206///
207/// </div>
208///
209/// *Complete version*: Will return an error if there's not enough input data.
210///
211/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
212///
213/// # Effective Signature
214///
215/// Assuming you are parsing a `&str` [Stream]:
216/// ```rust
217/// # use winnow::prelude::*;;
218/// # use winnow::stream::ContainsToken;
219/// # use winnow::error::ContextError;
220/// pub fn one_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
221/// # {
222/// #     winnow::token::one_of(set)
223/// # }
224/// ```
225///
226/// # Example
227///
228/// ```rust
229/// # use winnow::prelude::*;
230/// # use winnow::{error::ErrMode, error::ContextError};
231/// # use winnow::token::one_of;
232/// assert_eq!(one_of::<_, _, ContextError>(['a', 'b', 'c']).parse_peek("b"), Ok(("", 'b')));
233/// assert!(one_of::<_, _, ContextError>('a').parse_peek("bc").is_err());
234/// assert!(one_of::<_, _, ContextError>('a').parse_peek("").is_err());
235///
236/// fn parser_fn(i: &mut &str) -> ModalResult<char> {
237///     one_of(|c| c == 'a' || c == 'b').parse_next(i)
238/// }
239/// assert_eq!(parser_fn.parse_peek("abc"), Ok(("bc", 'a')));
240/// assert!(parser_fn.parse_peek("cd").is_err());
241/// assert!(parser_fn.parse_peek("").is_err());
242/// ```
243///
244/// ```rust
245/// # use winnow::prelude::*;
246/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
247/// # use winnow::Partial;
248/// # use winnow::token::one_of;
249/// assert_eq!(one_of::<_, _, ErrMode<ContextError>>(['a', 'b', 'c']).parse_peek(Partial::new("b")), Ok((Partial::new(""), 'b')));
250/// assert!(one_of::<_, _, ErrMode<ContextError>>('a').parse_peek(Partial::new("bc")).is_err());
251/// assert_eq!(one_of::<_, _, ErrMode<ContextError>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
252///
253/// fn parser_fn(i: &mut Partial<&str>) -> ModalResult<char> {
254///     one_of(|c| c == 'a' || c == 'b').parse_next(i)
255/// }
256/// assert_eq!(parser_fn.parse_peek(Partial::new("abc")), Ok((Partial::new("bc"), 'a')));
257/// assert!(parser_fn.parse_peek(Partial::new("cd")).is_err());
258/// assert_eq!(parser_fn.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
259/// ```
260#[inline(always)]
261#[doc(alias = "char")]
262#[doc(alias = "token")]
263#[doc(alias = "satisfy")]
264pub fn one_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
265where
266    Input: StreamIsPartial + Stream,
267    <Input as Stream>::Token: Clone,
268    Set: ContainsToken<<Input as Stream>::Token>,
269    Error: ParserError<Input>,
270{
271    trace(
272        "one_of",
273        any.verify(move |t: &<Input as Stream>::Token| set.contains_token(t.clone())),
274    )
275}
276
277/// Recognize a token that does not match a [set of tokens][ContainsToken]
278///
279/// *Complete version*: Will return an error if there's not enough input data.
280///
281/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
282///
283/// # Effective Signature
284///
285/// Assuming you are parsing a `&str` [Stream]:
286/// ```rust
287/// # use winnow::prelude::*;;
288/// # use winnow::stream::ContainsToken;
289/// # use winnow::error::ContextError;
290/// pub fn none_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
291/// # {
292/// #     winnow::token::none_of(set)
293/// # }
294/// ```
295///
296/// # Example
297///
298/// ```rust
299/// # use winnow::{error::ErrMode, error::ContextError};
300/// # use winnow::prelude::*;
301/// # use winnow::token::none_of;
302/// assert_eq!(none_of::<_, _, ContextError>(['a', 'b', 'c']).parse_peek("z"), Ok(("", 'z')));
303/// assert!(none_of::<_, _, ContextError>(['a', 'b']).parse_peek("a").is_err());
304/// assert!(none_of::<_, _, ContextError>('a').parse_peek("").is_err());
305/// ```
306///
307/// ```rust
308/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
309/// # use winnow::prelude::*;
310/// # use winnow::Partial;
311/// # use winnow::token::none_of;
312/// assert_eq!(none_of::<_, _, ErrMode<ContextError>>(['a', 'b', 'c']).parse_peek(Partial::new("z")), Ok((Partial::new(""), 'z')));
313/// assert!(none_of::<_, _, ErrMode<ContextError>>(['a', 'b']).parse_peek(Partial::new("a")).is_err());
314/// assert_eq!(none_of::<_, _, ErrMode<ContextError>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
315/// ```
316#[inline(always)]
317pub fn none_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
318where
319    Input: StreamIsPartial + Stream,
320    <Input as Stream>::Token: Clone,
321    Set: ContainsToken<<Input as Stream>::Token>,
322    Error: ParserError<Input>,
323{
324    trace(
325        "none_of",
326        any.verify(move |t: &<Input as Stream>::Token| !set.contains_token(t.clone())),
327    )
328}
329
330/// Recognize the longest (m <= len <= n) input slice that matches a [set of tokens][ContainsToken]
331///
332/// It will return an `ErrMode::Backtrack(_)` if the set of tokens wasn't met or is out
333/// of range (m <= len <= n).
334///
335/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if a member of the set of tokens reaches the end of the input or is too short.
336///
337/// To take a series of tokens, use [`repeat`][crate::combinator::repeat] to [`Accumulate`][crate::stream::Accumulate] into a `()` and then [`Parser::take`].
338///
339/// # Effective Signature
340///
341/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
342/// ```rust
343/// # use std::ops::RangeFrom;
344/// # use winnow::prelude::*;
345/// # use winnow::stream::ContainsToken;
346/// # use winnow::error::ContextError;
347/// pub fn take_while<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
348/// # {
349/// #     winnow::token::take_while(occurrences, set)
350/// # }
351/// ```
352///
353/// # Example
354///
355/// Zero or more tokens:
356/// ```rust
357/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
358/// # use winnow::prelude::*;
359/// use winnow::token::take_while;
360/// use winnow::stream::AsChar;
361///
362/// fn alpha<'i>(s: &mut &'i [u8]) -> ModalResult<&'i [u8]> {
363///   take_while(0.., AsChar::is_alpha).parse_next(s)
364/// }
365///
366/// assert_eq!(alpha.parse_peek(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
367/// assert_eq!(alpha.parse_peek(b"12345"), Ok((&b"12345"[..], &b""[..])));
368/// assert_eq!(alpha.parse_peek(b"latin"), Ok((&b""[..], &b"latin"[..])));
369/// assert_eq!(alpha.parse_peek(b""), Ok((&b""[..], &b""[..])));
370/// ```
371///
372/// ```rust
373/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
374/// # use winnow::prelude::*;
375/// # use winnow::Partial;
376/// use winnow::token::take_while;
377/// use winnow::stream::AsChar;
378///
379/// fn alpha<'i>(s: &mut Partial<&'i [u8]>) -> ModalResult<&'i [u8]> {
380///   take_while(0.., AsChar::is_alpha).parse_next(s)
381/// }
382///
383/// assert_eq!(alpha.parse_peek(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
384/// assert_eq!(alpha.parse_peek(Partial::new(b"12345")), Ok((Partial::new(&b"12345"[..]), &b""[..])));
385/// assert_eq!(alpha.parse_peek(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
386/// assert_eq!(alpha.parse_peek(Partial::new(b"")), Err(ErrMode::Incomplete(Needed::new(1))));
387/// ```
388///
389/// One or more tokens:
390/// ```rust
391/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
392/// # use winnow::prelude::*;
393/// use winnow::token::take_while;
394/// use winnow::stream::AsChar;
395///
396/// fn alpha<'i>(s: &mut &'i [u8]) -> ModalResult<&'i [u8]> {
397///   take_while(1.., AsChar::is_alpha).parse_next(s)
398/// }
399///
400/// assert_eq!(alpha.parse_peek(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
401/// assert_eq!(alpha.parse_peek(b"latin"), Ok((&b""[..], &b"latin"[..])));
402/// assert!(alpha.parse_peek(b"12345").is_err());
403///
404/// fn hex<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
405///   take_while(1.., ('0'..='9', 'A'..='F')).parse_next(s)
406/// }
407///
408/// assert_eq!(hex.parse_peek("123 and voila"), Ok((" and voila", "123")));
409/// assert_eq!(hex.parse_peek("DEADBEEF and others"), Ok((" and others", "DEADBEEF")));
410/// assert_eq!(hex.parse_peek("BADBABEsomething"), Ok(("something", "BADBABE")));
411/// assert_eq!(hex.parse_peek("D15EA5E"), Ok(("", "D15EA5E")));
412/// assert!(hex.parse_peek("").is_err());
413/// ```
414///
415/// ```rust
416/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
417/// # use winnow::prelude::*;
418/// # use winnow::Partial;
419/// use winnow::token::take_while;
420/// use winnow::stream::AsChar;
421///
422/// fn alpha<'i>(s: &mut Partial<&'i [u8]>) -> ModalResult<&'i [u8]> {
423///   take_while(1.., AsChar::is_alpha).parse_next(s)
424/// }
425///
426/// assert_eq!(alpha.parse_peek(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
427/// assert_eq!(alpha.parse_peek(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
428/// assert!(alpha.parse_peek(Partial::new(b"12345")).is_err());
429///
430/// fn hex<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
431///   take_while(1.., ('0'..='9', 'A'..='F')).parse_next(s)
432/// }
433///
434/// assert_eq!(hex.parse_peek(Partial::new("123 and voila")), Ok((Partial::new(" and voila"), "123")));
435/// assert_eq!(hex.parse_peek(Partial::new("DEADBEEF and others")), Ok((Partial::new(" and others"), "DEADBEEF")));
436/// assert_eq!(hex.parse_peek(Partial::new("BADBABEsomething")), Ok((Partial::new("something"), "BADBABE")));
437/// assert_eq!(hex.parse_peek(Partial::new("D15EA5E")), Err(ErrMode::Incomplete(Needed::new(1))));
438/// assert_eq!(hex.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
439/// ```
440///
441/// Arbitrary amount of tokens:
442/// ```rust
443/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
444/// # use winnow::prelude::*;
445/// use winnow::token::take_while;
446/// use winnow::stream::AsChar;
447///
448/// fn short_alpha<'i>(s: &mut &'i [u8]) -> ModalResult<&'i [u8]> {
449///   take_while(3..=6, AsChar::is_alpha).parse_next(s)
450/// }
451///
452/// assert_eq!(short_alpha.parse_peek(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
453/// assert_eq!(short_alpha.parse_peek(b"lengthy"), Ok((&b"y"[..], &b"length"[..])));
454/// assert_eq!(short_alpha.parse_peek(b"latin"), Ok((&b""[..], &b"latin"[..])));
455/// assert!(short_alpha.parse_peek(b"ed").is_err());
456/// assert!(short_alpha.parse_peek(b"12345").is_err());
457/// ```
458///
459/// ```rust
460/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
461/// # use winnow::prelude::*;
462/// # use winnow::Partial;
463/// use winnow::token::take_while;
464/// use winnow::stream::AsChar;
465///
466/// fn short_alpha<'i>(s: &mut Partial<&'i [u8]>) -> ModalResult<&'i [u8]> {
467///   take_while(3..=6, AsChar::is_alpha).parse_next(s)
468/// }
469///
470/// assert_eq!(short_alpha.parse_peek(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
471/// assert_eq!(short_alpha.parse_peek(Partial::new(b"lengthy")), Ok((Partial::new(&b"y"[..]), &b"length"[..])));
472/// assert_eq!(short_alpha.parse_peek(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
473/// assert_eq!(short_alpha.parse_peek(Partial::new(b"ed")), Err(ErrMode::Incomplete(Needed::new(1))));
474/// assert!(short_alpha.parse_peek(Partial::new(b"12345")).is_err());
475/// ```
476#[inline(always)]
477#[doc(alias = "is_a")]
478#[doc(alias = "take_while0")]
479#[doc(alias = "take_while1")]
480pub fn take_while<Set, Input, Error>(
481    occurrences: impl Into<Range>,
482    set: Set,
483) -> impl Parser<Input, <Input as Stream>::Slice, Error>
484where
485    Input: StreamIsPartial + Stream,
486    Set: ContainsToken<<Input as Stream>::Token>,
487    Error: ParserError<Input>,
488{
489    let Range {
490        start_inclusive,
491        end_inclusive,
492    } = occurrences.into();
493    trace("take_while", move |i: &mut Input| {
494        match (start_inclusive, end_inclusive) {
495            (0, None) => {
496                if <Input as StreamIsPartial>::is_partial_supported() {
497                    take_till0::<_, _, _, true>(i, |c| !set.contains_token(c))
498                } else {
499                    take_till0::<_, _, _, false>(i, |c| !set.contains_token(c))
500                }
501            }
502            (1, None) => {
503                if <Input as StreamIsPartial>::is_partial_supported() {
504                    take_till1::<_, _, _, true>(i, |c| !set.contains_token(c))
505                } else {
506                    take_till1::<_, _, _, false>(i, |c| !set.contains_token(c))
507                }
508            }
509            (start, end) => {
510                let end = end.unwrap_or(usize::MAX);
511                if <Input as StreamIsPartial>::is_partial_supported() {
512                    take_till_m_n::<_, _, _, true>(i, start, end, |c| !set.contains_token(c))
513                } else {
514                    take_till_m_n::<_, _, _, false>(i, start, end, |c| !set.contains_token(c))
515                }
516            }
517        }
518    })
519}
520
521fn take_till0<P, I: StreamIsPartial + Stream, E: ParserError<I>, const PARTIAL: bool>(
522    input: &mut I,
523    predicate: P,
524) -> Result<<I as Stream>::Slice, E>
525where
526    P: Fn(I::Token) -> bool,
527{
528    let offset = match input.offset_for(predicate) {
529        Some(offset) => offset,
530        None if PARTIAL && input.is_partial() => {
531            return Err(ParserError::incomplete(input, Needed::new(1)));
532        }
533        None => input.eof_offset(),
534    };
535    Ok(input.next_slice(offset))
536}
537
538fn take_till1<P, I: StreamIsPartial + Stream, E: ParserError<I>, const PARTIAL: bool>(
539    input: &mut I,
540    predicate: P,
541) -> Result<<I as Stream>::Slice, E>
542where
543    P: Fn(I::Token) -> bool,
544{
545    let offset = match input.offset_for(predicate) {
546        Some(offset) => offset,
547        None if PARTIAL && input.is_partial() => {
548            return Err(ParserError::incomplete(input, Needed::new(1)));
549        }
550        None => input.eof_offset(),
551    };
552    if offset == 0 {
553        Err(ParserError::from_input(input))
554    } else {
555        Ok(input.next_slice(offset))
556    }
557}
558
559fn take_till_m_n<P, I, Error: ParserError<I>, const PARTIAL: bool>(
560    input: &mut I,
561    m: usize,
562    n: usize,
563    predicate: P,
564) -> Result<<I as Stream>::Slice, Error>
565where
566    I: StreamIsPartial,
567    I: Stream,
568    P: Fn(I::Token) -> bool,
569{
570    if n < m {
571        return Err(ParserError::assert(
572            input,
573            "`occurrences` should be ascending, rather than descending",
574        ));
575    }
576
577    let mut final_count = 0;
578    for (processed, (offset, token)) in input.iter_offsets().enumerate() {
579        if predicate(token) {
580            if processed < m {
581                return Err(ParserError::from_input(input));
582            } else {
583                return Ok(input.next_slice(offset));
584            }
585        } else {
586            if processed == n {
587                return Ok(input.next_slice(offset));
588            }
589            final_count = processed + 1;
590        }
591    }
592    if PARTIAL && input.is_partial() {
593        if final_count == n {
594            Ok(input.finish())
595        } else {
596            let needed = if m > input.eof_offset() {
597                m - input.eof_offset()
598            } else {
599                1
600            };
601            Err(ParserError::incomplete(input, Needed::new(needed)))
602        }
603    } else {
604        if m <= final_count {
605            Ok(input.finish())
606        } else {
607            Err(ParserError::from_input(input))
608        }
609    }
610}
611
612/// Recognize the longest input slice (if any) till a member of a [set of tokens][ContainsToken] is found.
613///
614/// It doesn't consume the terminating token from the set.
615///
616/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if the match reaches the
617/// end of input or if there was not match.
618///
619/// See also
620/// - [`take_until`] for recognizing up-to a [`literal`] (w/ optional simd optimizations)
621/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
622///
623/// # Effective Signature
624///
625/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
626/// ```rust
627/// # use std::ops::RangeFrom;
628/// # use winnow::prelude::*;
629/// # use winnow::stream::ContainsToken;
630/// # use winnow::error::ContextError;
631/// pub fn take_till<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
632/// # {
633/// #     winnow::token::take_till(occurrences, set)
634/// # }
635/// ```
636///
637/// # Example
638///
639/// ```rust
640/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
641/// # use winnow::prelude::*;
642/// use winnow::token::take_till;
643///
644/// fn till_colon<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
645///   take_till(0.., |c| c == ':').parse_next(s)
646/// }
647///
648/// assert_eq!(till_colon.parse_peek("latin:123"), Ok((":123", "latin")));
649/// assert_eq!(till_colon.parse_peek(":empty matched"), Ok((":empty matched", ""))); //allowed
650/// assert_eq!(till_colon.parse_peek("12345"), Ok(("", "12345")));
651/// assert_eq!(till_colon.parse_peek(""), Ok(("", "")));
652/// ```
653///
654/// ```rust
655/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
656/// # use winnow::prelude::*;
657/// # use winnow::Partial;
658/// use winnow::token::take_till;
659///
660/// fn till_colon<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
661///   take_till(0.., |c| c == ':').parse_next(s)
662/// }
663///
664/// assert_eq!(till_colon.parse_peek(Partial::new("latin:123")), Ok((Partial::new(":123"), "latin")));
665/// assert_eq!(till_colon.parse_peek(Partial::new(":empty matched")), Ok((Partial::new(":empty matched"), ""))); //allowed
666/// assert_eq!(till_colon.parse_peek(Partial::new("12345")), Err(ErrMode::Incomplete(Needed::new(1))));
667/// assert_eq!(till_colon.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
668/// ```
669#[inline(always)]
670#[doc(alias = "is_not")]
671pub fn take_till<Set, Input, Error>(
672    occurrences: impl Into<Range>,
673    set: Set,
674) -> impl Parser<Input, <Input as Stream>::Slice, Error>
675where
676    Input: StreamIsPartial + Stream,
677    Set: ContainsToken<<Input as Stream>::Token>,
678    Error: ParserError<Input>,
679{
680    let Range {
681        start_inclusive,
682        end_inclusive,
683    } = occurrences.into();
684    trace("take_till", move |i: &mut Input| {
685        match (start_inclusive, end_inclusive) {
686            (0, None) => {
687                if <Input as StreamIsPartial>::is_partial_supported() {
688                    take_till0::<_, _, _, true>(i, |c| set.contains_token(c))
689                } else {
690                    take_till0::<_, _, _, false>(i, |c| set.contains_token(c))
691                }
692            }
693            (1, None) => {
694                if <Input as StreamIsPartial>::is_partial_supported() {
695                    take_till1::<_, _, _, true>(i, |c| set.contains_token(c))
696                } else {
697                    take_till1::<_, _, _, false>(i, |c| set.contains_token(c))
698                }
699            }
700            (start, end) => {
701                let end = end.unwrap_or(usize::MAX);
702                if <Input as StreamIsPartial>::is_partial_supported() {
703                    take_till_m_n::<_, _, _, true>(i, start, end, |c| set.contains_token(c))
704                } else {
705                    take_till_m_n::<_, _, _, false>(i, start, end, |c| set.contains_token(c))
706                }
707            }
708        }
709    })
710}
711
712/// Recognize an input slice containing the first N input elements (I[..N]).
713///
714/// *Complete version*: It will return `Err(ErrMode::Backtrack(_))` if the input is shorter than the argument.
715///
716/// *[Partial version][crate::_topic::partial]*: if the input has less than N elements, `take` will
717/// return a `ErrMode::Incomplete(Needed::new(M))` where M is the number of
718/// additional bytes the parser would need to succeed.
719/// It is well defined for `&[u8]` as the number of elements is the byte size,
720/// but for types like `&str`, we cannot know how many bytes correspond for
721/// the next few chars, so the result will be `ErrMode::Incomplete(Needed::Unknown)`
722///
723/// # Effective Signature
724///
725/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` ranges:
726/// ```rust
727/// # use std::ops::RangeFrom;
728/// # use winnow::prelude::*;
729/// # use winnow::stream::ContainsToken;
730/// # use winnow::error::ContextError;
731/// pub fn take<'i>(token_count: usize) -> impl Parser<&'i str, &'i str, ContextError>
732/// # {
733/// #     winnow::token::take(token_count)
734/// # }
735/// ```
736///
737/// # Example
738///
739/// ```rust
740/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
741/// # use winnow::prelude::*;
742/// use winnow::token::take;
743///
744/// fn take6<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
745///   take(6usize).parse_next(s)
746/// }
747///
748/// assert_eq!(take6.parse_peek("1234567"), Ok(("7", "123456")));
749/// assert_eq!(take6.parse_peek("things"), Ok(("", "things")));
750/// assert!(take6.parse_peek("short").is_err());
751/// assert!(take6.parse_peek("").is_err());
752/// ```
753///
754/// The units that are taken will depend on the input type. For example, for a
755/// `&str` it will take a number of `char`'s, whereas for a `&[u8]` it will
756/// take that many `u8`'s:
757///
758/// ```rust
759/// # use winnow::prelude::*;
760/// use winnow::error::ContextError;
761/// use winnow::token::take;
762///
763/// assert_eq!(take::<_, _, ContextError>(1usize).parse_peek("💙"), Ok(("", "💙")));
764/// assert_eq!(take::<_, _, ContextError>(1usize).parse_peek("💙".as_bytes()), Ok((b"\x9F\x92\x99".as_ref(), b"\xF0".as_ref())));
765/// ```
766///
767/// ```rust
768/// # use winnow::prelude::*;
769/// # use winnow::error::{ErrMode, ContextError, Needed};
770/// # use winnow::Partial;
771/// use winnow::token::take;
772///
773/// fn take6<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
774///   take(6usize).parse_next(s)
775/// }
776///
777/// assert_eq!(take6.parse_peek(Partial::new("1234567")), Ok((Partial::new("7"), "123456")));
778/// assert_eq!(take6.parse_peek(Partial::new("things")), Ok((Partial::new(""), "things")));
779/// // `Unknown` as we don't know the number of bytes that `count` corresponds to
780/// assert_eq!(take6.parse_peek(Partial::new("short")), Err(ErrMode::Incomplete(Needed::Unknown)));
781/// ```
782#[inline(always)]
783pub fn take<UsizeLike, Input, Error>(
784    token_count: UsizeLike,
785) -> impl Parser<Input, <Input as Stream>::Slice, Error>
786where
787    Input: StreamIsPartial + Stream,
788    UsizeLike: ToUsize,
789    Error: ParserError<Input>,
790{
791    let c = token_count.to_usize();
792    trace("take", move |i: &mut Input| {
793        if <Input as StreamIsPartial>::is_partial_supported() {
794            take_::<_, _, true>(i, c)
795        } else {
796            take_::<_, _, false>(i, c)
797        }
798    })
799}
800
801fn take_<I, Error: ParserError<I>, const PARTIAL: bool>(
802    i: &mut I,
803    c: usize,
804) -> Result<<I as Stream>::Slice, Error>
805where
806    I: StreamIsPartial,
807    I: Stream,
808{
809    match i.offset_at(c) {
810        Ok(offset) => Ok(i.next_slice(offset)),
811        Err(e) if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, e)),
812        Err(_needed) => Err(ParserError::from_input(i)),
813    }
814}
815
816/// Recognize the input slice up to the first occurrence of a [literal].
817///
818/// Feature `simd` will enable the use of [`memchr`](https://docs.rs/memchr/latest/memchr/).
819///
820/// It doesn't consume the literal.
821///
822/// *Complete version*: It will return `Err(ErrMode::Backtrack(_))`
823/// if the literal wasn't met.
824///
825/// *[Partial version][crate::_topic::partial]*: will return a `ErrMode::Incomplete(Needed::new(N))` if the input doesn't
826/// contain the literal or if the input is smaller than the literal.
827///
828/// See also
829/// - [`take_till`] for recognizing up-to a [set of tokens][ContainsToken]
830/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
831///
832/// # Effective Signature
833///
834/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
835/// ```rust
836/// # use std::ops::RangeFrom;
837/// # use winnow::prelude::*;;
838/// # use winnow::error::ContextError;
839/// pub fn take_until(occurrences: RangeFrom<usize>, literal: &str) -> impl Parser<&str, &str, ContextError>
840/// # {
841/// #     winnow::token::take_until(occurrences, literal)
842/// # }
843/// ```
844///
845/// # Example
846///
847/// ```rust
848/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
849/// # use winnow::prelude::*;
850/// use winnow::token::take_until;
851///
852/// fn until_eof<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
853///   take_until(0.., "eof").parse_next(s)
854/// }
855///
856/// assert_eq!(until_eof.parse_peek("hello, worldeof"), Ok(("eof", "hello, world")));
857/// assert!(until_eof.parse_peek("hello, world").is_err());
858/// assert!(until_eof.parse_peek("").is_err());
859/// assert_eq!(until_eof.parse_peek("1eof2eof"), Ok(("eof2eof", "1")));
860/// ```
861///
862/// ```rust
863/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
864/// # use winnow::prelude::*;
865/// # use winnow::Partial;
866/// use winnow::token::take_until;
867///
868/// fn until_eof<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
869///   take_until(0.., "eof").parse_next(s)
870/// }
871///
872/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
873/// assert_eq!(until_eof.parse_peek(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
874/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
875/// assert_eq!(until_eof.parse_peek(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
876/// ```
877///
878/// ```rust
879/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
880/// # use winnow::prelude::*;
881/// use winnow::token::take_until;
882///
883/// fn until_eof<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
884///   take_until(1.., "eof").parse_next(s)
885/// }
886///
887/// assert_eq!(until_eof.parse_peek("hello, worldeof"), Ok(("eof", "hello, world")));
888/// assert!(until_eof.parse_peek("hello, world").is_err());
889/// assert!(until_eof.parse_peek("").is_err());
890/// assert_eq!(until_eof.parse_peek("1eof2eof"), Ok(("eof2eof", "1")));
891/// assert!(until_eof.parse_peek("eof").is_err());
892/// ```
893///
894/// ```rust
895/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
896/// # use winnow::prelude::*;
897/// # use winnow::Partial;
898/// use winnow::token::take_until;
899///
900/// fn until_eof<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
901///   take_until(1.., "eof").parse_next(s)
902/// }
903///
904/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
905/// assert_eq!(until_eof.parse_peek(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
906/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
907/// assert_eq!(until_eof.parse_peek(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
908/// assert!(until_eof.parse_peek(Partial::new("eof")).is_err());
909/// ```
910#[inline(always)]
911pub fn take_until<Literal, Input, Error>(
912    occurrences: impl Into<Range>,
913    literal: Literal,
914) -> impl Parser<Input, <Input as Stream>::Slice, Error>
915where
916    Input: StreamIsPartial + Stream + FindSlice<Literal>,
917    Literal: Clone,
918    Error: ParserError<Input>,
919{
920    let Range {
921        start_inclusive,
922        end_inclusive,
923    } = occurrences.into();
924    trace("take_until", move |i: &mut Input| {
925        match (start_inclusive, end_inclusive) {
926            (0, None) => {
927                if <Input as StreamIsPartial>::is_partial_supported() {
928                    take_until0_::<_, _, _, true>(i, literal.clone())
929                } else {
930                    take_until0_::<_, _, _, false>(i, literal.clone())
931                }
932            }
933            (1, None) => {
934                if <Input as StreamIsPartial>::is_partial_supported() {
935                    take_until1_::<_, _, _, true>(i, literal.clone())
936                } else {
937                    take_until1_::<_, _, _, false>(i, literal.clone())
938                }
939            }
940            (start, end) => {
941                let end = end.unwrap_or(usize::MAX);
942                if <Input as StreamIsPartial>::is_partial_supported() {
943                    take_until_m_n_::<_, _, _, true>(i, start, end, literal.clone())
944                } else {
945                    take_until_m_n_::<_, _, _, false>(i, start, end, literal.clone())
946                }
947            }
948        }
949    })
950}
951
952fn take_until0_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
953    i: &mut I,
954    t: T,
955) -> Result<<I as Stream>::Slice, Error>
956where
957    I: StreamIsPartial,
958    I: Stream + FindSlice<T>,
959{
960    match i.find_slice(t) {
961        Some(range) => Ok(i.next_slice(range.start)),
962        None if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, Needed::Unknown)),
963        None => Err(ParserError::from_input(i)),
964    }
965}
966
967fn take_until1_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
968    i: &mut I,
969    t: T,
970) -> Result<<I as Stream>::Slice, Error>
971where
972    I: StreamIsPartial,
973    I: Stream + FindSlice<T>,
974{
975    match i.find_slice(t) {
976        None if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, Needed::Unknown)),
977        None => Err(ParserError::from_input(i)),
978        Some(range) => {
979            if range.start == 0 {
980                Err(ParserError::from_input(i))
981            } else {
982                Ok(i.next_slice(range.start))
983            }
984        }
985    }
986}
987
988fn take_until_m_n_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
989    i: &mut I,
990    start: usize,
991    end: usize,
992    t: T,
993) -> Result<<I as Stream>::Slice, Error>
994where
995    I: StreamIsPartial,
996    I: Stream + FindSlice<T>,
997{
998    if end < start {
999        return Err(ParserError::assert(
1000            i,
1001            "`occurrences` should be ascending, rather than descending",
1002        ));
1003    }
1004
1005    match i.find_slice(t) {
1006        Some(range) => {
1007            let start_offset = i.offset_at(start);
1008            let end_offset = i.offset_at(end).unwrap_or_else(|_err| i.eof_offset());
1009            if start_offset.map(|s| range.start < s).unwrap_or(true) {
1010                if PARTIAL && i.is_partial() {
1011                    return Err(ParserError::incomplete(i, Needed::Unknown));
1012                } else {
1013                    return Err(ParserError::from_input(i));
1014                }
1015            }
1016            if end_offset < range.start {
1017                return Err(ParserError::from_input(i));
1018            }
1019            Ok(i.next_slice(range.start))
1020        }
1021        None if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, Needed::Unknown)),
1022        None => Err(ParserError::from_input(i)),
1023    }
1024}
1025
1026/// Return the remaining input.
1027///
1028/// # Effective Signature
1029///
1030/// Assuming you are parsing a `&str` [Stream]:
1031/// ```rust
1032/// # use winnow::prelude::*;;
1033/// pub fn rest<'i>(input: &mut &'i str) -> ModalResult<&'i str>
1034/// # {
1035/// #     winnow::token::rest.parse_next(input)
1036/// # }
1037/// ```
1038///
1039/// # Example
1040///
1041/// ```rust
1042/// # use winnow::prelude::*;
1043/// # use winnow::error::ContextError;
1044/// use winnow::token::rest;
1045/// assert_eq!(rest::<_,ContextError>.parse_peek("abc"), Ok(("", "abc")));
1046/// assert_eq!(rest::<_,ContextError>.parse_peek(""), Ok(("", "")));
1047/// ```
1048#[inline]
1049pub fn rest<Input, Error>(input: &mut Input) -> Result<<Input as Stream>::Slice, Error>
1050where
1051    Input: Stream,
1052    Error: ParserError<Input>,
1053{
1054    trace("rest", move |input: &mut Input| Ok(input.finish())).parse_next(input)
1055}
1056
1057/// Return the length of the remaining input.
1058///
1059/// <div class="warning">
1060///
1061/// Note: this does not advance the [`Stream`]
1062///
1063/// </div>
1064///
1065/// # Effective Signature
1066///
1067/// Assuming you are parsing a `&str` [Stream]:
1068/// ```rust
1069/// # use winnow::prelude::*;;
1070/// pub fn rest_len(input: &mut &str) -> ModalResult<usize>
1071/// # {
1072/// #     winnow::token::rest_len.parse_next(input)
1073/// # }
1074/// ```
1075///
1076/// # Example
1077///
1078/// ```rust
1079/// # use winnow::prelude::*;
1080/// # use winnow::error::ContextError;
1081/// use winnow::token::rest_len;
1082/// assert_eq!(rest_len::<_,ContextError>.parse_peek("abc"), Ok(("abc", 3)));
1083/// assert_eq!(rest_len::<_,ContextError>.parse_peek(""), Ok(("", 0)));
1084/// ```
1085#[inline]
1086pub fn rest_len<Input, Error>(input: &mut Input) -> Result<usize, Error>
1087where
1088    Input: Stream,
1089    Error: ParserError<Input>,
1090{
1091    trace("rest_len", move |input: &mut Input| {
1092        let len = input.eof_offset();
1093        Ok(len)
1094    })
1095    .parse_next(input)
1096}
winnow/token/mod.rs

winnow/token/
mod.rs
