irc_rust/
tokenizer.rs

1use crate::errors::ParserError;
2use crate::parsed::Parsed;
3use crate::prefix::Prefix;
4use std::collections::{HashMap, HashSet};
5use std::fmt::Debug;
6use std::marker::PhantomData;
7
8/// Implements a Parser of IRC Messages as described in [IRCv3](https://ircv3.net/irc/) and
9/// [RFC 1459](https://tools.ietf.org/html/rfc1459).
10///
11/// The tokenizer implements both phases of a parser: Lexical and syntactical analysis. This
12/// is required as it implements a __Zero-allocation__ parser which is not allocating anything
13/// on the heap and returns the next element based on its state.
14///
15/// Transitions between states are implemented with methods [Tokenizer::tags], [Tokenizer::prefix],
16/// [Tokenizer::command], [Tokenizer::params] and [Tokenizer::trailing]. Based on the state
17/// different parts of the message can be parsed. If some parts of the message are not
18/// needed they are skipped by calling the wanted state transition method.
19#[derive(Eq, PartialEq, Debug, Copy, Clone)]
20pub struct Tokenizer<'a, T: State> {
21    raw: &'a str,
22    state: PhantomData<T>,
23}
24
25pub trait State: PartialEq + Eq + Debug {}
26
27impl<'a, S: State> Tokenizer<'a, S> {
28    fn skip_until_char(&mut self, ch: char, skip_char: bool) {
29        if self.raw.starts_with(ch) {
30            return;
31        }
32
33        let end = self
34            .raw
35            .find(ch)
36            .map(|space_pos| if skip_char { space_pos + 1 } else { space_pos })
37            .unwrap_or_else(|| self.raw.len());
38        self.raw = &self.raw[end..];
39    }
40
41    fn skip_until_str(&mut self, s: &str) {
42        if self.raw.starts_with(s) {
43            return;
44        }
45
46        let end = self.raw.find(s).unwrap_or_else(|| self.raw.len());
47        self.raw = &self.raw[end..];
48    }
49
50    fn skip_to_end(&mut self) {
51        self.raw = &self.raw[self.raw.len()..];
52    }
53
54    fn skip_tags(&mut self) {
55        // include ';' to also skip if tags have been partially parsed
56        if self.raw.starts_with(&['@', ';'][..]) {
57            self.skip_until_char(' ', true);
58        }
59    }
60
61    fn skip_prefix(&mut self) {
62        self.skip_tags();
63        if self.raw.starts_with(&[':', '!', '@'][..]) {
64            self.skip_until_char(' ', true);
65        }
66    }
67
68    fn skip_command(&mut self) {
69        self.skip_prefix();
70        self.skip_until_char(' ', false);
71    }
72
73    fn skip_params(&mut self) {
74        self.skip_command();
75        self.skip_until_str(" :");
76    }
77}
78
79#[derive(Eq, PartialEq, Debug, Copy, Clone)]
80pub struct Start;
81
82impl State for Start {}
83
84impl<'a> Tokenizer<'a, Start> {
85    pub fn new(raw: &'a str) -> Result<Self, ParserError> {
86        if raw.is_empty() {
87            Err(ParserError::NoCommand)
88        } else {
89            Ok(Tokenizer {
90                raw,
91                state: PhantomData::default(),
92            })
93        }
94    }
95
96    pub fn parse_partial(self, mut cfg: PartialCfg<'a>) -> Result<Parsed<'a>, ParserError> {
97        let mut result_tags = HashMap::new();
98        let mut result_prefix = None;
99        let mut result_command = None;
100        let mut result_params = Vec::new();
101        let mut result_trailing = None;
102
103        // Parse tags
104        let mut tokenizer = self.tags();
105        if !cfg.tags.is_empty() {
106            let mut tags = HashMap::with_capacity(cfg.tags.len());
107            let mut iter = tokenizer.as_iter();
108            while !cfg.tags.is_empty() {
109                match iter.next() {
110                    Some(Ok((key, val))) => {
111                        if cfg.tags.remove(&key) {
112                            tags.insert(key, val);
113                        }
114                    }
115                    Some(Err(why)) => return Err(why),
116                    None => break,
117                }
118            }
119            result_tags = tags;
120        }
121
122        // Parse prefix
123        let mut tokenizer = tokenizer.prefix();
124        if let Some((user, host)) = cfg.prefix {
125            result_prefix = Some((
126                tokenizer.name()?.ok_or(ParserError::PrefixWithoutName)?,
127                if user { tokenizer.user()? } else { None },
128                if host { tokenizer.name()? } else { None },
129            ));
130        }
131
132        // Command
133        let mut tokenizer = tokenizer.command();
134        if cfg.command {
135            result_command = Some(tokenizer.command()?);
136        }
137
138        // Params
139        let mut tokenizer = tokenizer.params();
140        if !cfg.params.is_empty() {
141            let mut params = Vec::with_capacity(cfg.params.len());
142            let mut iter = tokenizer.as_iter();
143            cfg.params.dedup();
144            cfg.params.sort_unstable();
145            let mut position = 0;
146            for index in cfg.params {
147                let delta = index - position;
148                // Fill with |delta| Nones
149                params.extend(vec![None; delta]);
150                position = index;
151                params.push(iter.nth(delta));
152            }
153            result_params = params;
154        }
155
156        // Trailing
157        let tokenizer = tokenizer.trailing();
158        if cfg.trailing {
159            result_trailing = tokenizer.trailing();
160        }
161
162        Ok(Parsed::new(
163            result_tags,
164            result_prefix,
165            result_command,
166            result_params,
167            result_trailing,
168        ))
169    }
170
171    pub fn tags(self) -> Tokenizer<'a, TagsState> {
172        Tokenizer {
173            raw: self.raw,
174            state: PhantomData::default(),
175        }
176    }
177
178    pub fn prefix(mut self) -> Tokenizer<'a, PrefixState> {
179        self.skip_tags();
180        Tokenizer {
181            raw: self.raw,
182            state: PhantomData::default(),
183        }
184    }
185
186    pub fn command(mut self) -> Tokenizer<'a, CommandState> {
187        self.skip_prefix();
188        Tokenizer {
189            raw: self.raw,
190            state: PhantomData::default(),
191        }
192    }
193
194    pub fn params(mut self) -> Tokenizer<'a, ParamsState> {
195        self.skip_command();
196        Tokenizer {
197            raw: self.raw,
198            state: PhantomData::default(),
199        }
200    }
201
202    pub fn trailing(mut self) -> Tokenizer<'a, TrailingState> {
203        self.skip_params();
204        Tokenizer {
205            raw: self.raw,
206            state: PhantomData::default(),
207        }
208    }
209}
210
211#[derive(Eq, PartialEq, Debug, Copy, Clone)]
212pub struct TagsState;
213
214impl State for TagsState {}
215
216impl<'a> Tokenizer<'a, TagsState> {
217    pub fn as_iter(&mut self) -> IntoTagsIter<'a> {
218        IntoTagsIter(*self)
219    }
220
221    pub fn prefix(mut self) -> Tokenizer<'a, PrefixState> {
222        self.skip_tags();
223        Tokenizer {
224            raw: self.raw,
225            state: PhantomData::default(),
226        }
227    }
228
229    pub fn command(mut self) -> Tokenizer<'a, CommandState> {
230        self.skip_prefix();
231        Tokenizer {
232            raw: self.raw,
233            state: PhantomData::default(),
234        }
235    }
236
237    pub fn params(mut self) -> Tokenizer<'a, ParamsState> {
238        self.skip_command();
239        Tokenizer {
240            raw: self.raw,
241            state: PhantomData::default(),
242        }
243    }
244
245    pub fn trailing(mut self) -> Tokenizer<'a, TrailingState> {
246        self.skip_params();
247        Tokenizer {
248            raw: self.raw,
249            state: PhantomData::default(),
250        }
251    }
252}
253
254impl<'a> IntoIterator for Tokenizer<'a, TagsState> {
255    type Item = Result<(&'a str, &'a str), ParserError>;
256    type IntoIter = IntoTagsIter<'a>;
257
258    fn into_iter(self) -> Self::IntoIter {
259        IntoTagsIter(self)
260    }
261}
262
263pub struct IntoTagsIter<'a>(Tokenizer<'a, TagsState>);
264
265impl<'a> Iterator for IntoTagsIter<'a> {
266    type Item = Result<(&'a str, &'a str), ParserError>;
267
268    fn next(&mut self) -> Option<Self::Item> {
269        match &self.0.raw[..1] {
270            "@" | ";" => {
271                let key_start = 1;
272                let key_end = self.0.raw[key_start..]
273                    .find(&['='][..])
274                    .map(|key_end| (key_end + key_start, key_end + key_start + 1))
275                    .or_else(|| {
276                        let key_end = key_start + self.0.raw[key_start..].find(&[' ', ';'][..])?;
277                        Some((key_end, key_end))
278                    });
279                if key_end.is_none() {
280                    // Skip till the end as only tags seem to be present
281                    self.0.skip_to_end();
282                    return Some(Err(ParserError::NoTagKeyEnd));
283                }
284                let (key_end, val_start) = key_end.unwrap();
285                let val_end = self.0.raw[val_start..].find(&[';', ' '][..]);
286                if val_end.is_none() {
287                    // Skip till the end as only tags seem to be present
288                    self.0.skip_to_end();
289                    return Some(Err(ParserError::NoTagValueEnd));
290                }
291                let val_end = val_start + val_end.unwrap();
292                let key_val = (
293                    &self.0.raw[key_start..key_end],
294                    &self.0.raw[val_start..val_end],
295                );
296                self.0.raw = &self.0.raw[val_end..];
297                Some(Ok(key_val))
298            }
299            _ => None,
300        }
301    }
302}
303
304#[derive(Eq, PartialEq, Debug, Copy, Clone)]
305pub struct PrefixState;
306
307impl State for PrefixState {}
308
309impl<'a> Tokenizer<'a, PrefixState> {
310    pub fn name(&mut self) -> Result<Option<&'a str>, ParserError> {
311        if self.raw.starts_with(' ') {
312            self.raw = &self.raw[1..];
313        }
314        let mut name = None;
315        if self.raw.starts_with(':') {
316            let end = self
317                .raw
318                .find(&['!', '@', ' '][..])
319                .ok_or(ParserError::NoCommand)?;
320            let split = self.raw.split_at(end);
321            name = Some(&split.0[1..]);
322            self.raw = split.1;
323        }
324        Ok(name)
325    }
326
327    pub fn user(&mut self) -> Result<Option<&'a str>, ParserError> {
328        let mut user = None;
329        if self.raw.starts_with('!') {
330            let end = self
331                .raw
332                .find(&['@', ' '][..])
333                .ok_or(ParserError::NoCommand)?;
334            let split = self.raw.split_at(end);
335            user = Some(&split.0[1..]);
336            self.raw = split.1;
337        }
338        Ok(user)
339    }
340
341    pub fn host(&mut self) -> Result<Option<&'a str>, ParserError> {
342        let mut host = None;
343        if self.raw.starts_with('@') {
344            let end = self.raw.find(' ').ok_or(ParserError::NoCommand)?;
345            let split = self.raw.split_at(end);
346            host = Some(&split.0[1..]);
347            self.raw = split.1;
348        }
349        Ok(host)
350    }
351
352    /// Returns [None] if the prefix is badly formatted or no prefix is present.
353    pub fn parts(&mut self) -> Result<Option<Prefix<'a>>, ParserError> {
354        if self.raw.starts_with(' ') {
355            self.raw = &self.raw[1..];
356        }
357        if !self.raw.starts_with(':') {
358            return Ok(None);
359        }
360        let (name, user, host) = (self.name()?, self.user()?, self.host()?);
361        if name.is_none() && (user.is_some() || host.is_some()) {
362            Err(ParserError::PrefixWithoutName)
363        } else {
364            Ok(Some((name.unwrap(), user, host)))
365        }
366    }
367
368    pub fn command(mut self) -> Tokenizer<'a, CommandState> {
369        self.skip_prefix();
370        Tokenizer {
371            raw: self.raw,
372            state: PhantomData::default(),
373        }
374    }
375
376    pub fn params(mut self) -> Tokenizer<'a, ParamsState> {
377        self.skip_command();
378        Tokenizer {
379            raw: self.raw,
380            state: PhantomData::default(),
381        }
382    }
383
384    pub fn trailing(mut self) -> Tokenizer<'a, TrailingState> {
385        self.skip_params();
386        Tokenizer {
387            raw: self.raw,
388            state: PhantomData::default(),
389        }
390    }
391}
392
393#[derive(Eq, PartialEq, Debug, Copy, Clone)]
394pub struct CommandState;
395
396impl State for CommandState {}
397
398impl<'a> Tokenizer<'a, CommandState> {
399    pub fn command(&mut self) -> Result<&'a str, ParserError> {
400        if self.raw.starts_with(' ') {
401            self.raw = &self.raw[1..];
402        }
403
404        let end = self.raw.find(' ').unwrap_or_else(|| self.raw.len());
405        let (command, rest) = self.raw.split_at(end);
406        if command.is_empty() {
407            return Err(ParserError::NoCommand);
408        }
409        self.raw = rest;
410        Ok(command)
411    }
412
413    pub fn params(mut self) -> Tokenizer<'a, ParamsState> {
414        self.skip_command();
415        Tokenizer {
416            raw: self.raw,
417            state: PhantomData::default(),
418        }
419    }
420
421    pub fn trailing(mut self) -> Tokenizer<'a, TrailingState> {
422        self.skip_params();
423        Tokenizer {
424            raw: self.raw,
425            state: PhantomData::default(),
426        }
427    }
428}
429
430#[derive(Eq, PartialEq, Debug, Copy, Clone)]
431pub struct ParamsState;
432
433impl State for ParamsState {}
434
435impl<'a> Tokenizer<'a, ParamsState> {
436    pub fn trailing(mut self) -> Tokenizer<'a, TrailingState> {
437        self.skip_params();
438        Tokenizer {
439            raw: self.raw,
440            state: PhantomData::default(),
441        }
442    }
443
444    pub fn as_iter(&mut self) -> IntoParamsIter<'a> {
445        IntoParamsIter(*self)
446    }
447}
448
449impl<'a> IntoIterator for Tokenizer<'a, ParamsState> {
450    type Item = &'a str;
451    type IntoIter = IntoParamsIter<'a>;
452
453    fn into_iter(self) -> Self::IntoIter {
454        IntoParamsIter(self)
455    }
456}
457
458pub struct IntoParamsIter<'a>(Tokenizer<'a, ParamsState>);
459
460impl<'a> Iterator for IntoParamsIter<'a> {
461    type Item = &'a str;
462
463    fn next(&mut self) -> Option<Self::Item> {
464        if !self.0.raw.starts_with(' ') || self.0.raw.starts_with(" :") {
465            return None;
466        }
467        self.0.raw = &self.0.raw[1..];
468        let end = self
469            .0
470            .raw
471            .find(' ')
472            .or_else(|| self.0.raw.find(" :"))
473            .unwrap_or_else(|| self.0.raw.len());
474        let (param, rest) = self.0.raw.split_at(end);
475        self.0.raw = rest;
476        Some(param)
477    }
478}
479
480#[derive(Eq, PartialEq, Debug, Copy, Clone)]
481pub struct TrailingState;
482
483impl State for TrailingState {}
484
485impl<'a> Tokenizer<'a, TrailingState> {
486    pub fn trailing(&self) -> Option<&'a str> {
487        if self.raw.starts_with(" :") {
488            Some(&self.raw[2..])
489        } else {
490            None
491        }
492    }
493}
494
495#[derive(Clone)]
496pub struct PartialCfg<'a> {
497    pub tags: HashSet<&'a str>,
498    pub prefix: Option<(bool, bool)>,
499    pub command: bool,
500    pub params: Vec<usize>,
501    pub trailing: bool,
502}
503
504impl<'a> Default for PartialCfg<'a> {
505    fn default() -> Self {
506        Self {
507            tags: HashSet::new(),
508            prefix: None,
509            command: true,
510            params: Vec::new(),
511            trailing: false,
512        }
513    }
514}
515
516#[cfg(test)]
517mod tests {
518    use crate::tokenizer::{ParserError, Tokenizer};
519    use std::error::Error;
520
521    #[test]
522    fn test_empty() {
523        assert_eq!(Err(ParserError::NoCommand), Tokenizer::new(""));
524    }
525
526    #[test]
527    fn test_command_only() -> Result<(), Box<dyn Error>> {
528        let mut tokenizer = Tokenizer::new("CMD")?.tags();
529        assert_eq!(None, tokenizer.as_iter().next());
530        let mut tokenizer = tokenizer.prefix();
531        assert_eq!(None, tokenizer.parts()?);
532        let mut tokenizer = tokenizer.command();
533        assert_eq!("CMD", tokenizer.command()?);
534        let mut tokenizer = tokenizer.params();
535        assert_eq!(None, tokenizer.as_iter().next());
536        assert_eq!(None, tokenizer.trailing().trailing());
537
538        Ok(())
539    }
540
541    #[test]
542    fn test_tag() -> Result<(), Box<dyn Error>> {
543        let mut tokenizer = Tokenizer::new("@key1=value1 CMD")?.tags();
544        let mut iter = tokenizer.as_iter();
545        assert_eq!(Some(Ok(("key1", "value1"))), iter.next());
546        assert_eq!(None, iter.next());
547        let mut tokenizer = tokenizer.prefix();
548        assert_eq!(None, tokenizer.parts()?);
549        let mut tokenizer = tokenizer.command();
550        assert_eq!("CMD", tokenizer.command()?);
551        let mut tokenizer = tokenizer.params();
552        let mut iter = tokenizer.as_iter();
553        assert_eq!(None, iter.next());
554        assert_eq!(None, tokenizer.trailing().trailing());
555
556        Ok(())
557    }
558
559    #[test]
560    fn test_tags() -> Result<(), Box<dyn Error>> {
561        let mut tokenizer = Tokenizer::new("@key1=value1;key2=value2;key3=;key4;key5 CMD")?.tags();
562        let mut iter = tokenizer.as_iter();
563        assert_eq!(Some(Ok(("key1", "value1"))), iter.next());
564        assert_eq!(Some(Ok(("key2", "value2"))), iter.next());
565        assert_eq!(Some(Ok(("key3", ""))), iter.next());
566        assert_eq!(Some(Ok(("key4", ""))), iter.next());
567        assert_eq!(Some(Ok(("key5", ""))), iter.next());
568        assert_eq!(None, iter.next());
569        let mut tokenizer = tokenizer.prefix();
570        assert_eq!(None, tokenizer.parts()?);
571        let mut tokenizer = tokenizer.command();
572        assert_eq!("CMD", tokenizer.command()?);
573        let mut tokenizer = tokenizer.params();
574        let mut iter = tokenizer.as_iter();
575        assert_eq!(None, iter.next());
576        assert_eq!(None, tokenizer.trailing().trailing());
577
578        Ok(())
579    }
580
581    #[test]
582    fn test_prefix() -> Result<(), Box<dyn Error>> {
583        let mut tokenizer = Tokenizer::new(":name!user@host CMD")?.tags();
584        let mut iter = tokenizer.as_iter();
585        assert_eq!(None, iter.next());
586        let mut tokenizer = tokenizer.prefix();
587        assert_eq!(
588            Some(("name", Some("user"), Some("host"))),
589            tokenizer.parts()?
590        );
591        let mut tokenizer = tokenizer.command();
592        assert_eq!("CMD", tokenizer.command()?);
593        let mut tokenizer = tokenizer.params();
594        let mut iter = tokenizer.as_iter();
595        assert_eq!(None, iter.next());
596        assert_eq!(None, tokenizer.trailing().trailing());
597
598        Ok(())
599    }
600
601    #[test]
602    fn test_prefix_name() -> Result<(), Box<dyn Error>> {
603        let mut tokenizer = Tokenizer::new(":name CMD")?.tags();
604        let mut iter = tokenizer.as_iter();
605        assert_eq!(None, iter.next());
606        let mut tokenizer = tokenizer.prefix();
607        assert_eq!(Some(("name", None, None)), tokenizer.parts()?);
608        let mut tokenizer = tokenizer.command();
609        assert_eq!("CMD", tokenizer.command()?);
610        let mut tokenizer = tokenizer.params();
611        let mut iter = tokenizer.as_iter();
612        assert_eq!(None, iter.next());
613        assert_eq!(None, tokenizer.trailing().trailing());
614
615        Ok(())
616    }
617
618    #[test]
619    fn test_prefix_user() -> Result<(), Box<dyn Error>> {
620        let mut tokenizer = Tokenizer::new(":name!user CMD")?.tags();
621        let mut iter = tokenizer.as_iter();
622        assert_eq!(None, iter.next());
623        let mut tokenizer = tokenizer.prefix();
624        assert_eq!(Some(("name", Some("user"), None)), tokenizer.parts()?);
625        let mut tokenizer = tokenizer.command();
626        assert_eq!("CMD", tokenizer.command()?);
627        let mut tokenizer = tokenizer.params();
628        let mut iter = tokenizer.as_iter();
629        assert_eq!(None, iter.next());
630        assert_eq!(None, tokenizer.trailing().trailing());
631
632        Ok(())
633    }
634
635    #[test]
636    fn test_prefix_host() -> Result<(), Box<dyn Error>> {
637        let mut tokenizer = Tokenizer::new(":name@host CMD")?.tags();
638        let mut iter = tokenizer.as_iter();
639        assert_eq!(None, iter.next());
640        let mut tokenizer = tokenizer.prefix();
641        assert_eq!(Some(("name", None, Some("host"))), tokenizer.parts()?);
642        let mut tokenizer = tokenizer.command();
643        assert_eq!("CMD", tokenizer.command()?);
644        let mut tokenizer = tokenizer.params();
645        let mut iter = tokenizer.as_iter();
646        assert_eq!(None, iter.next());
647        assert_eq!(None, tokenizer.trailing().trailing());
648
649        Ok(())
650    }
651
652    #[test]
653    fn test_params() -> Result<(), Box<dyn Error>> {
654        let mut tokenizer = Tokenizer::new("CMD param0 param1")?.tags();
655        let mut iter = tokenizer.as_iter();
656        assert_eq!(None, iter.next());
657        let mut tokenizer = tokenizer.prefix();
658        assert_eq!(None, tokenizer.parts()?);
659        let mut tokenizer = tokenizer.command();
660        assert_eq!("CMD", tokenizer.command()?);
661        let mut tokenizer = tokenizer.params();
662        let mut iter = tokenizer.as_iter();
663        assert_eq!(Some("param0"), iter.next());
664        assert_eq!(Some("param1"), iter.next());
665        assert_eq!(None, iter.next());
666        assert_eq!(None, tokenizer.trailing().trailing());
667
668        Ok(())
669    }
670
671    #[test]
672    fn test_params_trailing() -> Result<(), Box<dyn Error>> {
673        let mut tokenizer = Tokenizer::new("CMD param0 param1 :Trailing parameter!")?.tags();
674        let mut iter = tokenizer.as_iter();
675        assert_eq!(None, iter.next());
676        let mut tokenizer = tokenizer.prefix();
677        assert_eq!(None, tokenizer.parts()?);
678        let mut tokenizer = tokenizer.command();
679        assert_eq!("CMD", tokenizer.command()?);
680        let mut tokenizer = tokenizer.params();
681        let mut iter = tokenizer.as_iter();
682        assert_eq!(Some("param0"), iter.next());
683        assert_eq!(Some("param1"), iter.next());
684        assert_eq!(None, iter.next());
685        assert_eq!(Some("Trailing parameter!"), tokenizer.trailing().trailing());
686
687        Ok(())
688    }
689
690    #[test]
691    fn test_trailing() -> Result<(), Box<dyn Error>> {
692        let mut tokenizer = Tokenizer::new("CMD :Trailing parameter!")?.tags();
693        let mut iter = tokenizer.as_iter();
694        assert_eq!(None, iter.next());
695        let mut tokenizer = tokenizer.prefix();
696        assert_eq!(None, tokenizer.parts()?);
697        let mut tokenizer = tokenizer.command();
698        assert_eq!("CMD", tokenizer.command()?);
699        let mut tokenizer = tokenizer.params();
700        let mut iter = tokenizer.as_iter();
701        assert_eq!(None, iter.next());
702        assert_eq!(Some("Trailing parameter!"), tokenizer.trailing().trailing());
703
704        Ok(())
705    }
706
707    #[test]
708    fn test_all() -> Result<(), Box<dyn Error>> {
709        let mut tokenizer = Tokenizer::new(
710            "@key1=value1;key2=value2 :name!user@host CMD param0 param1 :Trailing parameter!@:=;",
711        )?
712        .tags();
713        let mut iter = tokenizer.as_iter();
714        assert_eq!(Some(Ok(("key1", "value1"))), iter.next());
715        assert_eq!(Some(Ok(("key2", "value2"))), iter.next());
716        let mut tokenizer = tokenizer.prefix();
717        assert_eq!(
718            Some(("name", Some("user"), Some("host"))),
719            tokenizer.parts()?
720        );
721        let mut tokenizer = tokenizer.command();
722        assert_eq!("CMD", tokenizer.command()?);
723        let mut tokenizer = tokenizer.params();
724        let mut iter = tokenizer.as_iter();
725        assert_eq!(Some("param0"), iter.next());
726        assert_eq!(Some("param1"), iter.next());
727        assert_eq!(
728            Some("Trailing parameter!@:=;"),
729            tokenizer.trailing().trailing()
730        );
731
732        Ok(())
733    }
734
735    #[test]
736    fn test_only_trailing() -> Result<(), Box<dyn Error>> {
737        let tokenizer = Tokenizer::new(
738            "@key1=value1;key2=value2 :name!user@host CMD param0 param1 :Trailing parameter!@:=;",
739        )?;
740        assert_eq!(
741            Some("Trailing parameter!@:=;"),
742            tokenizer.trailing().trailing()
743        );
744
745        Ok(())
746    }
747}
irc_rust/tokenizer.rs

irc_rust/
tokenizer.rs
