chordsketch_core/
token.rs

1//! Token and span types for the ChordPro lexer.
2//!
3//! This module defines the token types produced by the lexer. Tokens represent
4//! the smallest meaningful units in a ChordPro document. The lexer does not
5//! understand the structure of the document (that is the parser's job); it only
6//! identifies individual tokens and their positions.
7
8/// A position in the source text, identified by line and column numbers.
9///
10/// Both `line` and `column` are 1-based, matching the conventions used by
11/// editors and error messages.
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13pub struct Position {
14    /// 1-based line number.
15    pub line: usize,
16    /// 1-based column number (in characters, not bytes).
17    pub column: usize,
18}
19
20impl Position {
21    /// Creates a new `Position` with the given line and column.
22    #[must_use]
23    pub fn new(line: usize, column: usize) -> Self {
24        Self { line, column }
25    }
26}
27
28/// A span in the source text, defined by a start and end position.
29///
30/// The start position is inclusive and the end position is exclusive, following
31/// the convention of half-open intervals.
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
33pub struct Span {
34    /// The start position (inclusive).
35    pub start: Position,
36    /// The end position (exclusive).
37    pub end: Position,
38}
39
40impl Span {
41    /// Creates a new `Span` from the given start and end positions.
42    #[must_use]
43    pub fn new(start: Position, end: Position) -> Self {
44        Self { start, end }
45    }
46}
47
48/// The kind of a token.
49///
50/// These represent the distinct syntactic elements that the lexer recognizes
51/// in a ChordPro document.
52#[derive(Debug, Clone, PartialEq, Eq)]
53pub enum TokenKind {
54    /// Opening brace `{` — starts a directive.
55    DirectiveOpen,
56    /// Closing brace `}` — ends a directive.
57    DirectiveClose,
58    /// Opening bracket `[` — starts a chord annotation.
59    ChordOpen,
60    /// Closing bracket `]` — ends a chord annotation.
61    ChordClose,
62    /// Colon `:` — separates a directive name from its value.
63    ///
64    /// Only emitted when the lexer is inside a directive (between `{` and `}`).
65    Colon,
66    /// A run of text content (lyrics, directive names, directive values, chord
67    /// names, etc.).
68    ///
69    /// The lexer does not interpret text — it simply captures contiguous runs
70    /// of characters that are not special delimiters.
71    Text(String),
72    /// A newline character (`\n` or `\r\n`).
73    Newline,
74    /// End of input.
75    Eof,
76}
77
78/// A token produced by the lexer.
79///
80/// Each token carries its [`TokenKind`] and the [`Span`] that locates it in
81/// the original source text.
82#[derive(Debug, Clone, PartialEq, Eq)]
83pub struct Token {
84    /// The kind of this token.
85    pub kind: TokenKind,
86    /// The location of this token in the source text.
87    pub span: Span,
88}
89
90impl Token {
91    /// Creates a new `Token` with the given kind and span.
92    #[must_use]
93    pub fn new(kind: TokenKind, span: Span) -> Self {
94        Self { kind, span }
95    }
96}
97
98#[cfg(test)]
99mod tests {
100    use super::*;
101
102    #[test]
103    fn position_new() {
104        let pos = Position::new(1, 5);
105        assert_eq!(pos.line, 1);
106        assert_eq!(pos.column, 5);
107    }
108
109    #[test]
110    fn span_new() {
111        let span = Span::new(Position::new(1, 1), Position::new(1, 5));
112        assert_eq!(span.start, Position::new(1, 1));
113        assert_eq!(span.end, Position::new(1, 5));
114    }
115
116    #[test]
117    fn token_new() {
118        let span = Span::new(Position::new(1, 1), Position::new(1, 2));
119        let token = Token::new(TokenKind::DirectiveOpen, span);
120        assert_eq!(token.kind, TokenKind::DirectiveOpen);
121        assert_eq!(token.span, span);
122    }
123
124    #[test]
125    fn token_kind_text_equality() {
126        let a = TokenKind::Text("hello".to_string());
127        let b = TokenKind::Text("hello".to_string());
128        let c = TokenKind::Text("world".to_string());
129        assert_eq!(a, b);
130        assert_ne!(a, c);
131    }
132}
chordsketch_core/token.rs

chordsketch_core/
token.rs
