Struct biome_rowan::syntax::SyntaxToken

pub struct SyntaxToken<L: Language> { /* private fields */ }

Implementations

impl<L: Language> SyntaxToken<L>

pub fn new_detached<Leading, Trailing>( kind: L::Kind, text: &str, leading: Leading, trailing: Trailing ) -> Self

where Leading: IntoIterator<Item = TriviaPiece>, Leading::IntoIter: ExactSizeIterator, Trailing: IntoIterator<Item = TriviaPiece>, Trailing::IntoIter: ExactSizeIterator,

Create a new token detached from any tree

This is mainly useful for creating a small number of individual tokens when mutating an existing tree, the bulk of the tokens in a given file should be created through the crate::TreeBuilder abstraction instead as it will efficiently cache and reuse the created tokens

pub fn key(&self) -> SyntaxElementKey

pub fn kind(&self) -> L::Kind

pub fn text_range(&self) -> TextRange

pub fn text_trimmed_range(&self) -> TextRange

pub fn text(&self) -> &str

Returns the text of the token, including all trivia.

use biome_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
use biome_rowan::*;
let mut token = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
    builder.token_with_trivia(
        RawLanguageKind::LET_TOKEN,
        "\n\t let \t\t",
        &[TriviaPiece::whitespace(3)],
        &[TriviaPiece::whitespace(3)],
    );
})
.first_token()
.unwrap();
assert_eq!("\n\t let \t\t", token.text());

pub fn token_text(&self) -> TokenText

Returns the text of a token, including all trivia as an owned value.

use biome_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
use biome_rowan::*;
let mut token = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
    builder.token_with_trivia(
        RawLanguageKind::LET_TOKEN,
        "\n\t let \t\t",
        &[TriviaPiece::whitespace(3)],
        &[TriviaPiece::whitespace(3)],
    );
})
.first_token()
.unwrap();
assert_eq!("\n\t let \t\t", token.token_text());
assert_eq!(
    format!("{}", "\n\t let \t\t"),
    format!("{}", token.token_text())
);
assert_eq!(
    format!("{:?}", "\n\t let \t\t"),
    format!("{:?}", token.token_text())
);

pub fn token_text_trimmed(&self) -> TokenText

pub fn text_trimmed(&self) -> &str

Returns the text of the token, excluding all trivia.

use biome_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
use biome_rowan::*;
let mut token = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
    builder.token_with_trivia(
        RawLanguageKind::LET_TOKEN,
        "\n\t let \t\t",
        &[TriviaPiece::whitespace(3)],
        &[TriviaPiece::whitespace(3)],
    );
})
.first_token()
.unwrap();
assert_eq!("let", token.text_trimmed());

pub fn parent(&self) -> Option<SyntaxNode<L>>

pub fn ancestors(&self) -> impl Iterator<Item = SyntaxNode<L>>

pub fn next_sibling_or_token(&self) -> Option<SyntaxElement<L>>

pub fn prev_sibling_or_token(&self) -> Option<SyntaxElement<L>>

pub fn siblings_with_tokens( &self, direction: Direction ) -> impl Iterator<Item = SyntaxElement<L>>

pub fn next_token(&self) -> Option<SyntaxToken<L>>

Next token in the tree (i.e, not necessary a sibling).

pub fn prev_token(&self) -> Option<SyntaxToken<L>>

Previous token in the tree (i.e, not necessary a sibling).

pub fn detach(self) -> Self

Return a new version of this token detached from its parent node

pub fn with_leading_trivia<'a, I>(&self, trivia: I) -> Self

where I: IntoIterator<Item = (TriviaPieceKind, &'a str)>, I::IntoIter: ExactSizeIterator,

Return a new version of this token with its leading trivia replaced with trivia

pub fn with_leading_trivia_pieces<I>(&self, trivia: I) -> Self

where I: IntoIterator<Item = SyntaxTriviaPiece<L>>, I::IntoIter: ExactSizeIterator,

Return a new version of this token with its leading trivia replaced with trivia

pub fn with_trailing_trivia<'a, I>(&self, trivia: I) -> Self

where I: IntoIterator<Item = (TriviaPieceKind, &'a str)>, I::IntoIter: ExactSizeIterator,

Return a new version of this token with its trailing trivia replaced with trivia

pub fn with_trailing_trivia_pieces<I>(&self, trivia: I) -> Self

where I: IntoIterator<Item = SyntaxTriviaPiece<L>>, I::IntoIter: ExactSizeIterator,

Return a new version of this token with its trailing trivia replaced with trivia

pub fn prepend_trivia_pieces<I>(&self, trivia: I) -> Self

where I: IntoIterator<Item = SyntaxTriviaPiece<L>>, I::IntoIter: ExactSizeIterator,

Examples

use biome_rowan::raw_language::{RawLanguage, RawLanguageKind};
use biome_rowan::{RawSyntaxToken, SyntaxToken, TriviaPiece};

let token = SyntaxToken::<RawLanguage>::new_detached(
    RawLanguageKind::LET_TOKEN,
    "/*c*/ let \t",
    [TriviaPiece::multi_line_comment(5), TriviaPiece::whitespace(1)],
    [TriviaPiece::whitespace(2)]
);
let new_token = token.prepend_trivia_pieces(token.trailing_trivia().pieces());

assert_eq!(
    format!("{:?}", new_token),
    "LET_TOKEN@0..13 \"let\" [Whitespace(\" \\t\"), Comments(\"/*c*/\"), Whitespace(\" \")] [Whitespace(\" \\t\")]"
);

pub fn append_trivia_pieces<I>(&self, trivia: I) -> Self

where I: IntoIterator<Item = SyntaxTriviaPiece<L>>, I::IntoIter: ExactSizeIterator,

Examples

use biome_rowan::raw_language::{RawLanguage, RawLanguageKind};
use biome_rowan::{RawSyntaxToken, SyntaxToken, TriviaPiece};

let token = SyntaxToken::<RawLanguage>::new_detached(
    RawLanguageKind::LET_TOKEN,
    "\t let /*c*/",
    [TriviaPiece::whitespace(2)],
    [TriviaPiece::whitespace(1), TriviaPiece::multi_line_comment(5)],
);
let new_token = token.append_trivia_pieces(token.leading_trivia().pieces());

assert_eq!(
    format!("{:?}", new_token),
    "LET_TOKEN@0..13 \"let\" [Whitespace(\"\\t \")] [Whitespace(\" \"), Comments(\"/*c*/\"), Whitespace(\"\\t \")]"
);

pub fn trim_leading_trivia(&self) -> Self

Return a new version of this token without leading newlines and whitespaces.

Examples

use biome_rowan::raw_language::{RawLanguage, RawLanguageKind};
use biome_rowan::{RawSyntaxToken, SyntaxToken, TriviaPiece};

let token = SyntaxToken::<RawLanguage>::new_detached(
    RawLanguageKind::LET_TOKEN,
    "\n\t /*c*/ let \t",
    [TriviaPiece::newline(1), TriviaPiece::whitespace(2), TriviaPiece::multi_line_comment(5), TriviaPiece::whitespace(1)],
    [TriviaPiece::whitespace(2)]
);
let new_token = token.trim_leading_trivia();

assert_eq!(
    format!("{:?}", new_token),
    "LET_TOKEN@0..11 \"let\" [Comments(\"/*c*/\"), Whitespace(\" \")] [Whitespace(\" \\t\")]"
);

pub fn trim_trailing_trivia(&self) -> Self

Return a new version of this token without trailing whitespaces.

Examples

use biome_rowan::raw_language::{RawLanguage, RawLanguageKind};
use biome_rowan::{RawSyntaxToken, SyntaxToken, TriviaPiece};

let token = SyntaxToken::<RawLanguage>::new_detached(
    RawLanguageKind::LET_TOKEN,
    "\n let /*c*/\t ",
    [TriviaPiece::newline(1), TriviaPiece::whitespace(1)],
    [TriviaPiece::whitespace(1), TriviaPiece::multi_line_comment(5), TriviaPiece::whitespace(2)]
);
let new_token = token.trim_trailing_trivia();

assert_eq!(
    format!("{:?}", new_token),
    "LET_TOKEN@0..11 \"let\" [Newline(\"\\n\"), Whitespace(\" \")] [Whitespace(\" \"), Comments(\"/*c*/\")]"
);

pub fn indentation_trivia_pieces( &self ) -> impl ExactSizeIterator<Item = SyntaxTriviaPiece<L>>

Return whitespace that juxtapose the token until the first non-whitespace item.

pub fn leading_trivia(&self) -> SyntaxTrivia<L>

Returns the token’s leading trivia.

Looking backward in the text, a token owns all of its preceding trivia up to and including the first newline character.

use biome_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
use biome_rowan::*;
let mut token = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
    builder.token_with_trivia(
        RawLanguageKind::LET_TOKEN,
        "\n\t let \t\t",
        &[TriviaPiece::whitespace(3)],
        &[TriviaPiece::whitespace(3)],
    );
})
.first_token()
.unwrap();
assert_eq!("\n\t ", token.leading_trivia().text());

pub fn trailing_trivia(&self) -> SyntaxTrivia<L>

Returns the token’s trailing trivia.

A token owns all of its following trivia up to, but not including, the next newline character.

use biome_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
use biome_rowan::*;
let mut token = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
    builder.token_with_trivia(
        RawLanguageKind::LET_TOKEN,
        "\n\t let \t\t",
        &[TriviaPiece::whitespace(3)],
        &[TriviaPiece::whitespace(3)],
    );
})
.first_token()
.unwrap();
assert_eq!(" \t\t", token.trailing_trivia().text());

pub fn has_trailing_comments(&self) -> bool

Checks if the current token has trailing comments

pub fn has_leading_comments(&self) -> bool

Checks if the current token has leading comments

pub fn has_leading_non_whitespace_trivia(&self) -> bool

Checks if the token has any leading trivia that isn’t a whitespace nor a line break

pub fn has_leading_newline(&self) -> bool

Checks if the current token has leading newline

Trait Implementations

impl<L: Clone + Language> Clone for SyntaxToken<L>

fn clone(&self) -> SyntaxToken<L>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<L: Language> Debug for SyntaxToken<L>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<L: Language> Display for SyntaxToken<L>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<L: Language> From<SyntaxToken<L>> for SyntaxElement<L>

fn from(token: SyntaxToken<L>) -> SyntaxElement<L>

Converts to this type from the input type.

impl<L: Hash + Language> Hash for SyntaxToken<L>

fn hash<__H: Hasher>(&self, state: &mut __H)

Feeds this value into the given Hasher.

fn hash_slice<H>(data: &[Self], state: &mut H)

where H: Hasher, Self: Sized,

Feeds a slice of this type into the given Hasher.

impl<L: PartialEq + Language> PartialEq for SyntaxToken<L>

fn eq(&self, other: &SyntaxToken<L>) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl<L: Eq + Language> Eq for SyntaxToken<L>

impl<L: Language> StructuralPartialEq for SyntaxToken<L>