Skip to main content

tokenize

Function tokenize 

Source
pub fn tokenize<'a, K>(text: &'a str) -> Vec<FatToken<K>>
where K: TokenTrait + Logos<'a, Source = str>, <K as Logos<'a>>::Extras: Default,