Lines Matching refs:tokens

5 fn concat_helper(tokens: &[TokenTree]) -> Vec<(String, Span)> {
6     let mut tokens = tokens.iter();
10         match tokens.next() {
29                 let Some(TokenTree::Ident(ident)) = tokens.next() else {
50                 let tokens = group.stream().into_iter().collect::<Vec<TokenTree>>();
51                 segments.append(&mut concat_helper(tokens.as_slice()));
60 fn concat(tokens: &[TokenTree], group_span: Span) -> TokenTree {
61     let segments = concat_helper(tokens);
66 pub(crate) fn expand(tokens: &mut Vec<TokenTree>) {
67     for token in tokens.iter_mut() {
81                 // Recursively expand tokens inside the group
91     for i in (0..tokens.len().saturating_sub(3)).rev() {
94             (&tokens[i + 1], &tokens[i + 2]),
98             match &tokens[i + 3] {
100                     tokens.splice(i + 3..i + 4, group.stream());
105             match &tokens[i] {
107                     tokens.splice(i..i + 1, group.stream());