Searched refs:CTokenizer (Results 1 – 4 of 4) sorted by relevance
169 class CTokenizer(): class342 self.sub_tokeninzer = CTokenizer(sub_str)455 new = CTokenizer()584 if isinstance(source, CTokenizer):588 tokenizer = CTokenizer(source)592 new_tokenizer = CTokenizer(tokenizer.tokens[start:end + 1])616 if isinstance(source, CTokenizer):621 tokenizer = CTokenizer(source)627 new_tokenizer = CTokenizer()640 new = CTokenizer(tokenizer.tokens[start:end + 1])
8 from kdoc.c_lex import CMatch, CTokenizer143 source = CTokenizer(source)
16 from kdoc.c_lex import CTokenizer, tokenizer_set_log89 tokens = CTokenizer(text)
18 from kdoc.c_lex import CToken, CTokenizer51 tokenizer = CTokenizer(data["source"])65 tokens = CTokenizer(data["source"]).tokens145 tokens = CTokenizer(data["source"])