Spaces:
Build error
Build error
File size: 1,022 Bytes
51ff9e5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
import re
from pygments.lexers.python import PythonLexer
def tokenize_code(code):
lexer = PythonLexer()
tokens = process_pygments_tokens(lexer.get_tokens(code))
return tokens
def process_pygments_tokens(tokens):
new_tokens = []
for token in tokens:
if (
str(token[0]) == 'Token.Text'
and re.match(r'\s+', token[1])
or str(token[0]) == 'Token.Text.Whitespace'
):
continue
new_tokens.append(token[1])
new_tokens_final = []
i = 0
while i < len(new_tokens) - 2:
if (
new_tokens[i] == '"'
and new_tokens[i + 1] == 'STR'
and new_tokens[i + 2] == '"'
):
new_tokens_final.append('"STR"')
i = i + 3
else:
new_tokens_final.append(new_tokens[i])
i = i + 1
for i in range(len(new_tokens) - 2, len(new_tokens)):
if i >= 0:
new_tokens_final.append(new_tokens[i])
return new_tokens_final
|