55 lines
1.4 KiB
Python
55 lines
1.4 KiB
Python
from __future__ import annotations
|
|
from dataclasses import dataclass
|
|
from typing import Union
|
|
|
|
|
|
class LexError(Exception):
|
|
pass
|
|
|
|
|
|
@dataclass
|
|
class Token:
|
|
kind: str
|
|
value: Union[int, float, str, None]
|
|
|
|
|
|
def tokenize(src: str) -> list[Token]:
|
|
tokens: list[Token] = []
|
|
i = 0
|
|
n = len(src)
|
|
while i < n:
|
|
ch = src[i]
|
|
if ch in ' \t\r\n':
|
|
i += 1
|
|
continue
|
|
if ch.isdigit() or ch == '.':
|
|
j = i
|
|
while j < n and src[j].isdigit():
|
|
j += 1
|
|
if j < n and src[j] == '.':
|
|
j += 1
|
|
while j < n and src[j].isdigit():
|
|
j += 1
|
|
tokens.append(Token('NUMBER', float(src[i:j])))
|
|
else:
|
|
tokens.append(Token('NUMBER', int(src[i:j])))
|
|
i = j
|
|
continue
|
|
if ch == '+':
|
|
tokens.append(Token('PLUS', '+'))
|
|
elif ch == '-':
|
|
tokens.append(Token('MINUS', '-'))
|
|
elif ch == '*':
|
|
tokens.append(Token('STAR', '*'))
|
|
elif ch == '/':
|
|
tokens.append(Token('SLASH', '/'))
|
|
elif ch == '(':
|
|
tokens.append(Token('LPAREN', '('))
|
|
elif ch == ')':
|
|
tokens.append(Token('RPAREN', ')'))
|
|
else:
|
|
raise LexError(f"unexpected character {ch!r} at position {i}")
|
|
i += 1
|
|
tokens.append(Token('EOF', None))
|
|
return tokens
|