from dataclasses import dataclass from typing import Union class LexError(Exception): pass @dataclass class Token: kind: str value: Union[int, float, str, None] def tokenize(src: str) -> list: tokens = [] i = 0 while i < len(src): ch = src[i] if ch in ' \t': i += 1 continue if ch == '+': tokens.append(Token('PLUS', '+')) i += 1 elif ch == '-': tokens.append(Token('MINUS', '-')) i += 1 elif ch == '*': tokens.append(Token('STAR', '*')) i += 1 elif ch == '/': tokens.append(Token('SLASH', '/')) i += 1 elif ch == '(': tokens.append(Token('LPAREN', '(')) i += 1 elif ch == ')': tokens.append(Token('RPAREN', ')')) i += 1 elif ch.isdigit() or ch == '.': start = i while i < len(src) and (src[i].isdigit() or src[i] == '.'): i += 1 raw = src[start:i] if '.' in raw: tokens.append(Token('NUMBER', float(raw))) else: tokens.append(Token('NUMBER', int(raw))) else: raise LexError(f"unexpected character {ch!r} at position {i}") tokens.append(Token('EOF', None)) return tokens