from dataclasses import dataclass from typing import Union class LexError(Exception): pass @dataclass class Token: kind: str value: Union[int, float, None] = None def tokenize(src: str) -> list: tokens = [] i = 0 n = len(src) while i < n: ch = src[i] if ch in ' \t': i += 1 elif ch.isdigit() or ch == '.': j = i while j < n and src[j].isdigit(): j += 1 if j < n and src[j] == '.': j += 1 while j < n and src[j].isdigit(): j += 1 tokens.append(Token('NUMBER', float(src[i:j]))) else: tokens.append(Token('NUMBER', int(src[i:j]))) i = j elif ch == '+': tokens.append(Token('PLUS')) i += 1 elif ch == '-': tokens.append(Token('MINUS')) i += 1 elif ch == '*': tokens.append(Token('STAR')) i += 1 elif ch == '/': tokens.append(Token('SLASH')) i += 1 elif ch == '(': tokens.append(Token('LPAREN')) i += 1 elif ch == ')': tokens.append(Token('RPAREN')) i += 1 else: raise LexError(f"unexpected character {ch!r} at position {i}") tokens.append(Token('EOF')) return tokens