import { describe, it, expect } from 'vitest'; import { tokenize } from '../lexer'; const types = (s: string) => tokenize(s).map((t) => t.type); const values = (s: string) => tokenize(s).map((t) => t.value); describe('lexer', () => { it('tokenizes numbers incl. decimals and exponents', () => { expect(values('1 2.5 .5 1e3 1.2E-3')).toEqual([ '1', '2.5', '.5', '1e3', '1.2E-3', '', ]); expect(types('42')).toEqual(['NUMBER', 'EOF']); }); it('tokenizes operators and punctuation', () => { expect(types('+-*/^&():;!')).toEqual([ 'PLUS', 'MINUS', 'STAR', 'SLASH', 'CARET', 'AMP', 'LPAREN', 'RPAREN', 'COLON', 'SEMICOLON', 'BANG', 'EOF', ]); }); it('reads bare words as IDENT (cell refs, functions, sheet names alike)', () => { expect(types('SUM(A1:B2)')).toEqual([ 'IDENT', 'LPAREN', 'IDENT', 'COLON', 'IDENT', 'RPAREN', 'EOF', ]); }); it('reads single-quoted sheet names with spaces', () => { const t = tokenize("'Kosten Q3'!B2"); expect(t[0]).toMatchObject({ type: 'QUOTED', value: 'Kosten Q3' }); expect(t[1].type).toBe('BANG'); expect(t[2]).toMatchObject({ type: 'IDENT', value: 'B2' }); }); it('handles escaped double-quoted strings', () => { expect(tokenize('"a\\"b"')[0]).toMatchObject({ type: 'STRING', value: 'a"b' }); }); it('reads $-locked cell references as one IDENT', () => { expect(types('$A$1 A$1 $A1')).toEqual(['IDENT', 'IDENT', 'IDENT', 'EOF']); expect(values('$A$1 A$1 $A1')).toEqual(['$A$1', 'A$1', '$A1', '']); }); it('reads identifiers with non-ASCII letters (umlauts etc.) as a single IDENT', () => { // a table/sheet name routinely contains "ä"/"ö"/"ü"/"ß" in this // German-first app -- these must lex as ordinary IDENT characters, not // throw, so an unquoted qualifier like "Übersicht::A1" actually works. expect(types('Übersicht::B4')).toEqual(['IDENT', 'DBLCOLON', 'IDENT', 'EOF']); expect(values('Übersicht')).toEqual(['Übersicht', '']); expect(values('Größe_Prämie')).toEqual(['Größe_Prämie', '']); }); it('skips whitespace', () => { expect(types(' A1\t+\n B2 ')).toEqual(['IDENT', 'PLUS', 'IDENT', 'EOF']); }); it('throws on an unexpected character', () => { expect(() => tokenize('A1 @ B2')).toThrow(); }); it('tokenizes assignment/equality and comparison operators', () => { expect(types('= < > <= >= <>')).toEqual([ 'EQUALS', 'LT', 'GT', 'LTE', 'GTE', 'NEQ', 'EOF', ]); }); it('prefers the two-character comparison operators over their single-character prefix', () => { expect(types('A1<=B2')).toEqual(['IDENT', 'LTE', 'IDENT', 'EOF']); expect(types('A1>=B2')).toEqual(['IDENT', 'GTE', 'IDENT', 'EOF']); expect(types('A1<>B2')).toEqual(['IDENT', 'NEQ', 'IDENT', 'EOF']); expect(types('A1B2')).toEqual(['IDENT', 'GT', 'IDENT', 'EOF']); }); it('tokenizes every error code as one ERRORLIT token', () => { expect(types('#REF')).toEqual(['ERRORLIT', 'EOF']); expect(values('#REF')).toEqual(['#REF', '']); expect(types('#DIV/0!')).toEqual(['ERRORLIT', 'EOF']); expect(types('#NAME?')).toEqual(['ERRORLIT', 'EOF']); expect(types('#ERR')).toEqual(['ERRORLIT', 'EOF']); expect(types('#CIRC')).toEqual(['ERRORLIT', 'EOF']); expect(types('#VALUE!')).toEqual(['ERRORLIT', 'EOF']); expect(types('#N/A')).toEqual(['ERRORLIT', 'EOF']); }); it('an ERRORLIT is followed by whatever comes next, not swallowed into it', () => { expect(types('1+#REF')).toEqual(['NUMBER', 'PLUS', 'ERRORLIT', 'EOF']); // "#REF" matches greedily -- a trailing "!" some might expect (Excel // spells it "#REF!") is its own BANG token, not part of the literal; // this engine's own error code is "#REF" without "!" (see types.ts). expect(types('#REF!')).toEqual(['ERRORLIT', 'BANG', 'EOF']); }); it('throws on an unrecognized "#..." sequence', () => { expect(() => tokenize('#FOO')).toThrow(); }); });