numolus/app/frontend/engine/__tests__/lexer.spec.ts

import { describe, it, expect } from 'vitest';
import { tokenize } from '../lexer';

const types = (s: string) => tokenize(s).map((t) => t.type);
const values = (s: string) => tokenize(s).map((t) => t.value);

describe('lexer', () => {
  it('tokenizes numbers incl. decimals and exponents', () => {
    expect(values('1 2.5 .5 1e3 1.2E-3')).toEqual([
      '1',
      '2.5',
      '.5',
      '1e3',
      '1.2E-3',
      '',
    ]);
    expect(types('42')).toEqual(['NUMBER', 'EOF']);
  });

  it('tokenizes operators and punctuation', () => {
    expect(types('+-*/^&():;!')).toEqual([
      'PLUS',
      'MINUS',
      'STAR',
      'SLASH',
      'CARET',
      'AMP',
      'LPAREN',
      'RPAREN',
      'COLON',
      'SEMICOLON',
      'BANG',
      'EOF',
    ]);
  });

  it('reads bare words as IDENT (cell refs, functions, sheet names alike)', () => {
    expect(types('SUM(A1:B2)')).toEqual([
      'IDENT',
      'LPAREN',
      'IDENT',
      'COLON',
      'IDENT',
      'RPAREN',
      'EOF',
    ]);
  });

  it('reads single-quoted sheet names with spaces', () => {
    const t = tokenize("'Kosten Q3'!B2");
    expect(t[0]).toMatchObject({ type: 'QUOTED', value: 'Kosten Q3' });
    expect(t[1].type).toBe('BANG');
    expect(t[2]).toMatchObject({ type: 'IDENT', value: 'B2' });
  });

  it('handles escaped double-quoted strings', () => {
    expect(tokenize('"a\\"b"')[0]).toMatchObject({ type: 'STRING', value: 'a"b' });
  });

  it('reads $-locked cell references as one IDENT', () => {
    expect(types('$A$1 A$1 $A1')).toEqual(['IDENT', 'IDENT', 'IDENT', 'EOF']);
    expect(values('$A$1 A$1 $A1')).toEqual(['$A$1', 'A$1', '$A1', '']);
  });

  it('reads identifiers with non-ASCII letters (umlauts etc.) as a single IDENT', () => {
    // a table/sheet name routinely contains "ä"/"ö"/"ü"/"ß" in this
    // German-first app -- these must lex as ordinary IDENT characters, not
    // throw, so an unquoted qualifier like "Übersicht::A1" actually works.
    expect(types('Übersicht::B4')).toEqual(['IDENT', 'DBLCOLON', 'IDENT', 'EOF']);
    expect(values('Übersicht')).toEqual(['Übersicht', '']);
    expect(values('Größe_Prämie')).toEqual(['Größe_Prämie', '']);
  });

  it('skips whitespace', () => {
    expect(types('  A1\t+\n B2 ')).toEqual(['IDENT', 'PLUS', 'IDENT', 'EOF']);
  });

  it('throws on an unexpected character', () => {
    expect(() => tokenize('A1 @ B2')).toThrow();
  });

  it('tokenizes assignment/equality and comparison operators', () => {
    expect(types('= < > <= >= <>')).toEqual([
      'EQUALS',
      'LT',
      'GT',
      'LTE',
      'GTE',
      'NEQ',
      'EOF',
    ]);
  });

  it('prefers the two-character comparison operators over their single-character prefix', () => {
    expect(types('A1<=B2')).toEqual(['IDENT', 'LTE', 'IDENT', 'EOF']);
    expect(types('A1>=B2')).toEqual(['IDENT', 'GTE', 'IDENT', 'EOF']);
    expect(types('A1<>B2')).toEqual(['IDENT', 'NEQ', 'IDENT', 'EOF']);
    expect(types('A1<B2')).toEqual(['IDENT', 'LT', 'IDENT', 'EOF']);
    expect(types('A1>B2')).toEqual(['IDENT', 'GT', 'IDENT', 'EOF']);
  });

  it('tokenizes every error code as one ERRORLIT token', () => {
    expect(types('#REF')).toEqual(['ERRORLIT', 'EOF']);
    expect(values('#REF')).toEqual(['#REF', '']);
    expect(types('#DIV/0!')).toEqual(['ERRORLIT', 'EOF']);
    expect(types('#NAME?')).toEqual(['ERRORLIT', 'EOF']);
    expect(types('#ERR')).toEqual(['ERRORLIT', 'EOF']);
    expect(types('#CIRC')).toEqual(['ERRORLIT', 'EOF']);
    expect(types('#VALUE!')).toEqual(['ERRORLIT', 'EOF']);
    expect(types('#N/A')).toEqual(['ERRORLIT', 'EOF']);
  });

  it('an ERRORLIT is followed by whatever comes next, not swallowed into it', () => {
    expect(types('1+#REF')).toEqual(['NUMBER', 'PLUS', 'ERRORLIT', 'EOF']);
    // "#REF" matches greedily -- a trailing "!" some might expect (Excel
    // spells it "#REF!") is its own BANG token, not part of the literal;
    // this engine's own error code is "#REF" without "!" (see types.ts).
    expect(types('#REF!')).toEqual(['ERRORLIT', 'BANG', 'EOF']);
  });

  it('throws on an unrecognized "#..." sequence', () => {
    expect(() => tokenize('#FOO')).toThrow();
  });
});