numolus/app/frontend/engine/lexer.ts

/**
 * Tokenizer for the formula language.
 *
 * The lexer is deliberately dumb: it splits the input into a flat token stream
 * and leaves *all* structural decisions to the parser. In particular it does not
 * try to tell a cell reference ("A1") from a function name ("SUM") or a sheet
 * name ("Umsatz") -- they are all `IDENT` tokens and the parser disambiguates
 * them by look-ahead (`(` -> function, `!` -> sheet qualifier, digits -> cell).
 */

import { ERROR_CODES } from './types';

export type TokenType =
  | 'NUMBER'
  | 'STRING'
  | 'IDENT' // bare word: A1, SUM, Region, Q1 ...
  | 'QUOTED' // 'Kosten Q3' -- a sheet name in single quotes
  | 'ERRORLIT' // a literal error code, e.g. "#REF" or "#DIV/0!" -- see ERROR_CODES
  | 'PLUS'
  | 'MINUS'
  | 'STAR'
  | 'SLASH'
  | 'CARET'
  | 'AMP' // "&"  -- text concatenation
  | 'LPAREN'
  | 'RPAREN'
  | 'SEMICOLON'
  | 'COLON'
  | 'DBLCOLON' // "::"  -- Numbers-style table qualifier ('Tabelle'::A1)
  | 'BANG'
  | 'EQUALS' // "="  -- assignment (block body) or equality comparison
  | 'NEQ' // "<>"
  | 'LT' // "<"
  | 'GT' // ">"
  | 'LTE' // "<="
  | 'GTE' // ">="
  | 'EOF';

export interface Token {
  type: TokenType;
  /** Raw text of the token (unquoted for STRING/QUOTED). */
  value: string;
  /** Character offset in the source, useful for error messages. */
  pos: number;
}

export class LexError extends Error {}

const SINGLE: Record<string, TokenType> = {
  '+': 'PLUS',
  '-': 'MINUS',
  '*': 'STAR',
  '/': 'SLASH',
  '^': 'CARET',
  '&': 'AMP',
  '(': 'LPAREN',
  ')': 'RPAREN',
  ';': 'SEMICOLON',
  ':': 'COLON',
  '!': 'BANG',
  '=': 'EQUALS',
  '<': 'LT',
  '>': 'GT',
};

function isDigit(ch: string): boolean {
  return ch >= '0' && ch <= '9';
}
/** Any Unicode letter, not just ASCII -- a table/sheet name (used unquoted
 *  as a formula qualifier whenever it has no spaces) routinely contains
 *  "ä"/"ö"/"ü"/"ß" or other accented letters in this German-first app;
 *  rejecting those at the lexer level made every such name effectively
 *  unusable as a bare qualifier (`Übersicht::A1` used to throw outright). */
const IS_LETTER = /\p{L}/u;
function isWordStart(ch: string): boolean {
  // '$' only ever appears inside a cell reference ("$A1", "A$1", "$A$1"), but
  // the lexer doesn't distinguish those from any other bare word (see the
  // module doc) -- it's just part of the word here, and the parser is the one
  // that validates/interprets it as a locked reference axis.
  return IS_LETTER.test(ch) || ch === '_' || ch === '$';
}
function isWordPart(ch: string): boolean {
  return isWordStart(ch) || isDigit(ch);
}

/** Longest-first, so a prefix of another code (none collide today, but this
 *  stays correct if one ever did) never wins the match by accident. */
const ERROR_CODES_BY_LENGTH = [...ERROR_CODES].sort((a, b) => b.length - a.length);

export function tokenize(src: string): Token[] {
  const tokens: Token[] = [];
  let i = 0;
  const n = src.length;

  while (i < n) {
    const ch = src[i];

    // whitespace
    if (ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r') {
      i++;
      continue;
    }

    // number: 12  12.5  .5  1e3  1.2E-3
    if (isDigit(ch) || (ch === '.' && isDigit(src[i + 1]))) {
      const start = i;
      while (i < n && isDigit(src[i])) i++;
      if (src[i] === '.') {
        i++;
        while (i < n && isDigit(src[i])) i++;
      }
      if (src[i] === 'e' || src[i] === 'E') {
        let j = i + 1;
        if (src[j] === '+' || src[j] === '-') j++;
        if (isDigit(src[j])) {
          i = j;
          while (i < n && isDigit(src[i])) i++;
        }
      }
      tokens.push({ type: 'NUMBER', value: src.slice(start, i), pos: start });
      continue;
    }

    // double-quoted string literal
    if (ch === '"') {
      const start = i;
      i++;
      let out = '';
      while (i < n && src[i] !== '"') {
        if (src[i] === '\\' && i + 1 < n) {
          out += src[i + 1];
          i += 2;
        } else {
          out += src[i++];
        }
      }
      if (src[i] !== '"') throw new LexError(`Unterminated string at ${start}`);
      i++; // closing quote
      tokens.push({ type: 'STRING', value: out, pos: start });
      continue;
    }

    // single-quoted sheet name: 'Kosten Q3'  ('' -> literal quote)
    if (ch === "'") {
      const start = i;
      i++;
      let out = '';
      while (i < n) {
        if (src[i] === "'" && src[i + 1] === "'") {
          out += "'";
          i += 2;
        } else if (src[i] === "'") {
          break;
        } else {
          out += src[i++];
        }
      }
      if (src[i] !== "'") throw new LexError(`Unterminated sheet name at ${start}`);
      i++; // closing quote
      tokens.push({ type: 'QUOTED', value: out, pos: start });
      continue;
    }

    // bare word: identifiers, function names, cell refs
    if (isWordStart(ch)) {
      const start = i;
      i++;
      while (i < n && isWordPart(src[i])) i++;
      tokens.push({ type: 'IDENT', value: src.slice(start, i), pos: start });
      continue;
    }

    // "::" table qualifier -- must be checked before the single ":"
    if (ch === ':' && src[i + 1] === ':') {
      tokens.push({ type: 'DBLCOLON', value: '::', pos: i });
      i += 2;
      continue;
    }

    // two-character comparison operators -- must be checked before the
    // single-character "<"/">" fallback below
    if (ch === '<' && src[i + 1] === '=') {
      tokens.push({ type: 'LTE', value: '<=', pos: i });
      i += 2;
      continue;
    }
    if (ch === '>' && src[i + 1] === '=') {
      tokens.push({ type: 'GTE', value: '>=', pos: i });
      i += 2;
      continue;
    }
    if (ch === '<' && src[i + 1] === '>') {
      tokens.push({ type: 'NEQ', value: '<>', pos: i });
      i += 2;
      continue;
    }

    // an error-code literal, e.g. "#REF" or "#DIV/0!" -- lets a formula
    // literally contain/produce one, same as typing it in Excel; also how a
    // structural row/col delete marks a now-broken reference in formula text
    // (see store/document.ts's row/col delete). Recognized as one token
    // regardless of what follows (a subsequent "!"/"?" is already consumed
    // as part of the matched code, never re-tokenized as BANG etc.).
    if (ch === '#') {
      const code = ERROR_CODES_BY_LENGTH.find((c) => src.startsWith(c, i));
      if (code) {
        tokens.push({ type: 'ERRORLIT', value: code, pos: i });
        i += code.length;
        continue;
      }
    }

    // single-character punctuation / operators
    const t = SINGLE[ch];
    if (t) {
      tokens.push({ type: t, value: ch, pos: i });
      i++;
      continue;
    }

    throw new LexError(`Unexpected character '${ch}' at ${i}`);
  }

  tokens.push({ type: 'EOF', value: '', pos: n });
  return tokens;
}