numolus/src/engine/lexer.ts

/**
 * Tokenizer for the formula language.
 *
 * The lexer is deliberately dumb: it splits the input into a flat token stream
 * and leaves *all* structural decisions to the parser. In particular it does not
 * try to tell a cell reference ("A1") from a function name ("SUM") or a sheet
 * name ("Umsatz") -- they are all `IDENT` tokens and the parser disambiguates
 * them by look-ahead (`(` -> function, `!` -> sheet qualifier, digits -> cell).
 */

export type TokenType =
  | 'NUMBER'
  | 'STRING'
  | 'IDENT' // bare word: A1, SUM, Region, Q1 ...
  | 'QUOTED' // 'Kosten Q3' -- a sheet name in single quotes
  | 'PLUS'
  | 'MINUS'
  | 'STAR'
  | 'SLASH'
  | 'CARET'
  | 'LPAREN'
  | 'RPAREN'
  | 'SEMICOLON'
  | 'COLON'
  | 'DBLCOLON' // "::"  -- Numbers-style table qualifier ('Tabelle'::A1)
  | 'BANG'
  | 'EQUALS' // "="  -- assignment (block body) or equality comparison
  | 'NEQ' // "<>"
  | 'LT' // "<"
  | 'GT' // ">"
  | 'LTE' // "<="
  | 'GTE' // ">="
  | 'EOF';

export interface Token {
  type: TokenType;
  /** Raw text of the token (unquoted for STRING/QUOTED). */
  value: string;
  /** Character offset in the source, useful for error messages. */
  pos: number;
}

export class LexError extends Error {}

const SINGLE: Record<string, TokenType> = {
  '+': 'PLUS',
  '-': 'MINUS',
  '*': 'STAR',
  '/': 'SLASH',
  '^': 'CARET',
  '(': 'LPAREN',
  ')': 'RPAREN',
  ';': 'SEMICOLON',
  ':': 'COLON',
  '!': 'BANG',
  '=': 'EQUALS',
  '<': 'LT',
  '>': 'GT',
};

function isDigit(ch: string): boolean {
  return ch >= '0' && ch <= '9';
}
function isWordStart(ch: string): boolean {
  // '$' only ever appears inside a cell reference ("$A1", "A$1", "$A$1"), but
  // the lexer doesn't distinguish those from any other bare word (see the
  // module doc) -- it's just part of the word here, and the parser is the one
  // that validates/interprets it as a locked reference axis.
  return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch === '_' || ch === '$';
}
function isWordPart(ch: string): boolean {
  return isWordStart(ch) || isDigit(ch);
}

export function tokenize(src: string): Token[] {
  const tokens: Token[] = [];
  let i = 0;
  const n = src.length;

  while (i < n) {
    const ch = src[i];

    // whitespace
    if (ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r') {
      i++;
      continue;
    }

    // number: 12  12.5  .5  1e3  1.2E-3
    if (isDigit(ch) || (ch === '.' && isDigit(src[i + 1]))) {
      const start = i;
      while (i < n && isDigit(src[i])) i++;
      if (src[i] === '.') {
        i++;
        while (i < n && isDigit(src[i])) i++;
      }
      if (src[i] === 'e' || src[i] === 'E') {
        let j = i + 1;
        if (src[j] === '+' || src[j] === '-') j++;
        if (isDigit(src[j])) {
          i = j;
          while (i < n && isDigit(src[i])) i++;
        }
      }
      tokens.push({ type: 'NUMBER', value: src.slice(start, i), pos: start });
      continue;
    }

    // double-quoted string literal
    if (ch === '"') {
      const start = i;
      i++;
      let out = '';
      while (i < n && src[i] !== '"') {
        if (src[i] === '\\' && i + 1 < n) {
          out += src[i + 1];
          i += 2;
        } else {
          out += src[i++];
        }
      }
      if (src[i] !== '"') throw new LexError(`Unterminated string at ${start}`);
      i++; // closing quote
      tokens.push({ type: 'STRING', value: out, pos: start });
      continue;
    }

    // single-quoted sheet name: 'Kosten Q3'  ('' -> literal quote)
    if (ch === "'") {
      const start = i;
      i++;
      let out = '';
      while (i < n) {
        if (src[i] === "'" && src[i + 1] === "'") {
          out += "'";
          i += 2;
        } else if (src[i] === "'") {
          break;
        } else {
          out += src[i++];
        }
      }
      if (src[i] !== "'") throw new LexError(`Unterminated sheet name at ${start}`);
      i++; // closing quote
      tokens.push({ type: 'QUOTED', value: out, pos: start });
      continue;
    }

    // bare word: identifiers, function names, cell refs
    if (isWordStart(ch)) {
      const start = i;
      i++;
      while (i < n && isWordPart(src[i])) i++;
      tokens.push({ type: 'IDENT', value: src.slice(start, i), pos: start });
      continue;
    }

    // "::" table qualifier -- must be checked before the single ":"
    if (ch === ':' && src[i + 1] === ':') {
      tokens.push({ type: 'DBLCOLON', value: '::', pos: i });
      i += 2;
      continue;
    }

    // two-character comparison operators -- must be checked before the
    // single-character "<"/">" fallback below
    if (ch === '<' && src[i + 1] === '=') {
      tokens.push({ type: 'LTE', value: '<=', pos: i });
      i += 2;
      continue;
    }
    if (ch === '>' && src[i + 1] === '=') {
      tokens.push({ type: 'GTE', value: '>=', pos: i });
      i += 2;
      continue;
    }
    if (ch === '<' && src[i + 1] === '>') {
      tokens.push({ type: 'NEQ', value: '<>', pos: i });
      i += 2;
      continue;
    }

    // single-character punctuation / operators
    const t = SINGLE[ch];
    if (t) {
      tokens.push({ type: t, value: ch, pos: i });
      i++;
      continue;
    }

    throw new LexError(`Unexpected character '${ch}' at ${i}`);
  }

  tokens.push({ type: 'EOF', value: '', pos: n });
  return tokens;
}