- init

src/engine/lexer.ts

@@ -0,0 +1,165 @@
+/**
+ * Tokenizer for the formula language.
+ *
+ * The lexer is deliberately dumb: it splits the input into a flat token stream
+ * and leaves *all* structural decisions to the parser. In particular it does not
+ * try to tell a cell reference ("A1") from a function name ("SUM") or a sheet
+ * name ("Umsatz") -- they are all `IDENT` tokens and the parser disambiguates
+ * them by look-ahead (`(` -> function, `!` -> sheet qualifier, digits -> cell).
+ */
+
+export type TokenType =
+  | 'NUMBER'
+  | 'STRING'
+  | 'IDENT' // bare word: A1, SUM, Region, Q1 ...
+  | 'QUOTED' // 'Kosten Q3' -- a sheet name in single quotes
+  | 'PLUS'
+  | 'MINUS'
+  | 'STAR'
+  | 'SLASH'
+  | 'CARET'
+  | 'LPAREN'
+  | 'RPAREN'
+  | 'SEMICOLON'
+  | 'COLON'
+  | 'DBLCOLON' // "::"  -- Numbers-style table qualifier ('Tabelle'::A1)
+  | 'BANG'
+  | 'EOF';
+
+export interface Token {
+  type: TokenType;
+  /** Raw text of the token (unquoted for STRING/QUOTED). */
+  value: string;
+  /** Character offset in the source, useful for error messages. */
+  pos: number;
+}
+
+export class LexError extends Error {}
+
+const SINGLE: Record<string, TokenType> = {
+  '+': 'PLUS',
+  '-': 'MINUS',
+  '*': 'STAR',
+  '/': 'SLASH',
+  '^': 'CARET',
+  '(': 'LPAREN',
+  ')': 'RPAREN',
+  ';': 'SEMICOLON',
+  ':': 'COLON',
+  '!': 'BANG',
+};
+
+function isDigit(ch: string): boolean {
+  return ch >= '0' && ch <= '9';
+}
+function isWordStart(ch: string): boolean {
+  return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch === '_';
+}
+function isWordPart(ch: string): boolean {
+  return isWordStart(ch) || isDigit(ch);
+}
+
+export function tokenize(src: string): Token[] {
+  const tokens: Token[] = [];
+  let i = 0;
+  const n = src.length;
+
+  while (i < n) {
+    const ch = src[i];
+
+    // whitespace
+    if (ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r') {
+      i++;
+      continue;
+    }
+
+    // number: 12  12.5  .5  1e3  1.2E-3
+    if (isDigit(ch) || (ch === '.' && isDigit(src[i + 1]))) {
+      const start = i;
+      while (i < n && isDigit(src[i])) i++;
+      if (src[i] === '.') {
+        i++;
+        while (i < n && isDigit(src[i])) i++;
+      }
+      if (src[i] === 'e' || src[i] === 'E') {
+        let j = i + 1;
+        if (src[j] === '+' || src[j] === '-') j++;
+        if (isDigit(src[j])) {
+          i = j;
+          while (i < n && isDigit(src[i])) i++;
+        }
+      }
+      tokens.push({ type: 'NUMBER', value: src.slice(start, i), pos: start });
+      continue;
+    }
+
+    // double-quoted string literal
+    if (ch === '"') {
+      const start = i;
+      i++;
+      let out = '';
+      while (i < n && src[i] !== '"') {
+        if (src[i] === '\\' && i + 1 < n) {
+          out += src[i + 1];
+          i += 2;
+        } else {
+          out += src[i++];
+        }
+      }
+      if (src[i] !== '"') throw new LexError(`Unterminated string at ${start}`);
+      i++; // closing quote
+      tokens.push({ type: 'STRING', value: out, pos: start });
+      continue;
+    }
+
+    // single-quoted sheet name: 'Kosten Q3'  ('' -> literal quote)
+    if (ch === "'") {
+      const start = i;
+      i++;
+      let out = '';
+      while (i < n) {
+        if (src[i] === "'" && src[i + 1] === "'") {
+          out += "'";
+          i += 2;
+        } else if (src[i] === "'") {
+          break;
+        } else {
+          out += src[i++];
+        }
+      }
+      if (src[i] !== "'") throw new LexError(`Unterminated sheet name at ${start}`);
+      i++; // closing quote
+      tokens.push({ type: 'QUOTED', value: out, pos: start });
+      continue;
+    }
+
+    // bare word: identifiers, function names, cell refs
+    if (isWordStart(ch)) {
+      const start = i;
+      i++;
+      while (i < n && isWordPart(src[i])) i++;
+      tokens.push({ type: 'IDENT', value: src.slice(start, i), pos: start });
+      continue;
+    }
+
+    // "::" table qualifier -- must be checked before the single ":"
+    if (ch === ':' && src[i + 1] === ':') {
+      tokens.push({ type: 'DBLCOLON', value: '::', pos: i });
+      i += 2;
+      continue;
+    }
+
+    // single-character punctuation / operators
+    const t = SINGLE[ch];
+    if (t) {
+      tokens.push({ type: t, value: ch, pos: i });
+      i++;
+      continue;
+    }
+
+    throw new LexError(`Unexpected character '${ch}' at ${i}`);
+  }
+
+  tokens.push({ type: 'EOF', value: '', pos: n });
+  return tokens;
+}