From 87aaef56cbfe70d310f76f12aa1c3ff38940a8aa Mon Sep 17 00:00:00 2001
From: Affaan Mustafa <affaan@itomarkets.com>
Date: Sat, 17 Oct 2026 21:29:00 -0400
Subject: [PATCH] refactor(lib): split the gateguard code lexer by token class

lexCodeLines ran every state (code, line comment, block comment, string) through one loop body with three closures over its locals, and shCodeLines and psCodeLines each inlined their escape, quote and comment readers. The C-family lexer now keeps its partial line on a small lx record and dispatches by state to lexNewline, lexLineComment, lexBlockComment, lexString and lexCode, with lexCommentStart, startsJsx and lexStringStart under lexCode. The shell lexers share pushShellLine, readEscapedPair, readQuotedSpan and readLineComment; shCodeLines looks its readers up in a table and psCodeLines splits into psBlockComment and psToken.

Every reader returns the index of the last character it consumed, so the loops advance exactly as before, and the sentinel for an unlexable edit is still null from the C-family path and a null result from the shell paths. Exports are unchanged. A differential run against the previous version over the unit test strings plus 40k generated inputs, across all nine spec shapes change-profile builds, produced identical lines, normalized signatures and isTrivialEdit verdicts.
---
 scripts/lib/gateguard-code-lexer.js | 493 ++++++++++++++++------------
 1 file changed, 274 insertions(+), 219 deletions(-)

diff --git a/scripts/lib/gateguard-code-lexer.js b/scripts/lib/gateguard-code-lexer.js
index f89c9e101..c45d7e53b 100644
--- a/scripts/lib/gateguard-code-lexer.js
+++ b/scripts/lib/gateguard-code-lexer.js
@@ -110,141 +110,170 @@ function stringSpecials(quote) {
   return pattern;
 }
 
+// --- lexCodeLines state ---
+// lx carries the text, spec and partial line. Each step reads from index i and
+// returns the index of the last consumed character, or null when the text is
+// not lexable and the edit cannot be trivial.
+
+function appendCode(lx, piece) {
+  lx.code += piece;
+  lx.tail = (lx.tail + piece).slice(-2);
+}
+
+function endComment(lx, end) {
+  const marker = directiveMarker(lx.text.slice(lx.commentStart, end), lx.spec);
+  if (marker) appendCode(lx, marker);
+}
+
+function endLine(lx) {
+  if (endsWithContinuation(lx.code)) return false;
+  lx.lines.push(lx.code);
+  lx.code = '';
+  lx.tail = '';
+  return true;
+}
+
+function lexNewline(lx, i) {
+  const { text } = lx;
+  if (lx.state === 'string') return null;
+  if (lx.state === 'line') {
+    if (text.slice(lx.commentStart, i).trimEnd().endsWith('\\')) return null;
+    endComment(lx, i);
+    lx.state = 'code';
+  }
+  if (lx.state === 'block') {
+    lx.lines.push(lx.code);
+    lx.code = '';
+    lx.tail = '';
+  } else if (!endLine(lx)) {
+    return null;
+  }
+  return text[i] === '\r' && text[i + 1] === '\n' ? i + 1 : i;
+}
+
+function lexLineComment(lx, i) {
+  return nextSpecial(LINE_BREAK, lx.text, i) - 1;
+}
+
+function lexBlockComment(lx, i) {
+  const { text } = lx;
+  const ch = text[i];
+  if (ch !== '*' && ch !== '/') return nextSpecial(BLOCK_SPECIALS, text, i + 1) - 1;
+  if (ch === '*' && text[i + 1] === '/') {
+    endComment(lx, i);
+    lx.state = 'code';
+    appendCode(lx, ' ');
+    return i + 1;
+  }
+  if (ch === '/' && text[i + 1] === '*') return null;
+  return i;
+}
+
+function lexString(lx, i) {
+  const { text, quote } = lx;
+  const ch = text[i];
+  if (ch !== '\\' && ch !== quote) {
+    const end = nextSpecial(stringSpecials(quote), text, i + 1);
+    const run = text.slice(i, end);
+    appendCode(lx, run);
+    lx.stringBody += run;
+    return end - 1;
+  }
+  appendCode(lx, ch);
+  if (ch === '\\') {
+    const next = text[i + 1];
+    if (next === undefined || next === '\n' || next === '\r') return null;
+    appendCode(lx, next);
+    return i + 1;
+  }
+  if (lx.spec.noDollarInStrings && lx.stringBody.includes('$')) return null;
+  lx.state = 'code';
+  return i;
+}
+
+function lexCommentStart(lx, i) {
+  const { text, spec } = lx;
+  const ch = text[i];
+  if (spec.lineComment === '//' && ch === '/' && text[i + 1] === '/') {
+    lx.state = 'line';
+    lx.commentStart = i + 2;
+    return i + 1;
+  }
+  if (spec.lineComment === '#' && ch === '#') {
+    lx.state = 'line';
+    lx.commentStart = i + 1;
+    return i;
+  }
+  if (spec.blockComment && ch === '/' && text[i + 1] === '*') {
+    lx.state = 'block';
+    lx.commentStart = i + 2;
+    return i + 1;
+  }
+  return undefined;
+}
+
+function startsJsx(spec, text, i) {
+  const ch = text[i];
+  const next = text[i + 1];
+  if (spec.jsx && ch === '<') return isLetter(next) || next === '/' || next === '>' || next === '!';
+  return spec.jsx !== undefined && ch === '-' && next === '-' && text[i + 2] === '>';
+}
+
+function lexStringStart(lx, i) {
+  const { text, spec } = lx;
+  const ch = text[i];
+  if (spec.tripleQuotes && text[i + 1] === ch && text[i + 2] === ch) return null;
+  if (stringPrefixIsUnsafe(lx.tail, spec)) return null;
+  lx.state = 'string';
+  lx.quote = ch;
+  lx.stringBody = '';
+  appendCode(lx, ch);
+  return i;
+}
+
+function lexCode(lx, i) {
+  const { text, spec } = lx;
+  const ch = text[i];
+  if (!lx.specials.chars.has(ch)) {
+    const end = nextSpecial(lx.specials.pattern, text, i + 1);
+    appendCode(lx, text.slice(i, end));
+    return end - 1;
+  }
+  const comment = lexCommentStart(lx, i);
+  if (comment !== undefined) return comment;
+  if (spec.forbiddenCode && spec.forbiddenCode.includes(ch)) return null;
+  if (startsJsx(spec, text, i)) return null;
+  if (ch === "'" && spec.rustChars) {
+    const end = rustCharEnd(text, i);
+    appendCode(lx, end === -1 ? ch : text.slice(i, end + 1));
+    return end === -1 ? i : end;
+  }
+  if (spec.quotes.includes(ch)) return lexStringStart(lx, i);
+  appendCode(lx, ch);
+  return i;
+}
+
+const LEX_STEPS = { code: lexCode, line: lexLineComment, block: lexBlockComment, string: lexString };
+
+function finishLex(lx) {
+  if (lx.state === 'string' || lx.state === 'block') return null;
+  if (lx.state === 'line') {
+    if (lx.text.slice(lx.commentStart).trimEnd().endsWith('\\')) return null;
+    endComment(lx, lx.text.length);
+  }
+  return endLine(lx) ? lx.lines : null;
+}
+
 function lexCodeLines(text, spec) {
-  const specials = codeSpecials(spec);
-  const lines = [];
-  let code = '';
-  let state = 'code';
-  let quote = '';
-  let stringBody = '';
-  let commentStart = 0;
-  let tail = '';
-  const append = piece => {
-    code += piece;
-    tail = (tail + piece).slice(-2);
-  };
-  const endComment = end => {
-    const marker = directiveMarker(text.slice(commentStart, end), spec);
-    if (marker) append(marker);
-  };
-  const endLine = () => {
-    if (endsWithContinuation(code)) return false;
-    lines.push(code);
-    code = '';
-    tail = '';
-    return true;
-  };
-  if (spec.jsx !== undefined && text.startsWith('#!')) state = 'line';
+  const lx = { text, spec, specials: codeSpecials(spec), lines: [], code: '', state: 'code', quote: '', stringBody: '', commentStart: 0, tail: '' };
+  if (spec.jsx !== undefined && text.startsWith('#!')) lx.state = 'line';
   for (let i = 0; i < text.length; i++) {
     const ch = text[i];
-    if (ch === '\n' || ch === '\r') {
-      if (state === 'string') return null;
-      if (state === 'line') {
-        if (text.slice(commentStart, i).trimEnd().endsWith('\\')) return null;
-        endComment(i);
-        state = 'code';
-      }
-      if (state === 'block') {
-        lines.push(code);
-        code = '';
-        tail = '';
-      } else if (!endLine()) {
-        return null;
-      }
-      if (ch === '\r' && text[i + 1] === '\n') i++;
-      continue;
-    }
-    if (state === 'line') {
-      i = nextSpecial(LINE_BREAK, text, i) - 1;
-      continue;
-    }
-    if (state === 'block') {
-      if (ch !== '*' && ch !== '/') {
-        i = nextSpecial(BLOCK_SPECIALS, text, i + 1) - 1;
-        continue;
-      }
-      if (ch === '*' && text[i + 1] === '/') {
-        endComment(i);
-        state = 'code';
-        append(' ');
-        i++;
-      } else if (ch === '/' && text[i + 1] === '*') {
-        return null;
-      }
-      continue;
-    }
-    if (state === 'string') {
-      if (ch !== '\\' && ch !== quote) {
-        const end = nextSpecial(stringSpecials(quote), text, i + 1);
-        const run = text.slice(i, end);
-        append(run);
-        stringBody += run;
-        i = end - 1;
-        continue;
-      }
-      append(ch);
-      if (ch === '\\') {
-        const next = text[i + 1];
-        if (next === undefined || next === '\n' || next === '\r') return null;
-        append(next);
-        i++;
-      } else if (ch === quote) {
-        if (spec.noDollarInStrings && stringBody.includes('$')) return null;
-        state = 'code';
-      } else {
-        stringBody += ch;
-      }
-      continue;
-    }
-    if (!specials.chars.has(ch)) {
-      const end = nextSpecial(specials.pattern, text, i + 1);
-      append(text.slice(i, end));
-      i = end - 1;
-      continue;
-    }
-    if (spec.lineComment === '//' && ch === '/' && text[i + 1] === '/') {
-      state = 'line';
-      commentStart = i + 2;
-      i++;
-      continue;
-    }
-    if (spec.lineComment === '#' && ch === '#') {
-      state = 'line';
-      commentStart = i + 1;
-      continue;
-    }
-    if (spec.blockComment && ch === '/' && text[i + 1] === '*') {
-      state = 'block';
-      commentStart = i + 2;
-      i++;
-      continue;
-    }
-    if (spec.forbiddenCode && spec.forbiddenCode.includes(ch)) return null;
-    if (spec.jsx && ch === '<' && (isLetter(text[i + 1]) || text[i + 1] === '/' || text[i + 1] === '>' || text[i + 1] === '!')) return null;
-    if (spec.jsx !== undefined && ch === '-' && text[i + 1] === '-' && text[i + 2] === '>') return null;
-    if (ch === "'" && spec.rustChars) {
-      const end = rustCharEnd(text, i);
-      append(end === -1 ? ch : text.slice(i, end + 1));
-      if (end !== -1) i = end;
-      continue;
-    }
-    if (spec.quotes.includes(ch)) {
-      if (spec.tripleQuotes && text[i + 1] === ch && text[i + 2] === ch) return null;
-      if (stringPrefixIsUnsafe(tail, spec)) return null;
-      state = 'string';
-      quote = ch;
-      stringBody = '';
-      append(ch);
-      continue;
-    }
-    append(ch);
+    const next = ch === '\n' || ch === '\r' ? lexNewline(lx, i) : LEX_STEPS[lx.state](lx, i);
+    if (next === null) return null;
+    i = next;
   }
-  if (state === 'string' || state === 'block') return null;
-  if (state === 'line') {
-    if (text.slice(commentStart).trimEnd().endsWith('\\')) return null;
-    endComment(text.length);
-  }
-  return endLine() ? lines : null;
+  return finishLex(lx);
 }
 
 function normalizeCodeLine(line, spec) {
@@ -364,56 +393,86 @@ function isBlank(ch) {
   return ch === ' ' || ch === '\t';
 }
 
+// --- Shell and PowerShell readers ---
+// Each reader appends what it consumed to lx.code and returns the index of the
+// last consumed character, or -1 when the text is not lexable.
+
+function pushShellLine(lx, i) {
+  lx.lines.push(lx.code);
+  lx.code = '';
+  return lx.text[i] === '\r' && lx.text[i + 1] === '\n' ? i + 1 : i;
+}
+
+function readEscapedPair(lx, i) {
+  const next = lx.text[i + 1];
+  if (next === undefined || next === '\n' || next === '\r') return -1;
+  lx.code += lx.text[i] + next;
+  return i + 1;
+}
+
+function readQuotedSpan(lx, i, end) {
+  if (end === -1) return -1;
+  lx.code += lx.text.slice(i, end + 1);
+  return end;
+}
+
+function lineCommentEnd(text, i) {
+  while (i + 1 < text.length && text[i + 1] !== '\n' && text[i + 1] !== '\r') i++;
+  return i;
+}
+
+function readLineComment(lx, i) {
+  const end = lineCommentEnd(lx.text, i);
+  lx.code += directiveMarker(lx.text.slice(i + 1, end + 1), SHELL_SPEC);
+  return end;
+}
+
+function shSingleQuoted(lx, i) {
+  const { text } = lx;
+  if (text[i - 1] === '$') return -1;
+  const end = text.indexOf("'", i + 1);
+  if (end === -1) return -1;
+  const body = text.slice(i, end + 1);
+  if (body.includes('\n') || body.includes('\r')) return -1;
+  lx.code += body;
+  return end;
+}
+
+function shDoubleQuoted(lx, i) {
+  return readQuotedSpan(lx, i, shDoubleQuoteEnd(lx.text, i));
+}
+
+function shHash(lx, i) {
+  const starts = hashStartsComment(lx.text, i);
+  if (starts === null) return -1;
+  if (!starts) {
+    lx.code += '#';
+    return i;
+  }
+  return readLineComment(lx, i);
+}
+
+const SH_READERS = new Map([['\\', readEscapedPair], ["'", shSingleQuoted], ['"', shDoubleQuoted], ['#', shHash]]);
+
 function shCodeLines(text) {
   if (text.includes('<<') || text.includes('`') || hasLoneCarriageReturn(text)) return null;
-  const lines = [];
-  let code = '';
+  const lx = { text, code: '', lines: [] };
   for (let i = 0; i < text.length; i++) {
     const ch = text[i];
     if (ch === '\n' || ch === '\r') {
-      lines.push(code);
-      code = '';
-      if (ch === '\r' && text[i + 1] === '\n') i++;
+      i = pushShellLine(lx, i);
       continue;
     }
-    if (ch === '\\') {
-      const next = text[i + 1];
-      if (next === undefined || next === '\n' || next === '\r') return null;
-      code += ch + next;
-      i++;
+    const reader = SH_READERS.get(ch);
+    if (!reader) {
+      lx.code += ch;
       continue;
     }
-    if (ch === "'") {
-      if (text[i - 1] === '$') return null;
-      const end = text.indexOf("'", i + 1);
-      if (end === -1) return null;
-      const body = text.slice(i, end + 1);
-      if (body.includes('\n') || body.includes('\r')) return null;
-      code += body;
-      i = end;
-      continue;
-    }
-    if (ch === '"') {
-      const end = shDoubleQuoteEnd(text, i);
-      if (end === -1) return null;
-      code += text.slice(i, end + 1);
-      i = end;
-      continue;
-    }
-    if (ch === '#') {
-      const starts = hashStartsComment(text, i);
-      if (starts === null) return null;
-      if (starts) {
-        const start = i + 1;
-        while (i + 1 < text.length && text[i + 1] !== '\n' && text[i + 1] !== '\r') i++;
-        code += directiveMarker(text.slice(start, i + 1), SHELL_SPEC);
-        continue;
-      }
-    }
-    code += ch;
+    i = reader(lx, i);
+    if (i === -1) return null;
   }
-  lines.push(code);
-  return lines;
+  lx.lines.push(lx.code);
+  return lx.lines;
 }
 
 const PS_SMART_QUOTES = /[\u2018\u2019\u201a\u201b\u201c\u201d\u201e]/;
@@ -442,62 +501,58 @@ function psQuoteEnd(text, i) {
   return -1;
 }
 
+function psQuoted(lx, i) {
+  return readQuotedSpan(lx, i, psQuoteEnd(lx.text, i));
+}
+
+function psBlockStart(lx, i) {
+  if (!psHashStartsComment(lx.text, i)) return -1;
+  lx.block = true;
+  lx.blockStart = i + 2;
+  return i + 1;
+}
+
+function psHash(lx, i) {
+  if (!psHashStartsComment(lx.text, i)) return -1;
+  return readLineComment(lx, i);
+}
+
+function psBlockComment(lx, i) {
+  const { text } = lx;
+  if (text[i] === '<' && text[i + 1] === '#') return -1;
+  if (text[i] === '#' && text[i + 1] === '>') {
+    lx.block = false;
+    lx.code += directiveMarker(text.slice(lx.blockStart, i), SHELL_SPEC) || ' ';
+    return i + 1;
+  }
+  return i;
+}
+
+function psToken(lx, i) {
+  const ch = lx.text[i];
+  if (ch === '`') return readEscapedPair(lx, i);
+  if (ch === "'" || ch === '"') return psQuoted(lx, i);
+  if (ch === '<' && lx.text[i + 1] === '#') return psBlockStart(lx, i);
+  if (ch === '#') return psHash(lx, i);
+  lx.code += ch;
+  return i;
+}
+
 function psCodeLines(text) {
   if (text.includes('@"') || text.includes("@'") || PS_SMART_QUOTES.test(text)) return null;
-  const lines = [];
-  let code = '';
-  let block = false;
-  let blockStart = 0;
+  const lx = { text, code: '', lines: [], block: false, blockStart: 0 };
   for (let i = 0; i < text.length; i++) {
     const ch = text[i];
     if (ch === '\n' || ch === '\r') {
-      lines.push(code);
-      code = '';
-      if (ch === '\r' && text[i + 1] === '\n') i++;
+      i = pushShellLine(lx, i);
       continue;
     }
-    if (block) {
-      if (ch === '<' && text[i + 1] === '#') return null;
-      if (ch === '#' && text[i + 1] === '>') {
-        block = false;
-        code += directiveMarker(text.slice(blockStart, i), SHELL_SPEC) || ' ';
-        i++;
-      }
-      continue;
-    }
-    if (ch === '`') {
-      const next = text[i + 1];
-      if (next === undefined || next === '\n' || next === '\r') return null;
-      code += ch + next;
-      i++;
-      continue;
-    }
-    if (ch === "'" || ch === '"') {
-      const end = psQuoteEnd(text, i);
-      if (end === -1) return null;
-      code += text.slice(i, end + 1);
-      i = end;
-      continue;
-    }
-    if (ch === '<' && text[i + 1] === '#') {
-      if (!psHashStartsComment(text, i)) return null;
-      block = true;
-      blockStart = i + 2;
-      i++;
-      continue;
-    }
-    if (ch === '#') {
-      if (!psHashStartsComment(text, i)) return null;
-      const start = i + 1;
-      while (i + 1 < text.length && text[i + 1] !== '\n' && text[i + 1] !== '\r') i++;
-      code += directiveMarker(text.slice(start, i + 1), SHELL_SPEC);
-      continue;
-    }
-    code += ch;
+    i = lx.block ? psBlockComment(lx, i) : psToken(lx, i);
+    if (i === -1) return null;
   }
-  if (block) return null;
-  lines.push(code);
-  return lines;
+  if (lx.block) return null;
+  lx.lines.push(lx.code);
+  return lx.lines;
 }
 
 function batchCodeLines(text) {