refactor: tokenizer take chars (#1799)

intlify · Apr 13, 2024 · 8afb579 · 8afb579
1 parent dfd3663
commit 8afb579
Showing 1 changed file with 57 additions and 42 deletions.
diff --git a/packages/message-compiler/src/tokenizer.ts b/packages/message-compiler/src/tokenizer.ts
@@ -406,48 +406,57 @@ export function createTokenizer(
  return null
  }
 
+ function isIdentifier(ch: string): boolean {
+ const cc = ch.charCodeAt(0)
+ return (
+ (cc >= 97 && cc <= 122) || // a-z
+ (cc >= 65 && cc <= 90) || // A-Z
+ (cc >= 48 && cc <= 57) || // 0-9
+ cc === 95 || // _
+ cc === 36 // $
+ )
+ }
+
  function takeIdentifierChar(scnr: Scanner): string | undefined | null {
- const closure = (ch: string) => {
- const cc = ch.charCodeAt(0)
- return (
- (cc >= 97 && cc <= 122) || // a-z
- (cc >= 65 && cc <= 90) || // A-Z
- (cc >= 48 && cc <= 57) || // 0-9
- cc === 95 || // _
- cc === 36 // $
- )
- }
- return takeChar(scnr, closure)
+ return takeChar(scnr, isIdentifier)
+ }
+
+ function isNamedIdentifier(ch: string): boolean {
+ const cc = ch.charCodeAt(0)
+ return (
+ (cc >= 97 && cc <= 122) || // a-z
+ (cc >= 65 && cc <= 90) || // A-Z
+ (cc >= 48 && cc <= 57) || // 0-9
+ cc === 95 || // _
+ cc === 36 || // $
+ cc === 45 // -
+ )
  }
 
- // NOTE: It is assumed that this function is used in conjunction with `takeIdentifierChar` for named.
- // TODO: we need to refactor this function ...
  function takeNamedIdentifierChar(scnr: Scanner): string | undefined | null {
- const closure = (ch: string) => {
- const cc = ch.charCodeAt(0)
- return cc === 45 // -
- }
- return takeChar(scnr, closure)
+ return takeChar(scnr, isNamedIdentifier)
+ }
+
+ function isDigit(ch: string): boolean {
+ const cc = ch.charCodeAt(0)
+ return cc >= 48 && cc <= 57 // 0-9
  }
 
  function takeDigit(scnr: Scanner): string | undefined | null {
- const closure = (ch: string) => {
- const cc = ch.charCodeAt(0)
- return cc >= 48 && cc <= 57 // 0-9
- }
- return takeChar(scnr, closure)
+ return takeChar(scnr, isDigit)
+ }
+
+ function isHexDigit(ch: string): boolean {
+ const cc = ch.charCodeAt(0)
+ return (
+ (cc >= 48 && cc <= 57) || // 0-9
+ (cc >= 65 && cc <= 70) || // A-F
+ (cc >= 97 && cc <= 102)
+ ) // a-f
  }
 
  function takeHexDigit(scnr: Scanner): string | undefined | null {
- const closure = (ch: string) => {
- const cc = ch.charCodeAt(0)
- return (
- (cc >= 48 && cc <= 57) || // 0-9
- (cc >= 65 && cc <= 70) || // A-F
- (cc >= 97 && cc <= 102)
- ) // a-f
- }
- return takeChar(scnr, closure)
+ return takeChar(scnr, isHexDigit)
  }
 
  function getDigits(scnr: Scanner): string {
@@ -513,8 +522,7 @@ export function createTokenizer(
 
  let ch: string | undefined | null = ''
  let name = ''
- // eslint-disable-next-line no-cond-assign
- while ((ch = takeIdentifierChar(scnr) || takeNamedIdentifierChar(scnr))) {
+ while ((ch = takeNamedIdentifierChar(scnr))) {
  name += ch
  }
 
@@ -551,6 +559,10 @@ export function createTokenizer(
  return value
  }
 
+ function isLiteral(ch: string): boolean {
+ return ch !== LITERAL_DELIMITER && ch !== NEW_LINE
+ }
+
  function readLiteral(scnr: Scanner): string {
  skipSpaces(scnr)
 
@@ -559,8 +571,7 @@ export function createTokenizer(
 
  let ch: string | undefined | null = ''
  let literal = ''
- const fn = (x: string) => x !== LITERAL_DELIMITER && x !== NEW_LINE
- while ((ch = takeChar(scnr, fn))) {
+ while ((ch = takeChar(scnr, isLiteral))) {
  if (ch === '\\') {
  literal += readEscapeSequence(scnr)
  } else {
@@ -637,17 +648,21 @@ export function createTokenizer(
  return `\\${unicode}${sequence}`
  }
 
+ function isInvalidIdentifier(ch: string): boolean {
+ return (
+ ch !== TokenChars.BraceLeft &&
+ ch !== TokenChars.BraceRight &&
+ ch !== SPACE &&
+ ch !== NEW_LINE
+ )
+ }
+
  function readInvalidIdentifier(scnr: Scanner): string {
  skipSpaces(scnr)
 
  let ch: string | undefined | null = ''
  let identifiers = ''
- const closure = (ch: string) =>
- ch !== TokenChars.BraceLeft &&
- ch !== TokenChars.BraceRight &&
- ch !== SPACE &&
- ch !== NEW_LINE
- while ((ch = takeChar(scnr, closure))) {
+ while ((ch = takeChar(scnr, isInvalidIdentifier))) {
  identifiers += ch
  }