fix(message-compiler): cannot resolve none-identifier characters at l…

…inked key (#1813) * fix(message-compiler): cannot resolve none-identifier characters at linked key * fix: remove comment
intlify · Apr 20, 2024 · 8934933 · 8934933
1 parent a1d857e
commit 8934933
Show file tree

Hide file tree

Showing 6 changed files with 129 additions and 71 deletions.
diff --git a/packages/message-compiler/src/tokenizer.ts b/packages/message-compiler/src/tokenizer.ts
@@ -321,7 +321,7 @@ export function createTokenizer(
  return fn()
  } else {
  // other characters
- return isIdentifierStart(ch)
+ return isTextStart(scnr, false)
  }
  }
 
@@ -680,7 +680,7 @@ export function createTokenizer(
  }
 
  function readLinkedRefer(scnr: Scanner): string {
- const fn = (detect = false, buf: string): string => {
+ const fn = (buf: string): string => {
  const ch = scnr.currentChar()
  if (
  ch === TokenChars.BraceLeft ||
@@ -697,15 +697,15 @@ export function createTokenizer(
  } else if (ch === NEW_LINE || ch === DOT) {
  buf += ch
  scnr.next()
- return fn(detect, buf)
+ return fn(buf)
  } else {
  buf += ch
  scnr.next()
- return fn(true, buf)
+ return fn(buf)
  }
  }
 
- return fn(false, '')
+ return fn('')
  }
 
  function readPlural(scnr: Scanner): string {

diff --git a/packages/message-compiler/test/__snapshots__/parser.test.ts.snap b/packages/message-compiler/test/__snapshots__/parser.test.ts.snap
@@ -525,28 +525,6 @@ exports[`parse > linked key paren error with modifier: "@.lower:(foo)" 1`] = `
 }
 `;
 
-exports[`parse > linked key paren error with modifier: "@.lower:(foo)" errors 1`] = `
-[
- {
- "code": 13,
- "domain": "parser",
- "location": {
- "end": {
- "column": 14,
- "line": 1,
- "offset": 13,
- },
- "start": {
- "column": 8,
- "line": 1,
- "offset": 7,
- },
- },
- "message": "Unexpected empty linked key",
- },
-]
-`;
-
 exports[`parse > linked key paren error: "@:(foo)" 1`] = `
 {
  "body": {
@@ -640,28 +618,6 @@ exports[`parse > linked key paren error: "@:(foo)" 1`] = `
 }
 `;
 
-exports[`parse > linked key paren error: "@:(foo)" errors 1`] = `
-[
- {
- "code": 13,
- "domain": "parser",
- "location": {
- "end": {
- "column": 8,
- "line": 1,
- "offset": 7,
- },
- "start": {
- "column": 2,
- "line": 1,
- "offset": 1,
- },
- },
- "message": "Unexpected empty linked key",
- },
-]
-`;
-
 exports[`parse > linked key with named and modifier: "hi @._upper:{_name} !" 1`] = `
 {
  "body": {
@@ -2024,16 +1980,6 @@ exports[`parser options > location disable > linked key paren error with modifie
 }
 `;
 
-exports[`parser options > location disable > linked key paren error with modifier: "@.lower:(foo)" errors 1`] = `
-[
- {
- "code": 13,
- "domain": "parser",
- "message": "Unexpected empty linked key",
- },
-]
-`;
-
 exports[`parser options > location disable > linked key paren error: "@:(foo)" 1`] = `
 {
  "body": {
@@ -2056,16 +2002,6 @@ exports[`parser options > location disable > linked key paren error: "@:(foo)" 1
 }
 `;
 
-exports[`parser options > location disable > linked key paren error: "@:(foo)" errors 1`] = `
-[
- {
- "code": 13,
- "domain": "parser",
- "message": "Unexpected empty linked key",
- },
-]
-`;
-
 exports[`parser options > location disable > linked key with named and modifier: "hi @._upper:{_name} !" 1`] = `
 {
  "body": {

diff --git a/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap b/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap
@@ -4370,6 +4370,90 @@ exports[`token analysis > "hi @:name !" tokens 1`] = `
 ]
 `;
 
+exports[`token analysis > "hi @:名前" tokens 1`] = `
+[
+ {
+ "loc": {
+ "end": {
+ "column": 4,
+ "line": 1,
+ "offset": 3,
+ },
+ "start": {
+ "column": 1,
+ "line": 1,
+ "offset": 0,
+ },
+ },
+ "type": 0,
+ "value": "hi ",
+ },
+ {
+ "loc": {
+ "end": {
+ "column": 5,
+ "line": 1,
+ "offset": 4,
+ },
+ "start": {
+ "column": 4,
+ "line": 1,
+ "offset": 3,
+ },
+ },
+ "type": 8,
+ "value": "@",
+ },
+ {
+ "loc": {
+ "end": {
+ "column": 6,
+ "line": 1,
+ "offset": 5,
+ },
+ "start": {
+ "column": 5,
+ "line": 1,
+ "offset": 4,
+ },
+ },
+ "type": 10,
+ "value": ":",
+ },
+ {
+ "loc": {
+ "end": {
+ "column": 8,
+ "line": 1,
+ "offset": 7,
+ },
+ "start": {
+ "column": 6,
+ "line": 1,
+ "offset": 5,
+ },
+ },
+ "type": 11,
+ "value": "名前",
+ },
+ {
+ "loc": {
+ "end": {
+ "column": 8,
+ "line": 1,
+ "offset": 7,
+ },
+ "start": {
+ "column": 8,
+ "line": 1,
+ "offset": 7,
+ },
+ },
+ "type": 14,
+ },
+]
+`;
+
 exports[`token analysis > "hi @\\n. upper\\n: {'name'}\\n !" errors 1`] = `
 [
  {
@@ -10520,6 +10604,30 @@ exports[`tokenize options: location disable > "hi @:name !" tokens 1`] = `
 ]
 `;
 
+exports[`tokenize options: location disable > "hi @:名前" tokens 1`] = `
+[
+ {
+ "type": 0,
+ "value": "hi ",
+ },
+ {
+ "type": 8,
+ "value": "@",
+ },
+ {
+ "type": 10,
+ "value": ":",
+ },
+ {
+ "type": 11,
+ "value": "名前",
+ },
+ {
+ "type": 14,
+ },
+]
+`;
+
 exports[`tokenize options: location disable > "hi @\\n. upper\\n: {'name'}\\n !" errors 1`] = `
 [
  {

diff --git a/packages/message-compiler/test/tokenizer.test.ts b/packages/message-compiler/test/tokenizer.test.ts
@@ -80,6 +80,7 @@ const CASES = [
  `hi @.upper\n{name} !`,
  `hi @.upper {name} !`,
  `hi @:\nname !`,
+ `hi @:名前`,
  `hi @: {'name'} !`,
  `hi @\n. upper\n: {'name'}\n !`,
  ` | | |`,

diff --git a/packages/vue-i18n-core/test/issues.test.ts b/packages/vue-i18n-core/test/issues.test.ts
@@ -1437,3 +1437,16 @@ test('#1796', async () => {
  })
  ).toEqual('My message with hello world.')
 })
+
+test('#1809', async () => {
+ const i18n = createI18n({
+ locale: 'en',
+ messages: {
+ en: {
+ hi: 'hi @:名前',
+ 名前: 'kazupon'
+ }
+ }
+ })
+ expect(i18n.global.t('hi')).toEqual('hi kazupon')
+})
diff --git a/spec/syntax.ebnf b/spec/syntax.ebnf
@@ -1,5 +1,5 @@
 (*
- * Inltify message syntax v0.3
+ * Inltify message syntax v0.4.0
  * (vue-i18n compatible)
  *)
 
@@ -18,7 +18,7 @@ Named ::= Modulo? "{" Space? (NamedIdentifier) Space? "}";
 List ::= "{" Space? (NumberLiteral) Space? "}";
 Linked ::= "@" (LinkedModifier)? LinkedDelimiter LinkedRefer;
 LinkedRefer ::= LinkedKey | Placeholder;
-LinkedKey ::= Identifier;
+LinkedKey ::= Text;
 LinkedModifier ::= LinkedDot Identifier;
 LinkedDelimiter ::= ":";
 LinkedDot ::= ".";