From 64240e3ee842f0ec244e988964ea93e4f93fb230 Mon Sep 17 00:00:00 2001 From: kazuya kawaguchi Date: Sat, 20 Apr 2024 22:39:43 +0900 Subject: [PATCH 1/2] fix(message-compiler): cannot resolve none-identifier characters at linked key --- packages/message-compiler/src/tokenizer.ts | 11 +- .../test/__snapshots__/parser.test.ts.snap | 64 ----------- .../test/__snapshots__/tokenizer.test.ts.snap | 108 ++++++++++++++++++ .../message-compiler/test/tokenizer.test.ts | 1 + packages/vue-i18n-core/test/issues.test.ts | 13 +++ spec/syntax.ebnf | 4 +- 6 files changed, 130 insertions(+), 71 deletions(-) diff --git a/packages/message-compiler/src/tokenizer.ts b/packages/message-compiler/src/tokenizer.ts index b6b6a793f..5d31a8d33 100644 --- a/packages/message-compiler/src/tokenizer.ts +++ b/packages/message-compiler/src/tokenizer.ts @@ -321,7 +321,8 @@ export function createTokenizer( return fn() } else { // other characters - return isIdentifierStart(ch) + // return isIdentifierStart(ch) + return isTextStart(scnr, false) } } @@ -680,7 +681,7 @@ export function createTokenizer( } function readLinkedRefer(scnr: Scanner): string { - const fn = (detect = false, buf: string): string => { + const fn = (buf: string): string => { const ch = scnr.currentChar() if ( ch === TokenChars.BraceLeft || @@ -697,15 +698,15 @@ export function createTokenizer( } else if (ch === NEW_LINE || ch === DOT) { buf += ch scnr.next() - return fn(detect, buf) + return fn(buf) } else { buf += ch scnr.next() - return fn(true, buf) + return fn(buf) } } - return fn(false, '') + return fn('') } function readPlural(scnr: Scanner): string { diff --git a/packages/message-compiler/test/__snapshots__/parser.test.ts.snap b/packages/message-compiler/test/__snapshots__/parser.test.ts.snap index f7f09c888..f134f37b1 100644 --- a/packages/message-compiler/test/__snapshots__/parser.test.ts.snap +++ b/packages/message-compiler/test/__snapshots__/parser.test.ts.snap @@ -525,28 +525,6 @@ exports[`parse > linked key paren error with modifier: "@.lower:(foo)" 1`] = ` } `; -exports[`parse > linked key paren error with modifier: "@.lower:(foo)" errors 1`] = ` -[ - { - "code": 13, - "domain": "parser", - "location": { - "end": { - "column": 14, - "line": 1, - "offset": 13, - }, - "start": { - "column": 8, - "line": 1, - "offset": 7, - }, - }, - "message": "Unexpected empty linked key", - }, -] -`; - exports[`parse > linked key paren error: "@:(foo)" 1`] = ` { "body": { @@ -640,28 +618,6 @@ exports[`parse > linked key paren error: "@:(foo)" 1`] = ` } `; -exports[`parse > linked key paren error: "@:(foo)" errors 1`] = ` -[ - { - "code": 13, - "domain": "parser", - "location": { - "end": { - "column": 8, - "line": 1, - "offset": 7, - }, - "start": { - "column": 2, - "line": 1, - "offset": 1, - }, - }, - "message": "Unexpected empty linked key", - }, -] -`; - exports[`parse > linked key with named and modifier: "hi @._upper:{_name} !" 1`] = ` { "body": { @@ -2024,16 +1980,6 @@ exports[`parser options > location disable > linked key paren error with modifie } `; -exports[`parser options > location disable > linked key paren error with modifier: "@.lower:(foo)" errors 1`] = ` -[ - { - "code": 13, - "domain": "parser", - "message": "Unexpected empty linked key", - }, -] -`; - exports[`parser options > location disable > linked key paren error: "@:(foo)" 1`] = ` { "body": { @@ -2056,16 +2002,6 @@ exports[`parser options > location disable > linked key paren error: "@:(foo)" 1 } `; -exports[`parser options > location disable > linked key paren error: "@:(foo)" errors 1`] = ` -[ - { - "code": 13, - "domain": "parser", - "message": "Unexpected empty linked key", - }, -] -`; - exports[`parser options > location disable > linked key with named and modifier: "hi @._upper:{_name} !" 1`] = ` { "body": { diff --git a/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap b/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap index e0f31387a..94eb379b1 100644 --- a/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap +++ b/packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap @@ -4370,6 +4370,90 @@ exports[`token analysis > "hi @:name !" tokens 1`] = ` ] `; +exports[`token analysis > "hi @:名前" tokens 1`] = ` +[ + { + "loc": { + "end": { + "column": 4, + "line": 1, + "offset": 3, + }, + "start": { + "column": 1, + "line": 1, + "offset": 0, + }, + }, + "type": 0, + "value": "hi ", + }, + { + "loc": { + "end": { + "column": 5, + "line": 1, + "offset": 4, + }, + "start": { + "column": 4, + "line": 1, + "offset": 3, + }, + }, + "type": 8, + "value": "@", + }, + { + "loc": { + "end": { + "column": 6, + "line": 1, + "offset": 5, + }, + "start": { + "column": 5, + "line": 1, + "offset": 4, + }, + }, + "type": 10, + "value": ":", + }, + { + "loc": { + "end": { + "column": 8, + "line": 1, + "offset": 7, + }, + "start": { + "column": 6, + "line": 1, + "offset": 5, + }, + }, + "type": 11, + "value": "名前", + }, + { + "loc": { + "end": { + "column": 8, + "line": 1, + "offset": 7, + }, + "start": { + "column": 8, + "line": 1, + "offset": 7, + }, + }, + "type": 14, + }, +] +`; + exports[`token analysis > "hi @\\n. upper\\n: {'name'}\\n !" errors 1`] = ` [ { @@ -10520,6 +10604,30 @@ exports[`tokenize options: location disable > "hi @:name !" tokens 1`] = ` ] `; +exports[`tokenize options: location disable > "hi @:名前" tokens 1`] = ` +[ + { + "type": 0, + "value": "hi ", + }, + { + "type": 8, + "value": "@", + }, + { + "type": 10, + "value": ":", + }, + { + "type": 11, + "value": "名前", + }, + { + "type": 14, + }, +] +`; + exports[`tokenize options: location disable > "hi @\\n. upper\\n: {'name'}\\n !" errors 1`] = ` [ { diff --git a/packages/message-compiler/test/tokenizer.test.ts b/packages/message-compiler/test/tokenizer.test.ts index 01db8b8b4..303c3037d 100644 --- a/packages/message-compiler/test/tokenizer.test.ts +++ b/packages/message-compiler/test/tokenizer.test.ts @@ -80,6 +80,7 @@ const CASES = [ `hi @.upper\n{name} !`, `hi @.upper {name} !`, `hi @:\nname !`, + `hi @:名前`, `hi @: {'name'} !`, `hi @\n. upper\n: {'name'}\n !`, ` | | |`, diff --git a/packages/vue-i18n-core/test/issues.test.ts b/packages/vue-i18n-core/test/issues.test.ts index c635eef96..a742fe404 100644 --- a/packages/vue-i18n-core/test/issues.test.ts +++ b/packages/vue-i18n-core/test/issues.test.ts @@ -1437,3 +1437,16 @@ test('#1796', async () => { }) ).toEqual('My message with hello world.') }) + +test('#1809', async () => { + const i18n = createI18n({ + locale: 'en', + messages: { + en: { + hi: 'hi @:名前', + 名前: 'kazupon' + } + } + }) + expect(i18n.global.t('hi')).toEqual('hi kazupon') +}) diff --git a/spec/syntax.ebnf b/spec/syntax.ebnf index 326aaf635..f09313a3a 100644 --- a/spec/syntax.ebnf +++ b/spec/syntax.ebnf @@ -1,5 +1,5 @@ (* - * Inltify message syntax v0.3 + * Inltify message syntax v0.4.0 * (vue-i18n compatible) *) @@ -18,7 +18,7 @@ Named ::= Modulo? "{" Space? (NamedIdentifier) Space? "}"; List ::= "{" Space? (NumberLiteral) Space? "}"; Linked ::= "@" (LinkedModifier)? LinkedDelimiter LinkedRefer; LinkedRefer ::= LinkedKey | Placeholder; -LinkedKey ::= Identifier; +LinkedKey ::= Text; LinkedModifier ::= LinkedDot Identifier; LinkedDelimiter ::= ":"; LinkedDot ::= "."; From 03711c9c857891b6e9b6f1a53b32ef72953d50b1 Mon Sep 17 00:00:00 2001 From: kazuya kawaguchi Date: Sat, 20 Apr 2024 22:45:27 +0900 Subject: [PATCH 2/2] fix: remove comment --- packages/message-compiler/src/tokenizer.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/message-compiler/src/tokenizer.ts b/packages/message-compiler/src/tokenizer.ts index 5d31a8d33..1e12face5 100644 --- a/packages/message-compiler/src/tokenizer.ts +++ b/packages/message-compiler/src/tokenizer.ts @@ -321,7 +321,6 @@ export function createTokenizer( return fn() } else { // other characters - // return isIdentifierStart(ch) return isTextStart(scnr, false) } }