Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(message-compiler): cannot resolve none-identifier characters at linked key #1813

Merged
merged 2 commits into from
Apr 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions packages/message-compiler/src/tokenizer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ export function createTokenizer(
return fn()
} else {
// other characters
return isIdentifierStart(ch)
return isTextStart(scnr, false)
}
}

Expand Down Expand Up @@ -680,7 +680,7 @@ export function createTokenizer(
}

function readLinkedRefer(scnr: Scanner): string {
const fn = (detect = false, buf: string): string => {
const fn = (buf: string): string => {
const ch = scnr.currentChar()
if (
ch === TokenChars.BraceLeft ||
Expand All @@ -697,15 +697,15 @@ export function createTokenizer(
} else if (ch === NEW_LINE || ch === DOT) {
buf += ch
scnr.next()
return fn(detect, buf)
return fn(buf)
} else {
buf += ch
scnr.next()
return fn(true, buf)
return fn(buf)
}
}

return fn(false, '')
return fn('')
}

function readPlural(scnr: Scanner): string {
Expand Down
64 changes: 0 additions & 64 deletions packages/message-compiler/test/__snapshots__/parser.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -525,28 +525,6 @@ exports[`parse > linked key paren error with modifier: "@.lower:(foo)" 1`] = `
}
`;

exports[`parse > linked key paren error with modifier: "@.lower:(foo)" errors 1`] = `
[
{
"code": 13,
"domain": "parser",
"location": {
"end": {
"column": 14,
"line": 1,
"offset": 13,
},
"start": {
"column": 8,
"line": 1,
"offset": 7,
},
},
"message": "Unexpected empty linked key",
},
]
`;

exports[`parse > linked key paren error: "@:(foo)" 1`] = `
{
"body": {
Expand Down Expand Up @@ -640,28 +618,6 @@ exports[`parse > linked key paren error: "@:(foo)" 1`] = `
}
`;

exports[`parse > linked key paren error: "@:(foo)" errors 1`] = `
[
{
"code": 13,
"domain": "parser",
"location": {
"end": {
"column": 8,
"line": 1,
"offset": 7,
},
"start": {
"column": 2,
"line": 1,
"offset": 1,
},
},
"message": "Unexpected empty linked key",
},
]
`;

exports[`parse > linked key with named and modifier: "hi @._upper:{_name} !" 1`] = `
{
"body": {
Expand Down Expand Up @@ -2024,16 +1980,6 @@ exports[`parser options > location disable > linked key paren error with modifie
}
`;

exports[`parser options > location disable > linked key paren error with modifier: "@.lower:(foo)" errors 1`] = `
[
{
"code": 13,
"domain": "parser",
"message": "Unexpected empty linked key",
},
]
`;

exports[`parser options > location disable > linked key paren error: "@:(foo)" 1`] = `
{
"body": {
Expand All @@ -2056,16 +2002,6 @@ exports[`parser options > location disable > linked key paren error: "@:(foo)" 1
}
`;

exports[`parser options > location disable > linked key paren error: "@:(foo)" errors 1`] = `
[
{
"code": 13,
"domain": "parser",
"message": "Unexpected empty linked key",
},
]
`;

exports[`parser options > location disable > linked key with named and modifier: "hi @._upper:{_name} !" 1`] = `
{
"body": {
Expand Down
108 changes: 108 additions & 0 deletions packages/message-compiler/test/__snapshots__/tokenizer.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -4370,6 +4370,90 @@ exports[`token analysis > "hi @:name !" tokens 1`] = `
]
`;

exports[`token analysis > "hi @:名前" tokens 1`] = `
[
{
"loc": {
"end": {
"column": 4,
"line": 1,
"offset": 3,
},
"start": {
"column": 1,
"line": 1,
"offset": 0,
},
},
"type": 0,
"value": "hi ",
},
{
"loc": {
"end": {
"column": 5,
"line": 1,
"offset": 4,
},
"start": {
"column": 4,
"line": 1,
"offset": 3,
},
},
"type": 8,
"value": "@",
},
{
"loc": {
"end": {
"column": 6,
"line": 1,
"offset": 5,
},
"start": {
"column": 5,
"line": 1,
"offset": 4,
},
},
"type": 10,
"value": ":",
},
{
"loc": {
"end": {
"column": 8,
"line": 1,
"offset": 7,
},
"start": {
"column": 6,
"line": 1,
"offset": 5,
},
},
"type": 11,
"value": "名前",
},
{
"loc": {
"end": {
"column": 8,
"line": 1,
"offset": 7,
},
"start": {
"column": 8,
"line": 1,
"offset": 7,
},
},
"type": 14,
},
]
`;

exports[`token analysis > "hi @\\n. upper\\n: {'name'}\\n !" errors 1`] = `
[
{
Expand Down Expand Up @@ -10520,6 +10604,30 @@ exports[`tokenize options: location disable > "hi @:name !" tokens 1`] = `
]
`;

exports[`tokenize options: location disable > "hi @:名前" tokens 1`] = `
[
{
"type": 0,
"value": "hi ",
},
{
"type": 8,
"value": "@",
},
{
"type": 10,
"value": ":",
},
{
"type": 11,
"value": "名前",
},
{
"type": 14,
},
]
`;

exports[`tokenize options: location disable > "hi @\\n. upper\\n: {'name'}\\n !" errors 1`] = `
[
{
Expand Down
1 change: 1 addition & 0 deletions packages/message-compiler/test/tokenizer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ const CASES = [
`hi @.upper\n{name} !`,
`hi @.upper {name} !`,
`hi @:\nname !`,
`hi @:名前`,
`hi @: {'name'} !`,
`hi @\n. upper\n: {'name'}\n !`,
` | | |`,
Expand Down
13 changes: 13 additions & 0 deletions packages/vue-i18n-core/test/issues.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1437,3 +1437,16 @@ test('#1796', async () => {
})
).toEqual('My message with hello world.')
})

test('#1809', async () => {
const i18n = createI18n({
locale: 'en',
messages: {
en: {
hi: 'hi @:名前',
名前: 'kazupon'
}
}
})
expect(i18n.global.t('hi')).toEqual('hi kazupon')
})
4 changes: 2 additions & 2 deletions spec/syntax.ebnf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
(*
* Inltify message syntax v0.3
* Inltify message syntax v0.4.0
* (vue-i18n compatible)
*)

Expand All @@ -18,7 +18,7 @@ Named ::= Modulo? "{" Space? (NamedIdentifier) Space? "}";
List ::= "{" Space? (NumberLiteral) Space? "}";
Linked ::= "@" (LinkedModifier)? LinkedDelimiter LinkedRefer;
LinkedRefer ::= LinkedKey | Placeholder;
LinkedKey ::= Identifier;
LinkedKey ::= Text;
LinkedModifier ::= LinkedDot Identifier;
LinkedDelimiter ::= ":";
LinkedDot ::= ".";
Expand Down