How to proper prioritize a symbol that can be used for multiple goals? (hash symbol for arm assebly)

[maxcnunes]

On assembly for arm64 instruction sets, the harsh symbol can be either a comment or a prefix for a value. For example:

mov X1, #9
mov X1, #9 # my comment

It should have this parsing result:

• mov should be a mov_instruction
• X1 should be a arg_src
• #9 should be a arg_dst
• # comment should be a comment

In the first line, #9 is properly recognized as a arg_dst type. But on the second line the whole #9 # my comment becomes a comment, which is incorrect.

This is a minimal parse configuration I got to test it:

module.exports = grammar({
  name: "asm",

  extras: ($) => [/\s|\t|\r/, $.comment],

  rules: {
    program: ($) => repeat($._definition),

    _definition: ($) =>
      choice(
        $.mov_instruction,
        // TODO: other kinds of definitions
      ),

    mov_instruction: ($) => seq("mov", $.arg_src, ",", $.arg_dst),

    arg_src: ($) => /X[0-9]+/,
    arg_dst: ($) => /#[0-9]+/,

    comment: ($) =>
      choice(
        token(
          choice(
            // comment types
            seq("#", /.*/),
            seq(";", /.*/),
            seq("//", /.*/),
          ),
        ),
      ),
  },
});

Here is the parse debug output I get for mov X1, #9:

new_parse
process version:0, version_count:1, state:1, row:0, col:0
lex_internal state:5, row:0, column:0
  consume character:'m'
  consume character:'o'
  consume character:'v'
lexed_lookahead sym:mov, size:3
shift state:7
process version:0, version_count:1, state:7, row:0, col:3
lex_internal state:5, row:0, column:3
  skip character:' '
  consume character:'X'
  consume character:'1'
lexed_lookahead sym:arg_src, size:3
shift state:9
process version:0, version_count:1, state:9, row:0, col:6
lex_internal state:5, row:0, column:6
  consume character:','
lexed_lookahead sym:,, size:1
shift state:10
process version:0, version_count:1, state:10, row:0, col:7
lex_internal state:0, row:0, column:7
  skip character:' '
  consume character:'#'
  consume character:'9'
lexed_lookahead sym:arg_dst, size:3
shift state:6
process version:0, version_count:1, state:6, row:0, col:10
lex_internal state:5, row:0, column:10
  skip character:10
lexed_lookahead sym:end, size:1
reduce sym:mov_instruction, child_count:4
reduce sym:_definition, child_count:1
reduce sym:program_repeat1, child_count:1
reduce sym:program, child_count:1
accept
done
(program [0, 0] - [1, 0]
  (mov_instruction [0, 0] - [0, 10]
    (arg_src [0, 4] - [0, 6])
    (arg_dst [0, 8] - [0, 10])))

And this is the parse debug output I get for mov X1, #9 (with an extra space, I am not even including the second # there and it already becomes a comment by mistake):

new_parse
process version:0, version_count:1, state:1, row:0, col:0
lex_internal state:5, row:0, column:0
  consume character:'m'
  consume character:'o'
  consume character:'v'
lexed_lookahead sym:mov, size:3
shift state:7
process version:0, version_count:1, state:7, row:0, col:3
lex_internal state:5, row:0, column:3
  skip character:' '
  consume character:'X'
  consume character:'1'
lexed_lookahead sym:arg_src, size:3
shift state:9
process version:0, version_count:1, state:9, row:0, col:6
lex_internal state:5, row:0, column:6
  consume character:','
lexed_lookahead sym:,, size:1
shift state:10
process version:0, version_count:1, state:10, row:0, col:7
lex_internal state:0, row:0, column:7
  skip character:' '
  consume character:'#'
  consume character:'9'
  consume character:' '
lexed_lookahead sym:comment_token1, size:4
shift state:11
process version:0, version_count:1, state:11, row:0, col:11
no_lookahead_after_non_terminal_extra
reduce sym:comment, child_count:1
lex_internal state:0, row:0, column:11
  skip character:10
lexed_lookahead sym:end, size:1
detect_error
resume version:0
recover_with_missing symbol:arg_dst, state:6
recover_to_previous state:1, depth:4
recover_eof
process version:1, version_count:6, state:6, row:0, col:11
reduce sym:mov_instruction, child_count:4
detect_error
process version:2, version_count:6, state:5, row:0, col:11
reduce sym:_definition, child_count:1
detect_error
process version:3, version_count:6, state:4, row:0, col:11
reduce sym:program_repeat1, child_count:1
detect_error
process version:4, version_count:6, state:2, row:0, col:11
reduce sym:program, child_count:1
accept
select_smaller_error symbol:program, over_symbol:ERROR
process version:5, version_count:6, state:1, row:0, col:11
reduce sym:program, child_count:0
accept
select_smaller_error symbol:program, over_symbol:program
resume version:0
recover_eof
select_smaller_error symbol:program, over_symbol:ERROR
select_smaller_error symbol:program, over_symbol:ERROR
select_smaller_error symbol:program, over_symbol:ERROR
select_smaller_error symbol:program, over_symbol:ERROR
done
(program [0, 0] - [1, 0]
  (mov_instruction [0, 0] - [0, 11]
    (arg_src [0, 4] - [0, 6])
    (comment [0, 8] - [0, 11])
    (arg_dst [0, 11] - [0, 11])))

I really appreciate any feedback on how to solve it. I tried using prec and dynamic.prec on several places and none has worked so far.

[maxcnunes]

I got it to work by removing comment from extras, and properly configuring it on the parts it should be supported instead of automatically depending on the extras feature for that.

module.exports = grammar({
  name: "asm",

  extras: ($) => [/\s|\t|\r/],

  rules: {
    program: ($) => repeat(choice($._definition, $.comment)),

    _definition: ($) =>
      prec.left(
        1,
        seq(
          choice(
            $.mov_instruction,
            // TODO: other kinds of definitions
          ),
          optional($.comment),
        ),
      ),

    mov_instruction: ($) =>
      prec.left(1, seq("mov", $.arg_src, ",", $.arg_dst, optional($.comment))),

    arg_src: ($) => /X[0-9]+/,
    arg_dst: ($) => /#[0-9]+/,

    comment: ($) =>
      choice(
        token(
          choice(
            // comment types
            seq("#", /.*/),
            seq(";", /.*/),
            seq("//", /.*/),
          ),
        ),
      ),
  },
});

Now this:

# comment
; comment
// comment
mov X1, #9 # comment
mov X1, #9 ; comment
mov X1, #9 // comment

Outputs what I expected:

new_parse
process version:0, version_count:1, state:1, row:0, col:0
lex_internal state:6, row:0, column:0
  consume character:'#'
  consume character:' '
  consume character:'c'
  consume character:'o'
  consume character:'m'
  consume character:'m'
  consume character:'e'
  consume character:'n'
  consume character:'t'
lexed_lookahead sym:comment_token1, size:9
shift state:5
process version:0, version_count:1, state:5, row:0, col:9
lex_internal state:6, row:0, column:9
  skip character:10
  consume character:';'
  consume character:' '
  consume character:'c'
  consume character:'o'
  consume character:'m'
  consume character:'m'
  consume character:'e'
  consume character:'n'
  consume character:'t'
lexed_lookahead sym:comment_token1, size:10
reduce sym:comment, child_count:1
shift state:5
process version:0, version_count:1, state:5, row:1, col:9
lex_internal state:6, row:1, column:9
  skip character:10
  consume character:'/'
  consume character:'/'
  consume character:' '
  consume character:'c'
  consume character:'o'
  consume character:'m'
  consume character:'m'
  consume character:'e'
  consume character:'n'
  consume character:'t'
lexed_lookahead sym:comment_token1, size:11
reduce sym:comment, child_count:1
reduce sym:program_repeat1, child_count:2
shift state:5
process version:0, version_count:1, state:5, row:2, col:10
lex_internal state:6, row:2, column:10
  skip character:10
  consume character:'m'
  consume character:'o'
  consume character:'v'
lexed_lookahead sym:mov, size:4
reduce sym:comment, child_count:1
reduce sym:program_repeat1, child_count:2
shift state:8
process version:0, version_count:1, state:8, row:3, col:3
lex_internal state:0, row:3, column:3
  skip character:' '
  consume character:'X'
  consume character:'1'
lexed_lookahead sym:arg_src, size:3
shift state:10
process version:0, version_count:1, state:10, row:3, col:6
lex_internal state:0, row:3, column:6
  consume character:','
lexed_lookahead sym:,, size:1
shift state:11
process version:0, version_count:1, state:11, row:3, col:7
lex_internal state:0, row:3, column:7
  skip character:' '
  consume character:'#'
  consume character:'9'
lexed_lookahead sym:arg_dst, size:3
shift state:4
process version:0, version_count:1, state:4, row:3, col:10
lex_internal state:6, row:3, column:10
  skip character:' '
  consume character:'#'
  consume character:' '
  consume character:'c'
  consume character:'o'
  consume character:'m'
  consume character:'m'
  consume character:'e'
  consume character:'n'
  consume character:'t'
lexed_lookahead sym:comment_token1, size:10
reduce sym:mov_instruction, child_count:4
reduce sym:program_repeat1, child_count:2
shift state:5
process version:0, version_count:1, state:5, row:3, col:20
lex_internal state:6, row:3, column:20
  skip character:10
  consume character:'m'
  consume character:'o'
  consume character:'v'
lexed_lookahead sym:mov, size:4
reduce sym:comment, child_count:1
reduce sym:program_repeat1, child_count:2
shift state:8
process version:0, version_count:1, state:8, row:4, col:3
lex_internal state:0, row:4, column:3
  skip character:' '
  consume character:'X'
  consume character:'1'
lexed_lookahead sym:arg_src, size:3
shift state:10
process version:0, version_count:1, state:10, row:4, col:6
lex_internal state:0, row:4, column:6
  consume character:','
lexed_lookahead sym:,, size:1
shift state:11
process version:0, version_count:1, state:11, row:4, col:7
lex_internal state:0, row:4, column:7
  skip character:' '
  consume character:'#'
  consume character:'9'
lexed_lookahead sym:arg_dst, size:3
shift state:4
process version:0, version_count:1, state:4, row:4, col:10
lex_internal state:6, row:4, column:10
  skip character:' '
  consume character:';'
  consume character:' '
  consume character:'c'
  consume character:'o'
  consume character:'m'
  consume character:'m'
  consume character:'e'
  consume character:'n'
  consume character:'t'
lexed_lookahead sym:comment_token1, size:10
reduce sym:mov_instruction, child_count:4
reduce sym:program_repeat1, child_count:2
shift state:5
process version:0, version_count:1, state:5, row:4, col:20
lex_internal state:6, row:4, column:20
  skip character:10
  consume character:'m'
  consume character:'o'
  consume character:'v'
lexed_lookahead sym:mov, size:4
reduce sym:comment, child_count:1
reduce sym:program_repeat1, child_count:2
shift state:8
process version:0, version_count:1, state:8, row:5, col:3
lex_internal state:0, row:5, column:3
  skip character:' '
  consume character:'X'
  consume character:'1'
lexed_lookahead sym:arg_src, size:3
shift state:10
process version:0, version_count:1, state:10, row:5, col:6
lex_internal state:0, row:5, column:6
  consume character:','
lexed_lookahead sym:,, size:1
shift state:11
process version:0, version_count:1, state:11, row:5, col:7
lex_internal state:0, row:5, column:7
  skip character:' '
  consume character:'#'
  consume character:'9'
lexed_lookahead sym:arg_dst, size:3
shift state:4
process version:0, version_count:1, state:4, row:5, col:10
lex_internal state:6, row:5, column:10
  skip character:' '
  consume character:'/'
  consume character:'/'
  consume character:' '
  consume character:'c'
  consume character:'o'
  consume character:'m'
  consume character:'m'
  consume character:'e'
  consume character:'n'
  consume character:'t'
lexed_lookahead sym:comment_token1, size:11
reduce sym:mov_instruction, child_count:4
reduce sym:program_repeat1, child_count:2
shift state:5
process version:0, version_count:1, state:5, row:5, col:21
lex_internal state:6, row:5, column:21
  skip character:10
lexed_lookahead sym:end, size:1
reduce sym:comment, child_count:1
reduce sym:program_repeat1, child_count:2
reduce sym:program, child_count:1
accept
done
(program [0, 0] - [6, 0]
  (comment [0, 0] - [0, 9])
  (comment [1, 0] - [1, 9])
  (comment [2, 0] - [2, 10])
  (mov_instruction [3, 0] - [3, 10]
    (arg_src [3, 4] - [3, 6])
    (arg_dst [3, 8] - [3, 10]))
  (comment [3, 11] - [3, 20])
  (mov_instruction [4, 0] - [4, 10]
    (arg_src [4, 4] - [4, 6])
    (arg_dst [4, 8] - [4, 10]))
  (comment [4, 11] - [4, 20])
  (mov_instruction [5, 0] - [5, 10]
    (arg_src [5, 4] - [5, 6])
    (arg_dst [5, 8] - [5, 10]))
  (comment [5, 11] - [5, 21]))

I will go with that for now. But please, let me know if anyone has any better ideas.