fix(parser): remove all duplicated comments in trivia builder (#2689)

crates/oxc_parser/src/lexer/mod.rs

@@ -37,7 +37,7 @@ use oxc_span::{SourceType, Span};
 use self::{
  byte_handlers::handle_byte,
  source::{Source, SourcePosition},
- trivia_builder::{TriviaBuilder, TriviasCheckpoint},
+ trivia_builder::TriviaBuilder,
 };
 pub use self::{
  kind::Kind,
@@ -54,8 +54,6 @@ pub struct LexerCheckpoint<'a> {
  token: Token,
 
  errors_pos: usize,
-
- trivias: TriviasCheckpoint,
 }
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
@@ -155,14 +153,12 @@ impl<'a> Lexer<'a> {
  position: self.source.position(),
  token: self.token,
  errors_pos: self.errors.len(),
- trivias: self.trivia_builder.checkpoint(),
  }
  }
 
  /// Rewinds the lexer to the same state as when the passed in `checkpoint` was created.
  pub fn rewind(&mut self, checkpoint: LexerCheckpoint<'a>) {
  self.errors.truncate(checkpoint.errors_pos);
- self.trivia_builder.rewind(checkpoint.trivias);
  self.source.set_position(checkpoint.position);
  self.token = checkpoint.token;
  self.lookahead.clear();

crates/oxc_parser/src/lexer/trivia_builder.rs

@@ -1,45 +1,38 @@
+use std::collections::BTreeMap;
+
 use oxc_ast::{CommentKind, Trivias};
 use oxc_span::Span;
 
 #[derive(Debug, Default)]
 pub struct TriviaBuilder {
- trivias: Trivias,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct TriviasCheckpoint {
- comments_len: usize,
- irregular_whitespaces_len: usize,
+ // Duplicated comments can be added from rewind, use `BTreeMap` to ensure uniqueness
+ comments: BTreeMap<u32, (u32, CommentKind)>,
+ irregular_whitespaces: Vec<Span>,
 }
 
 impl TriviaBuilder {
  pub fn build(self) -> Trivias {
- self.trivias
- }
-
- pub fn checkpoint(&self) -> TriviasCheckpoint {
- TriviasCheckpoint {
- comments_len: self.trivias.comments.len(),
- irregular_whitespaces_len: self.trivias.irregular_whitespaces.len(),
+ Trivias {
+  comments: self
+ .comments
+  .into_iter()
+  .map(|(start, (end, kind))| (start, end, kind))
+  .collect(),
+ irregular_whitespaces: self.irregular_whitespaces,
  }
  }
 
- pub fn rewind(&mut self, checkpoint: TriviasCheckpoint) {
- self.trivias.comments.truncate(checkpoint.comments_len);
- self.trivias.irregular_whitespaces.truncate(checkpoint.irregular_whitespaces_len);
- }
-
- /// skip leading `//`
  pub fn add_single_line_comment(&mut self, start: u32, end: u32) {
- self.trivias.comments.push((start + 2, end, CommentKind::SingleLine));
+ // skip leading `//`
+ self.comments.insert(start + 2, (end, CommentKind::SingleLine));
  }
 
- /// skip leading `/*` and trailing `*/`
  pub fn add_multi_line_comment(&mut self, start: u32, end: u32) {
- self.trivias.comments.push((start + 2, end - 2, CommentKind::MultiLine));
+ // skip leading `/*` and trailing `*/`
+ self.comments.insert(start + 2, (end - 2, CommentKind::MultiLine));
  }
 
  pub fn add_irregular_whitespace(&mut self, start: u32, end: u32) {
- self.trivias.irregular_whitespaces.push(Span::new(start, end));
+ self.irregular_whitespaces.push(Span::new(start, end));
  }
 }

tasks/coverage/codegen_misc.snap

@@ -1,3 +1,3 @@
 codegen_misc Summary:
-AST Parsed : 12/12 (100.00%)
-Positive Passed: 12/12 (100.00%)
+AST Parsed : 14/14 (100.00%)
+Positive Passed: 14/14 (100.00%)

tasks/coverage/misc/pass/oxc-2592.ts

@@ -0,0 +1,4 @@
+export type Foo = (
+ // comment
+ "bar"
+)

tasks/coverage/misc/pass/oxc-2674.tsx

@@ -0,0 +1,9 @@
+const foo = <div
+ // comment
+/>;
+
+<div-
+ // comment
+/>;
+
+<div-- /* comment */ />;

tasks/coverage/parser_babel.snap

@@ -1,6 +1,6 @@
 parser_babel Summary:
 AST Parsed : 2090/2096 (99.71%)
-Positive Passed: 2080/2096 (99.24%)
+Positive Passed: 2083/2096 (99.38%)
 Negative Passed: 1362/1500 (90.80%)
 Expect Syntax Error: "annex-b/disabled/1.1-html-comments-close/input.js"
 Expect Syntax Error: "annex-b/disabled/3.1-sloppy-labeled-functions/input.js"
@@ -140,9 +140,6 @@ Expect Syntax Error: "typescript/types/read-only-3/input.ts"
 Expect Syntax Error: "typescript/types/read-only-4/input.ts"
 Expect Syntax Error: "typescript/types/tuple-optional-invalid/input.ts"
 Expect Syntax Error: "typescript/types/tuple-required-after-labeled-optional/input.ts"
-Duplicated comments " 3 ": "comments/basic/nested-parentheses/input.js"
-Duplicated comments " 1 ": "comments/basic/object-method/input.js"
-Duplicated comments " 3 ": "comments/basic/sequence-expression/input.js"
 Expect to Parse: "core/opts/allowNewTargetOutsideFunction-true/input.js"
 
  × Unexpected new.target expression

tasks/coverage/parser_misc.snap

@@ -1,6 +1,6 @@
 parser_misc Summary:
-AST Parsed : 12/12 (100.00%)
-Positive Passed: 12/12 (100.00%)
+AST Parsed : 14/14 (100.00%)
+Positive Passed: 14/14 (100.00%)
 Negative Passed: 8/8 (100.00%)
 
  × Unexpected token

tasks/coverage/parser_typescript.snap

@@ -1,6 +1,6 @@
 parser_typescript Summary:
 AST Parsed : 5240/5243 (99.94%)
-Positive Passed: 5231/5243 (99.77%)
+Positive Passed: 5233/5243 (99.81%)
 Negative Passed: 1062/4879 (21.77%)
 Expect Syntax Error: "compiler/ClassDeclaration10.ts"
 Expect Syntax Error: "compiler/ClassDeclaration11.ts"
@@ -2697,7 +2697,7 @@ Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardOfFormTypeOfEq
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardOfFormTypeOfNotEqualHasNoEffect.ts"
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardOfFormTypeOfOther.ts"
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsDefeat.ts"
-Duplicated comments " change value of x": "conformance/expressions/typeGuards/typeGuardsInIfStatement.ts"
+Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsInIfStatement.ts"
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsWithAny.ts"
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsWithInstanceOf.ts"
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsWithInstanceOfByConstructorSignature.ts"
@@ -3819,7 +3819,6 @@ Expect Syntax Error: "conformance/types/unknown/unknownControlFlow.ts"
 Expect Syntax Error: "conformance/types/unknown/unknownType1.ts"
 Expect Syntax Error: "conformance/types/unknown/unknownType2.ts"
 Expect Syntax Error: "conformance/types/witness/witness.ts"
-Duplicated comments "comment2": "compiler/awaitExpressionInnerCommentEmit.ts"
 Expect to Parse: "compiler/bom-utf16be.ts"
 
  × Invalid Character `￾`
@@ -3854,7 +3853,6 @@ Expect to Parse: "compiler/emitBundleWithShebangAndPrologueDirectives1.ts"
  · ─
  7 │ "use strict"
  ╰────
-Duplicated comments " must be identifier?": "compiler/jsxNestedWithinTernaryParsesCorrectly.tsx"
 Expect to Parse: "compiler/withStatementInternalComments.ts"
 
  × 'with' statements are not allowed

tasks/coverage/prettier_misc.snap

@@ -1,8 +1,9 @@
 prettier_misc Summary:
-AST Parsed : 12/12 (100.00%)
-Positive Passed: 7/12 (58.33%)
+AST Parsed : 14/14 (100.00%)
+Positive Passed: 8/14 (57.14%)
 Expect to Parse: "pass/oxc-1740.tsx"
 Expect to Parse: "pass/oxc-2087.ts"
 Expect to Parse: "pass/oxc-2394.ts"
+Expect to Parse: "pass/oxc-2674.tsx"
 Expect to Parse: "pass/swc-1627.js"
 Expect to Parse: "pass/swc-8243.tsx"

tasks/coverage/prettier_typescript.snap

@@ -1,6 +1,6 @@
 prettier_typescript Summary:
 AST Parsed : 5243/5243 (100.00%)
-Positive Passed: 2871/5243 (54.76%)
+Positive Passed: 2872/5243 (54.78%)
 Expect to Parse: "compiler/DeclarationErrorsNoEmitOnError.ts"
 Expect to Parse: "compiler/abstractInterfaceIdentifierName.ts"
 Expect to Parse: "compiler/abstractPropertyBasics.ts"
@@ -72,7 +72,6 @@ Expect to Parse: "compiler/augmentedTypesModules3b.ts"
 Expect to Parse: "compiler/augmentedTypesModules4.ts"
 Expect to Parse: "compiler/autonumberingInEnums.ts"
 Expect to Parse: "compiler/avoidCycleWithVoidExpressionReturnedFromArrow.ts"
-Expect to Parse: "compiler/awaitExpressionInnerCommentEmit.ts"
 Expect to Parse: "compiler/awaitUnionPromise.ts"
 Expect to Parse: "compiler/awaitedTypeCrash.ts"
 Expect to Parse: "compiler/awaitedTypeJQuery.ts"