fix(babel-parser): all index locations from nodes need to be offset b…

…y startIndex when reading source code
babel · Oct 25, 2024 · e8d74ca · e8d74ca
1 parent e5e3bd2
commit e8d74ca
Show file tree

Hide file tree

Showing 12 changed files with 80 additions and 25 deletions.
diff --git a/packages/babel-parser/src/parser/comments.ts b/packages/babel-parser/src/parser/comments.ts
@@ -167,7 +167,11 @@ export default class CommentsParser extends BaseParser {
     } else {
       /*:: invariant(commentWS.containingNode !== null) */
       const { containingNode: node, start: commentStart } = commentWS;
-      if (this.input.charCodeAt(commentStart - 1) === charCodes.comma) {
+      if (
+        this.input.charCodeAt(
+          this.state.offsetToSourcePos(commentStart) - 1,
+        ) === charCodes.comma
+      ) {
         // If a commentWhitespace follows a comma and the containingNode allows
         // list structures with trailing comma, merge it to the trailingComment
         // of the last non-null list element

diff --git a/packages/babel-parser/src/parser/expression.ts b/packages/babel-parser/src/parser/expression.ts
@@ -156,7 +156,8 @@ export default abstract class ExpressionParser extends LValParser {
     potentialArrowAt: number,
   ): expr is N.ArrowFunctionExpression {
     return (
-      expr.type === "ArrowFunctionExpression" && expr.start === potentialArrowAt
+      expr.type === "ArrowFunctionExpression" &&
+      this.state.offsetToSourcePos(expr.start) === potentialArrowAt
     );
   }
 
@@ -949,7 +950,7 @@ export default abstract class ExpressionParser extends LValParser {
       !this.canInsertSemicolon() &&
       // check there are no escape sequences, such as \u{61}sync
       base.end - base.start === 5 &&
-      base.start === this.state.potentialArrowAt
+      this.state.offsetToSourcePos(base.start) === this.state.potentialArrowAt
     );
   }
 
@@ -1662,7 +1663,14 @@ export default abstract class ExpressionParser extends LValParser {
     node: any,
   ): T {
     this.addExtra(node, "rawValue", value);
-    this.addExtra(node, "raw", this.input.slice(node.start, this.state.end));
+    this.addExtra(
+      node,
+      "raw",
+      this.input.slice(
+        this.state.offsetToSourcePos(node.start),
+        this.state.end,
+      ),
+    );
     node.value = value;
     this.next();
     return this.finishNode<T>(node, type);
@@ -1696,7 +1704,14 @@ export default abstract class ExpressionParser extends LValParser {
     flags: N.RegExpLiteral["flags"];
   }) {
     const node = this.startNode<N.RegExpLiteral>();
-    this.addExtra(node, "raw", this.input.slice(node.start, this.state.end));
+    this.addExtra(
+      node,
+      "raw",
+      this.input.slice(
+        this.state.offsetToSourcePos(node.start),
+        this.state.end,
+      ),
+    );
     node.pattern = value.pattern;
     node.flags = value.flags;
     this.next();
@@ -1950,7 +1965,12 @@ export default abstract class ExpressionParser extends LValParser {
     const endOffset = isTail ? -1 : -2;
     const elemEnd = end + endOffset;
     elem.value = {
-      raw: this.input.slice(elemStart, elemEnd).replace(/\r\n?/g, "\n"),
+      raw: this.input
+        .slice(
+          this.state.offsetToSourcePos(elemStart),
+          this.state.offsetToSourcePos(elemEnd),
+        )
+        .replace(/\r\n?/g, "\n"),
       cooked: value === null ? null : value.slice(1, endOffset),
     };
     elem.tail = isTail;

diff --git a/packages/babel-parser/src/parser/statement.ts b/packages/babel-parser/src/parser/statement.ts
@@ -252,7 +252,10 @@ export default abstract class StatementParser extends ExpressionParser {
 
     const directiveLiteral = directive.value;
     const expressionValue = directiveLiteral.value;
-    const raw = this.input.slice(directiveLiteral.start, directiveLiteral.end);
+    const raw = this.input.slice(
+      this.state.offsetToSourcePos(directiveLiteral.start),
+      this.state.offsetToSourcePos(directiveLiteral.end),
+    );
     const val = (directiveLiteral.value = raw.slice(1, -1)); // remove quotes
 
     this.addExtra(directiveLiteral, "raw", raw);
@@ -1282,7 +1285,7 @@ export default abstract class StatementParser extends ExpressionParser {
     for (let i = this.state.labels.length - 1; i >= 0; i--) {
       const label = this.state.labels[i];
       if (label.statementStart === node.start) {
-        label.statementStart = this.state.start;
+        label.statementStart = this.state.sourceToOffsetPos(this.state.start);
         label.kind = kind;
       } else {
         break;
@@ -1292,7 +1295,7 @@ export default abstract class StatementParser extends ExpressionParser {
     this.state.labels.push({
       name: maybeName,
       kind: kind,
-      statementStart: this.state.start,
+      statementStart: this.state.sourceToOffsetPos(this.state.start),
     });
     // https://tc39.es/ecma262/#prod-LabelledItem
     node.body =

diff --git a/packages/babel-parser/src/parser/util.ts b/packages/babel-parser/src/parser/util.ts
@@ -128,7 +128,7 @@ export default abstract class UtilParser extends Tokenizer {
   hasPrecedingLineBreak(): boolean {
     return hasNewLine(
       this.input,
-      this.state.lastTokEndLoc.index,
+      this.state.offsetToSourcePos(this.state.lastTokEndLoc.index),
       this.state.start,
     );
   }

diff --git a/packages/babel-parser/src/plugins/flow/index.ts b/packages/babel-parser/src/plugins/flow/index.ts
@@ -2180,7 +2180,11 @@ export default (superClass: typeof Parser) =>
       parse: () => T,
     ): T {
       let result: T;
-      if (this.state.noArrowParamsConversionAt.includes(node.start)) {
+      if (
+        this.state.noArrowParamsConversionAt.includes(
+          this.state.offsetToSourcePos(node.start),
+        )
+      ) {
         this.state.noArrowParamsConversionAt.push(this.state.start);
         result = parse();
         this.state.noArrowParamsConversionAt.pop();
@@ -3097,7 +3101,11 @@ export default (superClass: typeof Parser) =>
         | Array<N.Expression | N.SpreadElement>
         | Array<N.Expression | N.RestElement>,
     ): void {
-      if (this.state.noArrowParamsConversionAt.includes(node.start)) {
+      if (
+        this.state.noArrowParamsConversionAt.includes(
+          this.state.offsetToSourcePos(node.start),
+        )
+      ) {
         node.params = params as N.ArrowFunctionExpression["params"];
       } else {
         super.setArrowFunctionParameters(node, params);
@@ -3112,7 +3120,9 @@ export default (superClass: typeof Parser) =>
     ): void {
       if (
         isArrowFunction &&
-        this.state.noArrowParamsConversionAt.includes(node.start)
+        this.state.noArrowParamsConversionAt.includes(
+          this.state.offsetToSourcePos(node.start),
+        )
       ) {
         return;
       }
@@ -3134,7 +3144,10 @@ export default (superClass: typeof Parser) =>
 
     parseParenAndDistinguishExpression(canBeArrow: boolean): N.Expression {
       return super.parseParenAndDistinguishExpression(
-        canBeArrow && !this.state.noArrowAt.includes(this.state.start),
+        canBeArrow &&
+          !this.state.noArrowAt.includes(
+            this.state.sourceToOffsetPos(this.state.start),
+          ),
       );
     }
 

diff --git a/packages/babel-parser/src/plugins/placeholders.ts b/packages/babel-parser/src/plugins/placeholders.ts
@@ -391,7 +391,10 @@ export default (superClass: typeof Parser) =>
 
     // Throws if the current token and the prev one are separated by a space.
     assertNoSpace(): void {
-      if (this.state.start > this.state.lastTokEndLoc.index) {
+      if (
+        this.state.start >
+        this.state.offsetToSourcePos(this.state.lastTokEndLoc.index)
+      ) {
         this.raise(PlaceholderErrors.UnexpectedSpace, this.state.lastTokEndLoc);
       }
     }

diff --git a/packages/babel-parser/src/plugins/typescript/index.ts b/packages/babel-parser/src/plugins/typescript/index.ts
@@ -3237,7 +3237,7 @@ export default (superClass: ClassWithMixin<typeof Parser, IJSXParserMixin>) =>
             propertyName:
               key.type === "Identifier" && !node.computed
                 ? key.name
-                : `[${this.input.slice(key.start, key.end)}]`,
+                : `[${this.input.slice(this.state.offsetToSourcePos(key.start), this.state.offsetToSourcePos(key.end))}]`,
           },
         );
       }
@@ -4002,7 +4002,7 @@ export default (superClass: ClassWithMixin<typeof Parser, IJSXParserMixin>) =>
             methodName:
               key.type === "Identifier" && !method.computed
                 ? key.name
-                : `[${this.input.slice(key.start, key.end)}]`,
+                : `[${this.input.slice(this.state.offsetToSourcePos(key.start), this.state.offsetToSourcePos(key.end))}]`,
           });
         }
       }

diff --git a/packages/babel-parser/src/tokenizer/index.ts b/packages/babel-parser/src/tokenizer/index.ts
@@ -153,6 +153,8 @@ export default abstract class Tokenizer extends CommentsParser {
       curLine: state.curLine,
       lineStart: state.lineStart,
       curPosition: state.curPosition,
+      sourceToOffsetPos: state.sourceToOffsetPos,
+      offsetToSourcePos: state.offsetToSourcePos,
     };
   }
 

diff --git a/packages/babel-parser/src/tokenizer/state.ts b/packages/babel-parser/src/tokenizer/state.ts
@@ -174,7 +174,7 @@ export default class State {
    */
 
   curPosition(): Position {
-    const index = this.pos + this.startIndex;
+    const index = this.sourceToOffsetPos(this.pos);
     return new Position(this.curLine, index - this.lineStart, index);
   }
 
@@ -208,6 +208,14 @@ export default class State {
 
     return state;
   }
+
+  sourceToOffsetPos(sourcePos: number) {
+    return sourcePos + this.startIndex;
+  }
+
+  offsetToSourcePos(offsetPos: number) {
+    return offsetPos - this.startIndex;
+  }
 }
 
 export type LookaheadState = {
@@ -221,7 +229,9 @@ export type LookaheadState = {
   lastTokEndLoc: Position;
   curLine: number;
   lineStart: number;
-  curPosition: () => Position;
+  curPosition: State["curPosition"];
+  sourceToOffsetPos: State["sourceToOffsetPos"];
+  offsetToSourcePos: State["offsetToSourcePos"];
   /* Used only in readToken_mult_modulo */
   inType: boolean;
   // These boolean properties are not initialized in createLookaheadState()

diff --git a/packages/babel-parser/test/fixtures/core/categorized/startcolumn-specified/output.json b/packages/babel-parser/test/fixtures/core/categorized/startcolumn-specified/output.json
@@ -24,7 +24,7 @@
               "start":8,"end":9,"loc":{"start":{"line":1,"column":11,"index":8},"end":{"line":1,"column":12,"index":9}},
               "extra": {
                 "rawValue": 1,
-                "raw": ""
+                "raw": "1"
               },
               "value": 1
             }
@@ -48,7 +48,7 @@
               "start":16,"end":17,"loc":{"start":{"line":2,"column":7,"index":16},"end":{"line":2,"column":8,"index":17}},
               "extra": {
                 "rawValue": 2,
-                "raw": ""
+                "raw": "2"
               },
               "value": 2
             }

diff --git a/packages/babel-parser/test/fixtures/core/categorized/startindex-specified/output.json b/packages/babel-parser/test/fixtures/core/categorized/startindex-specified/output.json
@@ -24,7 +24,7 @@
               "start":8,"end":9,"loc":{"start":{"line":1,"column":11,"index":8},"end":{"line":1,"column":12,"index":9}},
               "extra": {
                 "rawValue": 1,
-                "raw": ""
+                "raw": "1"
               },
               "value": 1
             }
@@ -48,7 +48,7 @@
               "start":16,"end":17,"loc":{"start":{"line":2,"column":7,"index":16},"end":{"line":2,"column":8,"index":17}},
               "extra": {
                 "rawValue": 2,
-                "raw": ""
+                "raw": "2"
               },
               "value": 2
             }

diff --git a/...test/fixtures/core/categorized/startindex-startline-and-startcolumn-specified/output.json b/...test/fixtures/core/categorized/startindex-startline-and-startcolumn-specified/output.json
@@ -24,7 +24,7 @@
               "start":8,"end":9,"loc":{"start":{"line":3,"column":11,"index":8},"end":{"line":3,"column":12,"index":9}},
               "extra": {
                 "rawValue": 1,
-                "raw": ""
+                "raw": "1"
               },
               "value": 1
             }
@@ -48,7 +48,7 @@
               "start":16,"end":17,"loc":{"start":{"line":4,"column":7,"index":16},"end":{"line":4,"column":8,"index":17}},
               "extra": {
                 "rawValue": 2,
-                "raw": ""
+                "raw": "2"
               },
               "value": 2
             }