fix(babel-parser): all index locations from nodes need to be offset b…

…y startIndex when reading source code
babel · Oct 25, 2024 · b0825c4 · b0825c4
1 parent 61de718
commit b0825c4
Show file tree

Hide file tree

Showing 12 changed files with 82 additions and 26 deletions.
diff --git a/packages/babel-parser/src/parser/comments.ts b/packages/babel-parser/src/parser/comments.ts
@@ -167,7 +167,11 @@ export default class CommentsParser extends BaseParser {
     } else {
       /*:: invariant(commentWS.containingNode !== null) */
       const { containingNode: node, start: commentStart } = commentWS;
-      if (this.input.charCodeAt(commentStart - 1) === charCodes.comma) {
+      if (
+        this.input.charCodeAt(
+          this.state.offsetToSourcePos(commentStart) - 1,
+        ) === charCodes.comma
+      ) {
         // If a commentWhitespace follows a comma and the containingNode allows
         // list structures with trailing comma, merge it to the trailingComment
         // of the last non-null list element

diff --git a/packages/babel-parser/src/parser/expression.ts b/packages/babel-parser/src/parser/expression.ts
@@ -291,7 +291,9 @@ export default abstract class ExpressionParser extends LValParser {
     const { type } = this.state;
 
     if (type === tt.parenL || tokenIsIdentifier(type)) {
-      this.state.potentialArrowAt = this.state.start;
+      this.state.potentialArrowAt = this.state.sourceToOffsetPos(
+        this.state.start,
+      );
     }
 
     let left = this.parseMaybeConditional(refExpressionErrors);
@@ -1121,7 +1123,9 @@ export default abstract class ExpressionParser extends LValParser {
         return this.parseBooleanLiteral(false);
 
       case tt.parenL: {
-        const canBeArrow = this.state.potentialArrowAt === this.state.start;
+        const canBeArrow =
+          this.state.potentialArrowAt ===
+          this.state.sourceToOffsetPos(this.state.start);
         return this.parseParenAndDistinguishExpression(canBeArrow);
       }
 
@@ -1259,7 +1263,9 @@ export default abstract class ExpressionParser extends LValParser {
           ) {
             return this.parseModuleExpression();
           }
-          const canBeArrow = this.state.potentialArrowAt === this.state.start;
+          const canBeArrow =
+            this.state.potentialArrowAt ===
+            this.state.sourceToOffsetPos(this.state.start);
           const containsEsc = this.state.containsEsc;
           const id = this.parseIdentifier();
 
@@ -1662,7 +1668,14 @@ export default abstract class ExpressionParser extends LValParser {
     node: any,
   ): T {
     this.addExtra(node, "rawValue", value);
-    this.addExtra(node, "raw", this.input.slice(node.start, this.state.end));
+    this.addExtra(
+      node,
+      "raw",
+      this.input.slice(
+        this.state.offsetToSourcePos(node.start),
+        this.state.end,
+      ),
+    );
     node.value = value;
     this.next();
     return this.finishNode<T>(node, type);
@@ -1696,7 +1709,14 @@ export default abstract class ExpressionParser extends LValParser {
     flags: N.RegExpLiteral["flags"];
   }) {
     const node = this.startNode<N.RegExpLiteral>();
-    this.addExtra(node, "raw", this.input.slice(node.start, this.state.end));
+    this.addExtra(
+      node,
+      "raw",
+      this.input.slice(
+        this.state.offsetToSourcePos(node.start),
+        this.state.end,
+      ),
+    );
     node.pattern = value.pattern;
     node.flags = value.flags;
     this.next();
@@ -1950,7 +1970,12 @@ export default abstract class ExpressionParser extends LValParser {
     const endOffset = isTail ? -1 : -2;
     const elemEnd = end + endOffset;
     elem.value = {
-      raw: this.input.slice(elemStart, elemEnd).replace(/\r\n?/g, "\n"),
+      raw: this.input
+        .slice(
+          this.state.offsetToSourcePos(elemStart),
+          this.state.offsetToSourcePos(elemEnd),
+        )
+        .replace(/\r\n?/g, "\n"),
       cooked: value === null ? null : value.slice(1, endOffset),
     };
     elem.tail = isTail;
@@ -3141,7 +3166,9 @@ export default abstract class ExpressionParser extends LValParser {
   parseFSharpPipelineBody(this: Parser, prec: number): N.Expression {
     const startLoc = this.state.startLoc;
 
-    this.state.potentialArrowAt = this.state.start;
+    this.state.potentialArrowAt = this.state.sourceToOffsetPos(
+      this.state.start,
+    );
     const oldInFSharpPipelineDirectBody = this.state.inFSharpPipelineDirectBody;
     this.state.inFSharpPipelineDirectBody = true;
 

diff --git a/packages/babel-parser/src/parser/statement.ts b/packages/babel-parser/src/parser/statement.ts
@@ -252,7 +252,10 @@ export default abstract class StatementParser extends ExpressionParser {
 
     const directiveLiteral = directive.value;
     const expressionValue = directiveLiteral.value;
-    const raw = this.input.slice(directiveLiteral.start, directiveLiteral.end);
+    const raw = this.input.slice(
+      this.state.offsetToSourcePos(directiveLiteral.start),
+      this.state.offsetToSourcePos(directiveLiteral.end),
+    );
     const val = (directiveLiteral.value = raw.slice(1, -1)); // remove quotes
 
     this.addExtra(directiveLiteral, "raw", raw);
@@ -1282,7 +1285,7 @@ export default abstract class StatementParser extends ExpressionParser {
     for (let i = this.state.labels.length - 1; i >= 0; i--) {
       const label = this.state.labels[i];
       if (label.statementStart === node.start) {
-        label.statementStart = this.state.start;
+        label.statementStart = this.state.sourceToOffsetPos(this.state.start);
         label.kind = kind;
       } else {
         break;
@@ -1292,7 +1295,7 @@ export default abstract class StatementParser extends ExpressionParser {
     this.state.labels.push({
       name: maybeName,
       kind: kind,
-      statementStart: this.state.start,
+      statementStart: this.state.sourceToOffsetPos(this.state.start),
     });
     // https://tc39.es/ecma262/#prod-LabelledItem
     node.body =

diff --git a/packages/babel-parser/src/parser/util.ts b/packages/babel-parser/src/parser/util.ts
@@ -128,7 +128,7 @@ export default abstract class UtilParser extends Tokenizer {
   hasPrecedingLineBreak(): boolean {
     return hasNewLine(
       this.input,
-      this.state.lastTokEndLoc.index,
+      this.state.offsetToSourcePos(this.state.lastTokEndLoc.index),
       this.state.start,
     );
   }

diff --git a/packages/babel-parser/src/plugins/flow/index.ts b/packages/babel-parser/src/plugins/flow/index.ts
@@ -2107,7 +2107,9 @@ export default (superClass: typeof Parser) =>
       consequent: N.Expression;
       failed: boolean;
     } {
-      this.state.noArrowParamsConversionAt.push(this.state.start);
+      this.state.noArrowParamsConversionAt.push(
+        this.state.sourceToOffsetPos(this.state.start),
+      );
 
       const consequent = this.parseMaybeAssignAllowIn();
       const failed = !this.match(tt.colon);
@@ -2181,7 +2183,9 @@ export default (superClass: typeof Parser) =>
     ): T {
       let result: T;
       if (this.state.noArrowParamsConversionAt.includes(node.start)) {
-        this.state.noArrowParamsConversionAt.push(this.state.start);
+        this.state.noArrowParamsConversionAt.push(
+          this.state.sourceToOffsetPos(this.state.start),
+        );
         result = parse();
         this.state.noArrowParamsConversionAt.pop();
       } else {
@@ -3134,7 +3138,10 @@ export default (superClass: typeof Parser) =>
 
     parseParenAndDistinguishExpression(canBeArrow: boolean): N.Expression {
       return super.parseParenAndDistinguishExpression(
-        canBeArrow && !this.state.noArrowAt.includes(this.state.start),
+        canBeArrow &&
+          !this.state.noArrowAt.includes(
+            this.state.sourceToOffsetPos(this.state.start),
+          ),
       );
     }
 

diff --git a/packages/babel-parser/src/plugins/placeholders.ts b/packages/babel-parser/src/plugins/placeholders.ts
@@ -391,7 +391,10 @@ export default (superClass: typeof Parser) =>
 
     // Throws if the current token and the prev one are separated by a space.
     assertNoSpace(): void {
-      if (this.state.start > this.state.lastTokEndLoc.index) {
+      if (
+        this.state.start >
+        this.state.offsetToSourcePos(this.state.lastTokEndLoc.index)
+      ) {
         this.raise(PlaceholderErrors.UnexpectedSpace, this.state.lastTokEndLoc);
       }
     }

diff --git a/packages/babel-parser/src/plugins/typescript/index.ts b/packages/babel-parser/src/plugins/typescript/index.ts
@@ -3237,7 +3237,7 @@ export default (superClass: ClassWithMixin<typeof Parser, IJSXParserMixin>) =>
             propertyName:
               key.type === "Identifier" && !node.computed
                 ? key.name
-                : `[${this.input.slice(key.start, key.end)}]`,
+                : `[${this.input.slice(this.state.offsetToSourcePos(key.start), this.state.offsetToSourcePos(key.end))}]`,
           },
         );
       }
@@ -4002,7 +4002,7 @@ export default (superClass: ClassWithMixin<typeof Parser, IJSXParserMixin>) =>
             methodName:
               key.type === "Identifier" && !method.computed
                 ? key.name
-                : `[${this.input.slice(key.start, key.end)}]`,
+                : `[${this.input.slice(key.start - this.state.startIndex, key.end - this.state.startIndex)}]`,
           });
         }
       }

diff --git a/packages/babel-parser/src/tokenizer/index.ts b/packages/babel-parser/src/tokenizer/index.ts
@@ -153,6 +153,8 @@ export default abstract class Tokenizer extends CommentsParser {
       curLine: state.curLine,
       lineStart: state.lineStart,
       curPosition: state.curPosition,
+      sourceToOffsetPos: state.sourceToOffsetPos,
+      offsetToSourcePos: state.offsetToSourcePos,
     };
   }
 

diff --git a/packages/babel-parser/src/tokenizer/state.ts b/packages/babel-parser/src/tokenizer/state.ts
@@ -174,7 +174,7 @@ export default class State {
    */
 
   curPosition(): Position {
-    const index = this.pos + this.startIndex;
+    const index = this.sourceToOffsetPos(this.pos);
     return new Position(this.curLine, index - this.lineStart, index);
   }
 
@@ -208,6 +208,14 @@ export default class State {
 
     return state;
   }
+
+  sourceToOffsetPos(sourcePos: number) {
+    return sourcePos + this.startIndex;
+  }
+
+  offsetToSourcePos(offsetPos: number) {
+    return offsetPos - this.startIndex;
+  }
 }
 
 export type LookaheadState = {
@@ -221,7 +229,9 @@ export type LookaheadState = {
   lastTokEndLoc: Position;
   curLine: number;
   lineStart: number;
-  curPosition: () => Position;
+  curPosition: State["curPosition"];
+  sourceToOffsetPos: State["sourceToOffsetPos"];
+  offsetToSourcePos: State["offsetToSourcePos"];
   /* Used only in readToken_mult_modulo */
   inType: boolean;
   // These boolean properties are not initialized in createLookaheadState()

diff --git a/packages/babel-parser/test/fixtures/core/categorized/startcolumn-specified/output.json b/packages/babel-parser/test/fixtures/core/categorized/startcolumn-specified/output.json
@@ -24,7 +24,7 @@
               "start":8,"end":9,"loc":{"start":{"line":1,"column":11,"index":8},"end":{"line":1,"column":12,"index":9}},
               "extra": {
                 "rawValue": 1,
-                "raw": ""
+                "raw": "1"
               },
               "value": 1
             }
@@ -48,7 +48,7 @@
               "start":16,"end":17,"loc":{"start":{"line":2,"column":7,"index":16},"end":{"line":2,"column":8,"index":17}},
               "extra": {
                 "rawValue": 2,
-                "raw": ""
+                "raw": "2"
               },
               "value": 2
             }

diff --git a/packages/babel-parser/test/fixtures/core/categorized/startindex-specified/output.json b/packages/babel-parser/test/fixtures/core/categorized/startindex-specified/output.json
@@ -24,7 +24,7 @@
               "start":8,"end":9,"loc":{"start":{"line":1,"column":11,"index":8},"end":{"line":1,"column":12,"index":9}},
               "extra": {
                 "rawValue": 1,
-                "raw": ""
+                "raw": "1"
               },
               "value": 1
             }
@@ -48,7 +48,7 @@
               "start":16,"end":17,"loc":{"start":{"line":2,"column":7,"index":16},"end":{"line":2,"column":8,"index":17}},
               "extra": {
                 "rawValue": 2,
-                "raw": ""
+                "raw": "2"
               },
               "value": 2
             }

diff --git a/...test/fixtures/core/categorized/startindex-startline-and-startcolumn-specified/output.json b/...test/fixtures/core/categorized/startindex-startline-and-startcolumn-specified/output.json
@@ -24,7 +24,7 @@
               "start":8,"end":9,"loc":{"start":{"line":3,"column":11,"index":8},"end":{"line":3,"column":12,"index":9}},
               "extra": {
                 "rawValue": 1,
-                "raw": ""
+                "raw": "1"
               },
               "value": 1
             }
@@ -48,7 +48,7 @@
               "start":16,"end":17,"loc":{"start":{"line":4,"column":7,"index":16},"end":{"line":4,"column":8,"index":17}},
               "extra": {
                 "rawValue": 2,
-                "raw": ""
+                "raw": "2"
               },
               "value": 2
             }