Fix for some offset problems -- the biggest one being dedents now take on the

fwierzbicki · fwierzbicki · commit 1084af10fbee · 2008-09-04T16:57:30.000Z
offset values of the previous token.
diff --git a/grammar/Python.g b/grammar/Python.g
@@ -284,6 +284,12 @@ file_input
     List stypes = new ArrayList();
 }
 @after {
+    if (!stypes.isEmpty()) {
+        //The EOF token messes up the end offsets, so set them manually.
+        PythonTree stop = (PythonTree)stypes.get(stypes.size() -1);
+        mtype.setCharStopIndex(stop.getCharStopIndex());
+        mtype.setTokenStopIndex(stop.getTokenStopIndex());
+    }
     $file_input.tree = mtype;
 }
     : (NEWLINE
diff --git a/src/org/python/antlr/PythonTokenSource.java b/src/org/python/antlr/PythonTokenSource.java
@@ -157,14 +157,18 @@ protected void insertImaginaryIndentDedentTokens() {
         stream.consume();
 
         if (t.getType() == Token.EOF) {
+            Token prev = stream.LT(-1);
             if (!inSingle) {
-                Token prev = stream.LT(-1);
                 if (prev == null || prev.getType() != PythonLexer.NEWLINE) {
                     generateNewline(t);
                 }
             }
-
-            handleDedents(-1, (CommonToken)t);
+            if (prev != null) {
+                ((CommonToken)t).setStopIndex(((CommonToken)prev).getStopIndex());
+                handleDedents(-1, (CommonToken)prev);
+            } else {
+                handleDedents(-1, (CommonToken)t);
+            }
             enqueue(t);
         } else if (t.getType() == PythonLexer.NEWLINE) {
             // save NEWLINE in the queue
@@ -203,7 +207,12 @@ else if (t.getType() == PythonLexer.LEADING_WS) {
                 handleIndents(cpos, (CommonToken)t);
             }
             else if (cpos < lastIndent) { // they dedented
-                handleDedents(cpos, (CommonToken)t);
+                Token prev = stream.LT(-1);
+                if (prev != null) {
+                    handleDedents(cpos, (CommonToken)prev);
+                } else {
+                    handleDedents(cpos, (CommonToken)t);
+                }
             }
 
             if (t.getType() == Token.EOF && inSingle) {
diff --git a/src/org/python/antlr/PythonTreeAdaptor.java b/src/org/python/antlr/PythonTreeAdaptor.java
@@ -12,6 +12,8 @@ public void setTokenBoundaries(Object t, Token startToken, Token stopToken) {
         if (t==null) {
             return;
         }
+		//System.out.println("setting boundries on '"+t+"' with start: '" + startToken + "' and '" +
+        //        stopToken + "'");
         int start = 0;
         int stop = 0;
         int startChar = 0;

Original file line number	Diff line number	Diff line change
`@@ -284,6 +284,12 @@ file_input`
`284`	`284`	`List stypes = new ArrayList();`
`285`	`285`	`}`
`286`	`286`	`@after {`
	`287`	`+ if (!stypes.isEmpty()) {`
	`288`	`+ //The EOF token messes up the end offsets, so set them manually.`
	`289`	`+ PythonTree stop = (PythonTree)stypes.get(stypes.size() -1);`
	`290`	`+ mtype.setCharStopIndex(stop.getCharStopIndex());`
	`291`	`+ mtype.setTokenStopIndex(stop.getTokenStopIndex());`
	`292`	`+ }`
`287`	`293`	`$file_input.tree = mtype;`
`288`	`294`	`}`
`289`	`295`	`: (NEWLINE`
Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,8 @@ public void setTokenBoundaries(Object t, Token startToken, Token stopToken) {`
`12`	`12`	`if (t==null) {`
`13`	`13`	`return;`
`14`	`14`	`}`
	`15`	`+ //System.out.println("setting boundries on '"+t+"' with start: '" + startToken + "' and '" +`
	`16`	`+ // stopToken + "'");`
`15`	`17`	`int start = 0;`
`16`	`18`	`int stop = 0;`
`17`	`19`	`int startChar = 0;`