In the parser token stream, If a string contains unicodes, and

Nick Santos · hns · commit 323f151a2321 · 2012-05-14T09:20:48.000+02:00
converted to a reserved keyword, we convert the last character back to unicode. Fixes issue 490. Committed by Yi Zhu ------------- Created by MOE: http://code.google.com/p/moe-java MOE_MIGRATED_REVID=26371770
diff --git a/src/org/mozilla/javascript/TokenStream.java b/src/org/mozilla/javascript/TokenStream.java
@@ -156,7 +156,7 @@ private static int stringToKeyword(String name)
             Id_byte          = Token.RESERVED,  // ES3 only
             Id_catch         = Token.CATCH,
             Id_char          = Token.RESERVED,  // ES3 only
-            Id_class         = Token.RESERVED,  
+            Id_class         = Token.RESERVED,
             Id_const         = Token.CONST,     // reserved
             Id_debugger      = Token.DEBUGGER,
             Id_double        = Token.RESERVED,  // ES3 only
@@ -183,7 +183,7 @@ private static int stringToKeyword(String name)
             Id_synchronized  = Token.RESERVED,  // ES3 only
             Id_throw         = Token.THROW,
             Id_throws        = Token.RESERVED,  // ES3 only
-            Id_transient     = Token.RESERVED,  // ES3 only  
+            Id_transient     = Token.RESERVED,  // ES3 only
             Id_try           = Token.TRY,
             Id_volatile      = Token.RESERVED;  // ES3 only
 
@@ -428,7 +428,7 @@ final int getToken() throws IOException
                         }
                         // Save the string in case we need to use in
                         // object literal definitions.
-                        this.string = (String)allStrings.intern(str);                        
+                        this.string = (String)allStrings.intern(str);
                         if (result != Token.RESERVED) {
                             return result;
                         } else if (!parser.compilerEnv.
@@ -437,6 +437,10 @@ final int getToken() throws IOException
                             return result;
                         }
                     }
+                } else if (isKeyword(str)) {
+                    // If a string contains unicodes, and converted to a keyword,
+                    // we convert the last character back to unicode
+                    str = convertLastCharToHex(str);
                 }
                 this.string = (String)allStrings.intern(str);
                 return Token.NAME;
@@ -1568,6 +1572,19 @@ final String getAndResetCurrentComment() {
         }
     }
 
+    private String convertLastCharToHex(String str) {
+      int lastIndex = str.length()-1;
+      StringBuffer buf = new StringBuffer(
+          str.substring(0, lastIndex));
+      buf.append("\\u");
+      String hexCode = Integer.toHexString(str.charAt(lastIndex));
+      for (int i = 0; i < 4-hexCode.length(); ++i) {
+        buf.append('0');
+      }
+      buf.append(hexCode);
+      return buf.toString();
+    }
+
     // stuff other than whitespace since start of line
     private boolean dirtyLine;
 
diff --git a/testsrc/org/mozilla/javascript/tests/ParserTest.java b/testsrc/org/mozilla/javascript/tests/ParserTest.java
@@ -1083,6 +1083,35 @@ public void testParseUnicodeFormatName() {
         assertEquals("AB", first.getString());
     }
 
+    public void testParseUnicodeReservedKeywords1() {
+        AstRoot root = parse("\\u0069\\u0066");
+        AstNode first = ((ExpressionStatement)
+            root.getFirstChild()).getExpression();
+        assertEquals("i\\u0066", first.getString());
+    }
+
+    public void testParseUnicodeReservedKeywords2() {
+        AstRoot root = parse("v\\u0061\\u0072");
+        AstNode first = ((ExpressionStatement)
+            root.getFirstChild()).getExpression();
+        assertEquals("va\\u0072", first.getString());
+    }
+
+    public void testParseUnicodeReservedKeywords3() {
+        // All are keyword "while"
+        AstRoot root = parse("w\\u0068\\u0069\\u006C\\u0065;" +
+            "\\u0077\\u0068il\\u0065; \\u0077h\\u0069le;");
+        AstNode first = ((ExpressionStatement)
+            root.getFirstChild()).getExpression();
+        AstNode second = ((ExpressionStatement)
+            root.getFirstChild().getNext()).getExpression();
+        AstNode third = ((ExpressionStatement)
+            root.getFirstChild().getNext().getNext()).getExpression();
+        assertEquals("whil\\u0065", first.getString());
+        assertEquals("whil\\u0065", second.getString());
+        assertEquals("whil\\u0065", third.getString());
+    }
+
     public void testParseObjectLiteral1() {
       allowKeywordsAsObjectLiteralsKeys = true;
 
