Code Python 3.7 compiler - LOAD_METHOD/CALL_METHOD

DinoV · DinoV · commit 9f8cc2e81b96 · 2021-01-28T11:14:38.000-08:00
Summary: First step towards having a compiler that supports Python 3.7 - we now have a code generator which subclasses the normal one, and modifies the code we generate for calling methods.  In order to support running/testing on Python 3.6 we need to define the new opcodes manually.

Test Plan: ./python -m test.test_compiler
diff --git a/compiler/pyassem.py b/compiler/pyassem.py
@@ -38,6 +38,17 @@ def cast_signed_byte_to_unsigned(i):
 FVS_MASK      = 0x4
 FVS_HAVE_SPEC = 0x4
 
+OPNAMES = list(dis.opname)
+OPMAP = dict(dis.opmap)
+
+if "LOAD_METHOD" not in OPMAP:
+    OPMAP["LOAD_METHOD"] = 160
+    OPNAMES[160] = "LOAD_METHOD"
+
+if "CALL_METHOD" not in OPMAP:
+    OPMAP["CALL_METHOD"] = 161
+    OPNAMES[161] = "CALL_METHOD"
+
 
 class Instruction:
     __slots__ = ('opname', 'oparg', 'target')
@@ -465,10 +476,10 @@ def flattenGraph(self):
 
     hasjrel = set()
     for i in dis.hasjrel:
-        hasjrel.add(dis.opname[i])
+        hasjrel.add(OPNAMES[i])
     hasjabs = set()
     for i in dis.hasjabs:
-        hasjabs.add(dis.opname[i])
+        hasjabs.add(OPNAMES[i])
 
     def convertArgs(self):
         """Convert arguments from symbolic to concrete form"""
@@ -545,6 +556,7 @@ def _convert_NAME(self, arg):
     _convert_STORE_ATTR = _convert_NAME
     _convert_LOAD_ATTR = _convert_NAME
     _convert_DELETE_ATTR = _convert_NAME
+    _convert_LOAD_METHOD = _convert_NAME
 
     def _convert_GLOBAL(self, arg):
         return self._lookupName(arg, self.names)
@@ -598,18 +610,13 @@ def makeByteCode(self):
                     lnotab.addCode(EXTENDED_ARG, (oparg >> 16) & 0xff)
                 if oparg > 0xff:
                     lnotab.addCode(EXTENDED_ARG, (oparg >> 8) & 0xff)
-                lnotab.addCode(self.opnum[t.opname], oparg & 0xff)
+                lnotab.addCode(OPMAP[t.opname], oparg & 0xff)
             except ValueError:
                 print(t.opname, oparg)
-                print(self.opnum[t.opname], oparg)
+                print(OPMAP[t.opname], oparg)
                 raise
         self.stage = DONE
 
-    opnum = {}
-    for num in range(len(dis.opname)):
-        opnum[dis.opname[num]] = num
-    del num
-
     def newCodeObject(self):
         assert self.stage == DONE
         if (self.flags & CO_NEWLOCALS) == 0:
@@ -867,4 +874,6 @@ def getTable(self):
     # else 1->1.
     FORMAT_VALUE = lambda oparg: -1 if (oparg & FVS_MASK) == FVS_HAVE_SPEC else 0,
     SET_LINENO = 0,
+    LOAD_METHOD = 1,
+    CALL_METHOD = lambda oparg: -oparg - 1,
 )
diff --git a/compiler/pycodegen.py b/compiler/pycodegen.py
@@ -1937,6 +1937,23 @@ def flow_graph(cls, name, filename, args=(), kwonlyargs=(), starargs=(), optimiz
         return pyassem.PyFlowGraph(name, filename, args, kwonlyargs, starargs, optimized, klass, peephole_enabled=False)
 
 
+class Python37CodeGenerator(CodeGenerator):
+    def visitCall(self, node):
+        if (node.keywords or
+            not isinstance(node.func, ast.Attribute) or
+            not isinstance(node.func.ctx, ast.Load) or
+            any(isinstance(arg, ast.Starred) for arg in node.args)):
+            # We cannot optimize this call
+            return super().visitCall(node)
+
+        self.update_lineno(node)
+        self.visit(node.func.value)
+        self.emit('LOAD_METHOD', self.mangle(node.func.attr))
+        for arg in node.args:
+            self.visit(arg)
+        self.emit('CALL_METHOD', len(node.args))
+
+
 def get_docstring(node):
     if node.body and isinstance(node.body[0], ast.Expr) \
        and isinstance(node.body[0].value, ast.Str):
diff --git a/test_compiler/__init__.py b/test_compiler/__init__.py
@@ -2,6 +2,7 @@
 from .test_code_sbs import CodeTests
 from .test_errors import ErrorTests, ErrorTestsBuiltin
 from .test_peephole import PeepHoleTests
+from .test_py37 import Python37Tests
 from .test_flags import FlagTests
 from .test_graph import GraphTests
 from .test_sbs_stdlib import SbsCompileTests
diff --git a/test_compiler/__main__.py b/test_compiler/__main__.py
@@ -4,6 +4,7 @@
 from .test_peephole import PeepHoleTests
 from .test_flags import FlagTests
 from .test_graph import GraphTests
+from .test_py37 import Python37Tests
 from .test_sbs_stdlib import SbsCompileTests
 from .test_errors import ErrorTests, ErrorTestsBuiltin
 from .test_symbols import SymbolVisitorTests
diff --git a/test_compiler/test_py37.py b/test_compiler/test_py37.py
@@ -0,0 +1,34 @@
+import dis
+from .common import CompilerTest
+from compiler.pycodegen import Python37CodeGenerator
+
+
+LOAD_METHOD = "LOAD_METHOD"
+if LOAD_METHOD not in dis.opmap:
+    LOAD_METHOD = "<160>"
+
+CALL_METHOD = "CALL_METHOD"
+if CALL_METHOD not in dis.opmap:
+    CALL_METHOD = "<161>"
+
+class Python37Tests(CompilerTest):
+    def test_compile_method(self):
+        code = self.compile('x.f()', Python37CodeGenerator)
+        self.assertInBytecode(code, LOAD_METHOD)
+        self.assertInBytecode(code, CALL_METHOD, 0)
+
+        code = self.compile('x.f(42)', Python37CodeGenerator)
+        self.assertInBytecode(code, LOAD_METHOD)
+        self.assertInBytecode(code, CALL_METHOD, 1)
+
+    def test_compile_method_varargs(self):
+        code = self.compile('x.f(*foo)', Python37CodeGenerator)
+        self.assertNotInBytecode(code, LOAD_METHOD)
+
+    def test_compile_method_kwarg(self):
+        code = self.compile('x.f(kwarg=1)', Python37CodeGenerator)
+        self.assertNotInBytecode(code, LOAD_METHOD)
+
+    def test_compile_method_normal(self):
+        code = self.compile('f()', Python37CodeGenerator)
+        self.assertNotInBytecode(code, LOAD_METHOD)
