diff --git a/python/common/org/python/types/Closure.java b/python/common/org/python/types/Closure.java index 0f58ecb166..28501dca51 100644 --- a/python/common/org/python/types/Closure.java +++ b/python/common/org/python/types/Closure.java @@ -1,7 +1,7 @@ package org.python.types; public class Closure extends org.python.types.Object { - public java.util.Map closure_vars; + public java.util.List> locals_list; /** * A utility method to update the internal value of this object. @@ -12,9 +12,9 @@ public class Closure extends org.python.types.Object { void setValue(org.python.Object obj) { } - public Closure(java.util.Map vars) { + public Closure(java.util.List> locals_list) { super(); - this.closure_vars = vars; + this.locals_list = locals_list; } @org.python.Method( @@ -23,4 +23,8 @@ public Closure(java.util.Map vars) { public org.python.Object __repr__() { return new org.python.types.Str(String.format("", this.typeName(), this.hashCode())); } + + public java.util.Map get_locals(int level) { + return this.locals_list.get(level - 1); + } } diff --git a/python/common/org/python/types/Generator.java b/python/common/org/python/types/Generator.java index 7f1b97e78a..6edb6af069 100644 --- a/python/common/org/python/types/Generator.java +++ b/python/common/org/python/types/Generator.java @@ -5,6 +5,7 @@ public class Generator extends org.python.types.Object { java.lang.reflect.Method expression; public int yield_point; public java.util.Map stack; + public org.python.types.Closure closure; // null for non-closure Generator private boolean just_started = true; public org.python.Object message; @@ -27,6 +28,25 @@ public Generator( this.yield_point = 0; this.stack = stack; this.message = new org.python.types.NoneType(); + this.closure = null; + } + + public Generator( + java.lang.String name, + java.lang.reflect.Method expression, + java.util.Map stack, + org.python.types.Closure closure + ) { + // System.out.println("GENERATOR: " + expression); + // for (org.python.Object obj: stack) { + // System.out.println(" : " + obj); + // } + this.name = name; + this.expression = expression; + this.yield_point = 0; + this.stack = stack; + this.message = new org.python.types.NoneType(); + this.closure = closure; } public void yield(java.util.Map stack, int yield_point) { diff --git a/tests/structures/test_for.py b/tests/structures/test_for.py index 5d8ef69c06..776ce213c9 100644 --- a/tests/structures/test_for.py +++ b/tests/structures/test_for.py @@ -1,5 +1,3 @@ -from unittest import expectedFailure - from ..utils import TranspileTestCase @@ -133,25 +131,6 @@ def test_multiple_values_iterator(self): """) def test_recursive(self): - self.assertCodeExecution(""" - def process(data): - print('process: ', data) - for datum in data: - process(datum) - print('data processed: ', data) - - data = [[], [[], [], []], [[]]] - - process(data) - """, run_in_function=False) - - # FIXME: this is the same as the previous test, but the in-function - # version fails because recursive functions defined *in* a function - # don't work. Once they *do* work, the previous test can be used - # without the run_in_function=False qualifier, and this test can be - # deleted. - @expectedFailure - def test_recursive_in_function(self): self.assertCodeExecution(""" def process(data): print('process: ', data) diff --git a/tests/structures/test_nonlocal.py b/tests/structures/test_nonlocal.py new file mode 100644 index 0000000000..934bc0807c --- /dev/null +++ b/tests/structures/test_nonlocal.py @@ -0,0 +1,148 @@ +from unittest import expectedFailure + +from ..utils import TranspileTestCase + + +class NonlocalTests(TranspileTestCase): + def test_nonlocal_func(self): + self.assertCodeExecution(""" + def func(): + a = 'a from outer' + b = 'b from outer' + def nested_func(): + nonlocal a + print(a) + a = 'a from inner' + print(a) + b = 'b from inner' + print(b) + + nested_func() + print(a) + print(b) + + func() + + def func2(): + a = 'a from outer' + b = 'b from outer' + def nested_func2(): + nonlocal a + print(a) + a = 'a from inner' + print(a) + def nested_nested_func(): + nonlocal b + print(b) + b = 'b from innest' + print(b) + + nested_nested_func() + + nested_func2() + print(a) + print(b) + + func2() + """) + + self.assertCodeExecution(""" + def func(): + a = None + def nested(): + nonlocal a + a = 'changed by nested' + print(a) + + def nested2(): + print(a) + + return (nested, nested2) + + nested, nested2 = func() + nested2() + nested() + nested2() + """) + + @expectedFailure + def test_nonlocal_class(self): + self.assertCodeExecution(""" + def func(): + a = 'a from outer' + b = 'b from outer' + class Inner(): + nonlocal a + print(a) + a = 'a from inner' + b = 'b from inner' + print(a) + print(b) + + Inner() + print(a) + print(b) + + func() + """) + + @expectedFailure + def test_nonlocal_method(self): + self.assertCodeExecution(""" + def func(): + a = 'a from outer' + b = 'b from outer' + class Klass: + def method(self): + nonlocal a + print(a) + a = 'a from inner' + print(a) + print(b) + + Klass().method() + print(a) + print(b) + + # make sure closure variables are not exposed + print(hasattr(Klass(), '$closure-a')) + print(hasattr(Klass(), '$closure-b')) + + func() + """) + + def test_nonlocal_generator(self): + self.assertCodeExecution(""" + def func(): + a = 'a from outer' + b = 'b from outer' + def gen(): + nonlocal a + print(a) + print(b) + a = 'a from inner' + yield a + + print(next(gen())) + print(a) + print(b) + + func() + + def func2(): + a = 'a from outer' + b = 'b from outer' + def gen(): + nonlocal a + print(a) + print(b) + a = 'a from inner' + yield a + + print(next(gen())) + print(a) + print(b) + yield + + next(func2()) + """) diff --git a/voc/python/ast.py b/voc/python/ast.py index cec660c1d3..ed170f012d 100644 --- a/voc/python/ast.py +++ b/voc/python/ast.py @@ -863,7 +863,9 @@ def visit_Global(self, node): @node_visitor def visit_Nonlocal(self, node): # identifier* names): - raise NotImplementedError('No handler for Nonlocal') + for name in node.names: + self.context.nonlocal_vars.append(name) + self.context.local_vars.pop(name, None) @node_visitor def visit_Pass(self, node): diff --git a/voc/python/klass.py b/voc/python/klass.py index 501ce4dc1a..f685c0238a 100644 --- a/voc/python/klass.py +++ b/voc/python/klass.py @@ -13,7 +13,8 @@ ) from .blocks import Block, IgnoreBlock from .methods import ( - InitMethod, ClosureInitMethod, GeneratorMethod, Method, CO_GENERATOR + InitMethod, ClosureInitMethod, + GeneratorMethod, Method, CO_GENERATOR, ) from .types import java, python from .types.primitives import ( @@ -72,7 +73,7 @@ def module(self): return self._parent.module def store_module(self): - # Stores the current module as a local variable + # Stores the current module as a local variable if ('#module') not in self.local_vars: self.add_opcodes( JavaOpcodes.GETSTATIC('python/sys', 'modules', 'Lorg/python/types/Dict;'), @@ -253,7 +254,7 @@ def add_function(self, name, code, parameter_signatures, return_signature): generator=code.co_name, parameters=parameter_signatures, returns=return_signature, - static=True, + static=True ) else: @@ -263,7 +264,7 @@ def add_function(self, name, code, parameter_signatures, return_signature): code=code, parameters=parameter_signatures, returns=return_signature, - static=True, + static=True ) # Add the method to the list that need to be @@ -368,7 +369,7 @@ def transpile(self): class ClosureClass(Class): CONSTRUCTOR = ClosureInitMethod - def __init__(self, parent, name, closure_var_names, verbosity=0): + def __init__(self, parent, name, verbosity=0): super().__init__( parent=parent, name=name, @@ -377,4 +378,3 @@ def __init__(self, parent, name, closure_var_names, verbosity=0): verbosity=verbosity, include_default_constructor=False, ) - self.closure_var_names = closure_var_names diff --git a/voc/python/methods.py b/voc/python/methods.py index 90d63b936c..ae1f12ab0d 100644 --- a/voc/python/methods.py +++ b/voc/python/methods.py @@ -6,7 +6,7 @@ from .blocks import Block, Accumulator, BlockCodeTooLarge from .structures import ( TRY, CATCH, END_TRY, - ArgType, + ArgType, IF, END_IF ) from .types import java, python from .types.primitives import ( @@ -320,7 +320,15 @@ def store_dynamic(self): def load_name(self, name): if name in self.local_vars: self.add_opcodes( - ALOAD_name(name) + ALOAD_name('#locals'), + java.Map.get(name), + JavaOpcodes.DUP(), + IF([], JavaOpcodes.IFNONNULL), + java.THROW( + 'org/python/exceptions/UnboundLocalError', + ['Ljava/lang/String;', JavaOpcodes.LDC_W(name)] + ), + END_IF(), ) else: self.add_opcodes( @@ -345,7 +353,9 @@ def load_vars(self): def delete_name(self, name): try: self.add_opcodes( - free_name(name) + free_name(name), + ALOAD_name('#locals'), + java.Map.remove(name) ) except NameError: self.add_opcodes( @@ -392,7 +402,7 @@ def add_class(self, class_name, extends, implements): self.module, name=class_name, extends=extends, - implements=implements, + implements=implements ) self.module.classes.append(klass) @@ -453,11 +463,16 @@ def add_function(self, name, code, parameter_signatures, return_signature): klass = ClosureClass( parent=self._parent, name=name, - closure_var_names=code.co_freevars, ) self.module.classes.append(klass) klass.visitor_setup() + + if hasattr(self, "outer_contexts") and self.outer_contexts: + outer_contexts = self.outer_contexts + [self] + else: + outer_contexts = [self] + if code.co_flags & CO_GENERATOR: closure = GeneratorClosure( klass, @@ -465,6 +480,7 @@ def add_function(self, name, code, parameter_signatures, return_signature): generator=code.co_name, parameters=parameter_signatures, returns=return_signature, + outer_contexts=outer_contexts ) else: closure = Closure( @@ -472,6 +488,7 @@ def add_function(self, name, code, parameter_signatures, return_signature): code=code, parameters=parameter_signatures, returns=return_signature, + outer_contexts=outer_contexts ) klass.methods.append(closure) @@ -481,28 +498,39 @@ def add_function(self, name, code, parameter_signatures, return_signature): klass.visitor_teardown() + # closure has reference to outer context's local variables by maintaining a list of #locals self.add_opcodes( java.New(klass.descriptor), - # Define the closure vars - java.Map(), + java.List(), + JavaOpcodes.DUP(), + ALOAD_name('#locals'), + java.List.add(), ) - for var_name in code.co_freevars: + if isinstance(self, Closure): self.add_opcodes( JavaOpcodes.DUP(), - JavaOpcodes.LDC_W(var_name), + ALOAD_name(''), + JavaOpcodes.CHECKCAST('org/python/types/Closure'), + JavaOpcodes.GETFIELD('org/python/types/Closure', 'locals_list', 'Ljava/util/List;'), + java.List.addAll(), ) - self.load_name(var_name) + elif isinstance(self, GeneratorClosure): self.add_opcodes( - java.Map.put(), + JavaOpcodes.DUP(), + ALOAD_name(''), + JavaOpcodes.CHECKCAST('org/python/types/Generator'), + JavaOpcodes.GETFIELD('org/python/types/Generator', 'closure', 'Lorg/python/types/Closure;'), + JavaOpcodes.GETFIELD('org/python/types/Closure', 'locals_list', 'Ljava/util/List;'), + java.List.addAll(), ) self.add_opcodes( - java.Init(klass.descriptor, 'Ljava/util/Map;'), + java.Init(klass.descriptor, 'Ljava/util/List;'), + python.Type.for_name(klass.descriptor), ) - # Store the closure instance as an accessible symbol. self.add_callable(closure) self.add_opcodes( @@ -517,6 +545,16 @@ def visitor_setup(self): java.Map(), ASTORE_name('#locals') ) + + # stores parameters in #locals + for param in self.parameters: + self.add_opcodes( + ALOAD_name('#locals'), + JavaOpcodes.LDC_W(param['name']), + ALOAD_name(param['name']), + java.Map.put(), + ) + self.store_module() def visitor_teardown(self): @@ -604,7 +642,6 @@ def __init__(self, klass, args=None, super_args=None, parameters=None): self.store_module() - def __repr__(self): return '' % (self.klass.name, len(self.parameters)) @@ -693,6 +730,7 @@ def __init__(self, klass, name, code, parameters, returns=None, static=False): returns=returns, static=static, ) + self.store_module() def __repr__(self): @@ -874,7 +912,7 @@ def can_ignore_empty(self): def store_name(self, name, declare=False): self.add_opcodes( ASTORE_name('#value'), - ALOAD_name('#module'), # #module is available as a local var after visitor_setup has been called + ALOAD_name('#module'), # #module is available as a local var after visitor_setup has been called ALOAD_name('#value'), python.Object.set_attr(name), @@ -894,13 +932,13 @@ def store_dynamic(self): def load_name(self, name): self.add_opcodes( - ALOAD_name('#module'), # #module is available as a local var after visitor_setup has been called + ALOAD_name('#module'), # #module is available as a local var after visitor_setup has been called python.Object.get_attribute(name), ) def delete_name(self, name): self.add_opcodes( - ALOAD_name('#module'), # #module is available as a local var after visitor_setup has been called + ALOAD_name('#module'), # #module is available as a local var after visitor_setup has been called python.Object.del_attr(name), ) @@ -964,8 +1002,23 @@ def method_attributes(self): return [] +def _get_enclosing_context_level(child_context, name): + # returns level of enclosing context that defined the variable `name` + # i.e. level = 2 means `name` is found two levels up from the nested `child_context` + if name in child_context.local_vars: + return None + else: + level = 0 + for context in child_context.outer_contexts[::-1]: + level += 1 + if name in context.local_vars and context.local_vars[name] is not None: + return level + + return None + + class Closure(Function): - def __init__(self, klass, code, parameters, returns=None, static=False): + def __init__(self, klass, code, parameters, returns=None, static=False, outer_contexts=None): super().__init__( klass, name='invoke', @@ -974,12 +1027,52 @@ def __init__(self, klass, code, parameters, returns=None, static=False): returns=returns, static=static, ) + self.nonlocal_vars = [] # holds nonlocal variable names for `store_name` + self.outer_contexts = outer_contexts # parent scopes of this closure, excluding global scope + self.store_module() def __repr__(self): - return '' % ( - self.name, len(self.parameters), len(self.klass.closure_var_names) - ) + return '' % (self.name, len(self.parameters)) + + def store_name(self, name, declare=False): + if name in self.nonlocal_vars: + # updates closure + self.add_opcodes( + ALOAD_name(''), + JavaOpcodes.CHECKCAST('org/python/types/Closure'), + ICONST_val(_get_enclosing_context_level(self, name)), + JavaOpcodes.INVOKEVIRTUAL( + 'org/python/types/Closure', + 'get_locals', + args=['I'], + returns='Ljava/util/Map;' + ), + JavaOpcodes.SWAP(), + JavaOpcodes.LDC_W(name), + JavaOpcodes.SWAP(), + java.Map.put() + ) + else: + super().store_name(name, declare) + + def load_name(self, name): + parent_level = _get_enclosing_context_level(self, name) + if parent_level: + self.add_opcodes( + ALOAD_name(''), + JavaOpcodes.CHECKCAST('org/python/types/Closure'), + ICONST_val(parent_level), + JavaOpcodes.INVOKEVIRTUAL( + 'org/python/types/Closure', + 'get_locals', + args=['I'], + returns='Ljava/util/Map;' + ), + java.Map.get(name), + ) + else: + super().load_name(name) @property def klass(self): @@ -1001,25 +1094,6 @@ def add_self(self): self.local_vars[''] = len(self.local_vars) self.has_self = True - def load_name(self, name): - if name in self.local_vars: - self.add_opcodes( - ALOAD_name(name) - ) - elif name in self.klass.closure_var_names: - self.add_opcodes( - ALOAD_name(''), - JavaOpcodes.CHECKCAST('org/python/types/Closure'), - JavaOpcodes.GETFIELD('org/python/types/Closure', 'closure_vars', 'Ljava/util/Map;'), - - java.Map.get(name), - ) - else: - self.add_opcodes( - ALOAD_name('#module'), - python.Object.get_attribute(name), - ) - class ClosureInitMethod(InitMethod): def __init__(self, klass): @@ -1044,12 +1118,12 @@ def __repr__(self): @property def signature(self): - return '(Ljava/util/Map;)V' + return '(Ljava/util/List;)V' def visitor_teardown(self): self.add_opcodes( JavaOpcodes.ALOAD_1(), - java.Init(self.klass.extends_descriptor, 'Ljava/util/Map;'), + java.Init(self.klass.extends_descriptor, 'Ljava/util/List;'), JavaOpcodes.RETURN() ) @@ -1150,16 +1224,33 @@ def transpile_wrapper(self): java.Map.put(), ) - # Construct and return the generator object. - wrapper.add_opcodes( - java.Init( - 'org/python/types/Generator', - 'Ljava/lang/String;', - 'Ljava/lang/reflect/Method;', - 'Ljava/util/Map;', - ), - JavaOpcodes.ARETURN(), - ) + if isinstance(self, GeneratorClosure): + # stores a copy of closure variables + wrapper.add_opcodes( + JavaOpcodes.ALOAD_0(), # first register contains initialized Closure object reference + JavaOpcodes.CHECKCAST('org/python/types/Closure') + ) + wrapper.add_opcodes( + java.Init( + 'org/python/types/Generator', + 'Ljava/lang/String;', + 'Ljava/lang/reflect/Method;', + 'Ljava/util/Map;', + 'Lorg/python/types/Closure;' + ), + JavaOpcodes.ARETURN(), + ) + else: + # Construct and return the generator object. + wrapper.add_opcodes( + java.Init( + 'org/python/types/Generator', + 'Ljava/lang/String;', + 'Ljava/lang/reflect/Method;', + 'Ljava/util/Map;' + ), + JavaOpcodes.ARETURN(), + ) return [ JavaMethod( @@ -1209,10 +1300,15 @@ def store_name(self, name, declare=False): ) def load_name(self, name): - if name in self.local_vars: + if name == '': # `` is not included in #locals self.add_opcodes( ALOAD_name(name) ) + elif name in self.local_vars: + self.add_opcodes( + ALOAD_name('#locals'), + java.Map.get(name), + ) else: # Unlike other Functions, GeneratorFunctions do not cache the current Module # locally, so it must be fetched on each use. @@ -1229,7 +1325,9 @@ def load_name(self, name): def delete_name(self, name): try: self.add_opcodes( - free_name(name) + free_name(name), + ALOAD_name('#locals'), + java.Map.remove(name) ) except NameError: # Unlike other Functions, GeneratorFunctions do not cache the current Module @@ -1244,6 +1342,7 @@ def delete_name(self, name): python.Object.del_attr(name), ) + class GeneratorMethod(Method): def __init__(self, klass, name, code, generator, parameters, returns=None, static=False): super().__init__( @@ -1365,9 +1464,17 @@ def transpile_wrapper(self): ) ] + def load_name(self, name): + if name == '': # `` is not included in #locals + self.add_opcodes( + ALOAD_name(name) + ) + else: + super().load_name(name) + class GeneratorClosure(GeneratorFunction): - def __init__(self, module, code, generator, parameters, returns=None, static=False): + def __init__(self, module, code, generator, parameters, returns=None, static=False, outer_contexts=None): super().__init__( module, name='invoke', @@ -1377,6 +1484,8 @@ def __init__(self, module, code, generator, parameters, returns=None, static=Fal returns=returns, static=static, ) + self.nonlocal_vars = [] # holds nonlocal variable names for `store_name` + self.outer_contexts = outer_contexts # parent scopes of this closure, excluding global scope def __repr__(self): return '' % ( @@ -1394,3 +1503,45 @@ def module(self): @property def class_descriptor(self): return self.klass.descriptor + + def store_name(self, name, declare=False): + if name in self.nonlocal_vars: + # updates closure + self.add_opcodes( + ALOAD_name(''), + JavaOpcodes.CHECKCAST('org/python/types/Generator'), + JavaOpcodes.GETFIELD('org/python/types/Generator', 'closure', 'Lorg/python/types/Closure;'), + ICONST_val(_get_enclosing_context_level(self, name)), + JavaOpcodes.INVOKEVIRTUAL( + 'org/python/types/Closure', + 'get_locals', + args=['I'], + returns='Ljava/util/Map;' + ), + JavaOpcodes.SWAP(), + JavaOpcodes.LDC_W(name), + JavaOpcodes.SWAP(), + java.Map.put() + ) + else: + super().store_name(name, declare) + + def load_name(self, name): + parent_level = _get_enclosing_context_level(self, name) + + if parent_level: + self.add_opcodes( + ALOAD_name(''), + JavaOpcodes.CHECKCAST('org/python/types/Generator'), + JavaOpcodes.GETFIELD('org/python/types/Generator', 'closure', 'Lorg/python/types/Closure;'), + ICONST_val(parent_level), + JavaOpcodes.INVOKEVIRTUAL( + 'org/python/types/Closure', + 'get_locals', + args=['I'], + returns='Ljava/util/Map;' + ), + java.Map.get(name), + ) + else: + super().load_name(name) diff --git a/voc/python/modules.py b/voc/python/modules.py index f18c07a316..91b062a96d 100644 --- a/voc/python/modules.py +++ b/voc/python/modules.py @@ -78,7 +78,7 @@ def build_child_class_descriptor(self, child_name): return '/'.join([self.descriptor, child_name]) def store_module(self): - # Stores the current module as a local variable + # Stores the current module as a local variable if ('#module') not in self.local_vars: self.add_opcodes( JavaOpcodes.GETSTATIC('python/sys', 'modules', 'Lorg/python/types/Dict;'), diff --git a/voc/python/types/java.py b/voc/python/types/java.py index d658864932..49b445de55 100644 --- a/voc/python/types/java.py +++ b/voc/python/types/java.py @@ -119,6 +119,18 @@ def process(self, context): JavaOpcodes.POP(), ) + class addAll: + def process(self, context): + context.add_opcodes( + JavaOpcodes.INVOKEINTERFACE( + 'java/util/List', + 'addAll', + args=['Ljava/util/Collection;'], + returns='Z' + ), + JavaOpcodes.POP(), + ) + class Map: def process(self, context): @@ -166,6 +178,22 @@ def process(self, context): ), ) + class remove: + def __init__(self, key): + self.key = key + + def process(self, context): + context.add_opcodes( + JavaOpcodes.LDC_W(self.key), + JavaOpcodes.INVOKEINTERFACE( + 'java/util/Map', + 'remove', + args=['Ljava/lang/Object;'], + returns='Ljava/lang/Object;' + ), + JavaOpcodes.POP() + ) + class Class: class forName: