From b117e5e076655d541571a1acda785f1812a349e4 Mon Sep 17 00:00:00 2001 From: synodriver Date: Sat, 2 Apr 2022 13:26:52 +0800 Subject: [PATCH 01/15] add nested dict support --- lupa/_lupa.pyx | 51 ++++--- lupa/tests/test.py | 369 +++++++++++++++++++++++++++++---------------- setup.py | 7 +- 3 files changed, 281 insertions(+), 146 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 9af27175..6e085afe 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -48,10 +48,11 @@ cdef object exc_info from sys import exc_info cdef object Mapping +cdef object Sequence try: - from collections.abc import Mapping + from collections.abc import Mapping, Sequence except ImportError: - from collections import Mapping # Py2 + from collections import Mapping, Sequence # Py2 cdef object wraps from functools import wraps @@ -408,7 +409,7 @@ cdef class LuaRuntime: """ return self.table_from(items, kwargs) - def table_from(self, *args): + def table_from(self, *args, bint recursive=False): """Create a new table from Python mapping or iterable. table_from() accepts either a dict/mapping or an iterable with items. @@ -416,7 +417,7 @@ cdef class LuaRuntime: are placed in the table in order. Nested mappings / iterables are passed to Lua as userdata - (wrapped Python objects); they are not converted to Lua tables. + (wrapped Python objects) if recursive is False; they are not converted to Lua tables. """ assert self._state is not NULL cdef lua_State *L = self._state @@ -426,12 +427,12 @@ cdef class LuaRuntime: try: check_lua_stack(L, 5) lua.lua_newtable(L) - # FIXME: how to check for failure? + # FIXME: how to check for failure? and nested dict for obj in args: if isinstance(obj, dict): - for key, value in obj.iteritems(): - py_to_lua(self, L, key) - py_to_lua(self, L, value) + for key, value in obj.iteritems(): # in python3, this is called items + py_to_lua(self, L, key, False, recursive) + py_to_lua(self, L, value, False, recursive) lua.lua_rawset(L, -3) elif isinstance(obj, _LuaTable): @@ -447,12 +448,12 @@ cdef class LuaRuntime: elif isinstance(obj, Mapping): for key in obj: value = obj[key] - py_to_lua(self, L, key) - py_to_lua(self, L, value) + py_to_lua(self, L, key, False, recursive) + py_to_lua(self, L, value, False, recursive) lua.lua_rawset(L, -3) else: for arg in obj: - py_to_lua(self, L, arg) + py_to_lua(self, L, arg, False, recursive) lua.lua_rawseti(L, -2, i) i += 1 return py_from_lua(self, L, -1) @@ -508,12 +509,12 @@ cdef class LuaRuntime: luaL_openlib(L, "python", py_lib, 0) # lib lua.lua_pushlightuserdata(L, self) # lib udata lua.lua_pushcclosure(L, py_args, 1) # lib function - lua.lua_setfield(L, -2, "args") # lib + lua.lua_setfield(L, -2, "args") # lib # register our own object metatable lua.luaL_newmetatable(L, POBJECT) # lib metatbl luaL_openlib(L, NULL, py_object_lib, 0) - lua.lua_pop(L, 1) # lib + lua.lua_pop(L, 1) # lib # create and store the python references table lua.lua_newtable(L) # lib tbl @@ -1135,7 +1136,7 @@ cdef object resume_lua_thread(_LuaThread thread, tuple args): # already terminated raise StopIteration if args: - nargs = len(args) + nargs = len(args) push_lua_arguments(thread._runtime, co, args) with nogil: status = lua.lua_resume(co, L, nargs, &nres) @@ -1381,7 +1382,7 @@ cdef py_object* unpack_userdata(lua_State *L, int n) nogil: cdef int py_function_result_to_lua(LuaRuntime runtime, lua_State *L, object o) except -1: if runtime._unpack_returned_tuples and isinstance(o, tuple): push_lua_arguments(runtime, L, o) - return len(o) + return len(o) check_lua_stack(L, 1) return py_to_lua(runtime, L, o) @@ -1410,7 +1411,7 @@ cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) e lua.lua_settop(L, old_top) raise -cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False) except -1: +cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False, bint recursive=False) except -1: """Converts Python object to Lua Preconditions: 1 extra slot in the Lua stack @@ -1462,6 +1463,19 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa elif isinstance(o, float): lua.lua_pushnumber(L, o) pushed_values_count = 1 + elif recursive and isinstance(o, Sequence): + lua.lua_createtable(L, len(o), 0) # create a table at the top of stack, with narr already known + for i, v in enumerate(o): + py_to_lua(runtime, L, v, wrap_none, recursive) + lua.lua_rawseti(L, -2, i+1) + pushed_values_count = 1 + elif recursive and isinstance(o, Mapping): + lua.lua_createtable(L, 0, len(o)) # create a table at the top of stack, with nrec already known + for key, value in o.iteritems(): # to compatible with py2 + py_to_lua(runtime, L, key, wrap_none, recursive) + py_to_lua(runtime, L, value, wrap_none, recursive) + lua.lua_rawset(L, -3) + pushed_values_count = 1 else: if isinstance(o, _PyProtocolWrapper): type_flags = (<_PyProtocolWrapper>o)._type_flags @@ -1626,7 +1640,7 @@ cdef object execute_lua_call(LuaRuntime runtime, lua_State *L, Py_ssize_t nargs) lua.lua_replace(L, -2) lua.lua_insert(L, 1) errfunc = 1 - result_status = lua.lua_pcall(L, nargs, lua.LUA_MULTRET, errfunc) + result_status = lua.lua_pcall(L, nargs, lua.LUA_MULTRET, errfunc) if errfunc: lua.lua_remove(L, 1) results = unpack_lua_results(runtime, L) @@ -1736,7 +1750,7 @@ cdef int py_object_gc_with_gil(py_object *py_obj, lua_State* L) with gil: return 0 finally: py_obj.obj = NULL - + cdef int py_object_gc(lua_State* L) nogil: if not lua.lua_isuserdata(L, 1): return 0 @@ -1762,7 +1776,6 @@ cdef bint call_python(LuaRuntime runtime, lua_State *L, py_object* py_obj) excep else: args = () kwargs = {} - for i in range(nargs): arg = py_from_lua(runtime, L, i+2) if isinstance(arg, _PyArguments): diff --git a/lupa/tests/test.py b/lupa/tests/test.py index d8706919..e95d3780 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -24,6 +24,7 @@ def _next(o): if IS_PYTHON2: unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp + class SetupLuaRuntimeMixin(object): lua_runtime_kwargs = {} @@ -97,7 +98,7 @@ def test_eval(self): self.assertEqual(2, self.lua.eval('1+1')) def test_eval_multi(self): - self.assertEqual((1,2,3), self.lua.eval('1,2,3')) + self.assertEqual((1, 2, 3), self.lua.eval('1,2,3')) def test_eval_args(self): self.assertEqual(2, self.lua.eval('...', 2)) @@ -170,7 +171,7 @@ def test_recursive_function(self): return fac ''') self.assertNotEqual(None, fac) - self.assertEqual(6, fac(3)) + self.assertEqual(6, fac(3)) self.assertEqual(3628800, fac(10)) def test_double_recursive_function(self): @@ -185,8 +186,8 @@ def test_double_recursive_function(self): ''' calc = self.lua.execute(func_code) self.assertNotEqual(None, calc) - self.assertEqual(3, calc(3)) - self.assertEqual(109, calc(10)) + self.assertEqual(3, calc(3)) + self.assertEqual(109, calc(10)) self.assertEqual(13529, calc(20)) def test_double_recursive_function_pycallback(self): @@ -199,14 +200,15 @@ def test_double_recursive_function_pycallback(self): end return calc ''' + def pycallback(i): - return i**2 + return i ** 2 calc = self.lua.execute(func_code) self.assertNotEqual(None, calc) - self.assertEqual(12, calc(pycallback, 3)) - self.assertEqual(1342, calc(pycallback, 10)) + self.assertEqual(12, calc(pycallback, 3)) + self.assertEqual(1342, calc(pycallback, 10)) self.assertEqual(185925, calc(pycallback, 20)) def test_none(self): @@ -243,6 +245,7 @@ def test_call_str_py(self): def test_call_str_class(self): called = [False] + class test(object): def __str__(self): called[0] = True @@ -267,7 +270,7 @@ def test_len_table_array(self): def test_len_table_dict(self): table = self.lua.eval('{a=1, b=2, c=3}') - self.assertEqual(0, len(table)) # as returned by Lua's "#" operator + self.assertEqual(0, len(table)) # as returned by Lua's "#" operator def test_table_delattr(self): table = self.lua.eval('{a=1, b=2, c=3}') @@ -293,27 +296,27 @@ def test_len_table(self): def test_iter_table(self): table = self.lua.eval('{2,3,4,5,6}') - self.assertEqual([1,2,3,4,5], list(table)) + self.assertEqual([1, 2, 3, 4, 5], list(table)) def test_iter_table_list_repeat(self): table = self.lua.eval('{2,3,4,5,6}') - self.assertEqual([1,2,3,4,5], list(table)) # 1 - self.assertEqual([1,2,3,4,5], list(table)) # 2 - self.assertEqual([1,2,3,4,5], list(table)) # 3 + self.assertEqual([1, 2, 3, 4, 5], list(table)) # 1 + self.assertEqual([1, 2, 3, 4, 5], list(table)) # 2 + self.assertEqual([1, 2, 3, 4, 5], list(table)) # 3 def test_iter_array_table_values(self): table = self.lua.eval('{2,3,4,5,6}') - self.assertEqual([2,3,4,5,6], list(table.values())) + self.assertEqual([2, 3, 4, 5, 6], list(table.values())) def test_iter_array_table_repeat(self): table = self.lua.eval('{2,3,4,5,6}') - self.assertEqual([2,3,4,5,6], list(table.values())) # 1 - self.assertEqual([2,3,4,5,6], list(table.values())) # 2 - self.assertEqual([2,3,4,5,6], list(table.values())) # 3 + self.assertEqual([2, 3, 4, 5, 6], list(table.values())) # 1 + self.assertEqual([2, 3, 4, 5, 6], list(table.values())) # 2 + self.assertEqual([2, 3, 4, 5, 6], list(table.values())) # 3 def test_iter_multiple_tables(self): count = 10 - table_values = [self.lua.eval('{%s}' % ','.join(map(str, range(2, count+2)))).values() + table_values = [self.lua.eval('{%s}' % ','.join(map(str, range(2, count + 2)))).values() for _ in range(4)] # round robin @@ -322,11 +325,11 @@ def test_iter_multiple_tables(self): for table in table_values: sublist.append(_next(table)) - self.assertEqual([[i]*len(table_values) for i in range(2, count+2)], l) + self.assertEqual([[i] * len(table_values) for i in range(2, count + 2)], l) def test_iter_table_repeat(self): count = 10 - table_values = [self.lua.eval('{%s}' % ','.join(map(str, range(2, count+2)))).values() + table_values = [self.lua.eval('{%s}' % ','.join(map(str, range(2, count + 2)))).values() for _ in range(4)] # one table after the other @@ -335,7 +338,7 @@ def test_iter_table_repeat(self): for sublist in l: sublist.append(_next(table)) - self.assertEqual([[i]*len(table_values) for i in range(2,count+2)], l) + self.assertEqual([[i] * len(table_values) for i in range(2, count + 2)], l) def test_iter_table_refcounting(self): lua_func = self.lua.eval(''' @@ -387,20 +390,20 @@ def test_iter_table_values_int_keys(self): table = self.lua.eval('{%s}' % ','.join('[%d]=%d' % (i, -i) for i in range(10))) l = list(table.values()) l.sort() - self.assertEqual(list(range(-9,1)), l) + self.assertEqual(list(range(-9, 1)), l) def test_iter_table_items(self): keys = list('abcdefg') table = self.lua.eval('{%s}' % ','.join('%s=%d' % (c, i) for i, c in enumerate(keys))) l = list(table.items()) l.sort() - self.assertEqual(list(zip(keys,range(len(keys)))), l) + self.assertEqual(list(zip(keys, range(len(keys)))), l) def test_iter_table_items_int_keys(self): table = self.lua.eval('{%s}' % ','.join('[%d]=%d' % (i, -i) for i in range(10))) l = list(table.items()) l.sort() - self.assertEqual(list(zip(range(10), range(0,-10,-1))), l) + self.assertEqual(list(zip(range(10), range(0, -10, -1))), l) def test_iter_table_values_mixed(self): keys = list('abcdefg') @@ -433,7 +436,7 @@ def test_string_values(self): def test_int_values(self): function = self.lua.eval('function(i) return i + 5 end') - self.assertEqual(3+5, function(3)) + self.assertEqual(3 + 5, function(3)) def test_long_values(self): try: @@ -441,11 +444,11 @@ def test_long_values(self): except NameError: _long = int function = self.lua.eval('function(i) return i + 5 end') - self.assertEqual(3+5, function(_long(3))) + self.assertEqual(3 + 5, function(_long(3))) def test_float_values(self): function = self.lua.eval('function(i) return i + 5 end') - self.assertEqual(float(3)+5, function(float(3))) + self.assertEqual(float(3) + 5, function(float(3))) def test_str_function(self): func = self.lua.eval('function() return 1 end') @@ -456,7 +459,7 @@ def test_str_table(self): self.assertEqual(' Date: Fri, 8 Apr 2022 11:30:16 +0800 Subject: [PATCH 02/15] revert ide's change --- lupa/_lupa.pyx | 26 ++-- lupa/tests/test.py | 329 +++++++++++++++++++-------------------------- 2 files changed, 156 insertions(+), 199 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 6e085afe..46b3c103 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -157,6 +157,10 @@ def lua_type(obj): lua.lua_settop(L, old_top) unlock_runtime(lua_object._runtime) +cdef int _len_as_int(Py_ssize_t obj) except -1: + if obj > LONG_MAX: + raise OverflowError + return obj @cython.no_gc_clear cdef class LuaRuntime: @@ -430,9 +434,9 @@ cdef class LuaRuntime: # FIXME: how to check for failure? and nested dict for obj in args: if isinstance(obj, dict): - for key, value in obj.iteritems(): # in python3, this is called items - py_to_lua(self, L, key, False, recursive) - py_to_lua(self, L, value, False, recursive) + for key, value in obj.iteritems(): + py_to_lua(self, L, key, wrap_none=False, recursive=recursive) + py_to_lua(self, L, value, wrap_none=False, recursive=recursive) lua.lua_rawset(L, -3) elif isinstance(obj, _LuaTable): @@ -1136,6 +1140,8 @@ cdef object resume_lua_thread(_LuaThread thread, tuple args): # already terminated raise StopIteration if args: + if len(args) > LONG_MAX: + raise OverflowError nargs = len(args) push_lua_arguments(thread._runtime, co, args) with nogil: @@ -1463,6 +1469,10 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa elif isinstance(o, float): lua.lua_pushnumber(L, o) pushed_values_count = 1 + elif isinstance(o, _PyProtocolWrapper): + type_flags = (<_PyProtocolWrapper> o)._type_flags + o = (<_PyProtocolWrapper> o)._obj + pushed_values_count = py_to_lua_custom(runtime, L, o, type_flags) elif recursive and isinstance(o, Sequence): lua.lua_createtable(L, len(o), 0) # create a table at the top of stack, with narr already known for i, v in enumerate(o): @@ -1471,18 +1481,14 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa pushed_values_count = 1 elif recursive and isinstance(o, Mapping): lua.lua_createtable(L, 0, len(o)) # create a table at the top of stack, with nrec already known - for key, value in o.iteritems(): # to compatible with py2 + for key, value in o.items(): py_to_lua(runtime, L, key, wrap_none, recursive) py_to_lua(runtime, L, value, wrap_none, recursive) lua.lua_rawset(L, -3) pushed_values_count = 1 else: - if isinstance(o, _PyProtocolWrapper): - type_flags = (<_PyProtocolWrapper>o)._type_flags - o = (<_PyProtocolWrapper>o)._obj - else: - # prefer __getitem__ over __getattr__ by default - type_flags = OBJ_AS_INDEX if hasattr(o, '__getitem__') else 0 + # prefer __getitem__ over __getattr__ by default + type_flags = OBJ_AS_INDEX if hasattr(o, '__getitem__') else 0 pushed_values_count = py_to_lua_custom(runtime, L, o, type_flags) return pushed_values_count diff --git a/lupa/tests/test.py b/lupa/tests/test.py index e95d3780..da3734d9 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -24,7 +24,6 @@ def _next(o): if IS_PYTHON2: unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp - class SetupLuaRuntimeMixin(object): lua_runtime_kwargs = {} @@ -98,7 +97,7 @@ def test_eval(self): self.assertEqual(2, self.lua.eval('1+1')) def test_eval_multi(self): - self.assertEqual((1, 2, 3), self.lua.eval('1,2,3')) + self.assertEqual((1,2,3), self.lua.eval('1,2,3')) def test_eval_args(self): self.assertEqual(2, self.lua.eval('...', 2)) @@ -171,7 +170,7 @@ def test_recursive_function(self): return fac ''') self.assertNotEqual(None, fac) - self.assertEqual(6, fac(3)) + self.assertEqual(6, fac(3)) self.assertEqual(3628800, fac(10)) def test_double_recursive_function(self): @@ -186,8 +185,8 @@ def test_double_recursive_function(self): ''' calc = self.lua.execute(func_code) self.assertNotEqual(None, calc) - self.assertEqual(3, calc(3)) - self.assertEqual(109, calc(10)) + self.assertEqual(3, calc(3)) + self.assertEqual(109, calc(10)) self.assertEqual(13529, calc(20)) def test_double_recursive_function_pycallback(self): @@ -200,15 +199,14 @@ def test_double_recursive_function_pycallback(self): end return calc ''' - def pycallback(i): - return i ** 2 + return i**2 calc = self.lua.execute(func_code) self.assertNotEqual(None, calc) - self.assertEqual(12, calc(pycallback, 3)) - self.assertEqual(1342, calc(pycallback, 10)) + self.assertEqual(12, calc(pycallback, 3)) + self.assertEqual(1342, calc(pycallback, 10)) self.assertEqual(185925, calc(pycallback, 20)) def test_none(self): @@ -245,7 +243,6 @@ def test_call_str_py(self): def test_call_str_class(self): called = [False] - class test(object): def __str__(self): called[0] = True @@ -270,7 +267,7 @@ def test_len_table_array(self): def test_len_table_dict(self): table = self.lua.eval('{a=1, b=2, c=3}') - self.assertEqual(0, len(table)) # as returned by Lua's "#" operator + self.assertEqual(0, len(table)) # as returned by Lua's "#" operator def test_table_delattr(self): table = self.lua.eval('{a=1, b=2, c=3}') @@ -296,27 +293,27 @@ def test_len_table(self): def test_iter_table(self): table = self.lua.eval('{2,3,4,5,6}') - self.assertEqual([1, 2, 3, 4, 5], list(table)) + self.assertEqual([1,2,3,4,5], list(table)) def test_iter_table_list_repeat(self): table = self.lua.eval('{2,3,4,5,6}') - self.assertEqual([1, 2, 3, 4, 5], list(table)) # 1 - self.assertEqual([1, 2, 3, 4, 5], list(table)) # 2 - self.assertEqual([1, 2, 3, 4, 5], list(table)) # 3 + self.assertEqual([1,2,3,4,5], list(table)) # 1 + self.assertEqual([1,2,3,4,5], list(table)) # 2 + self.assertEqual([1,2,3,4,5], list(table)) # 3 def test_iter_array_table_values(self): table = self.lua.eval('{2,3,4,5,6}') - self.assertEqual([2, 3, 4, 5, 6], list(table.values())) + self.assertEqual([2,3,4,5,6], list(table.values())) def test_iter_array_table_repeat(self): table = self.lua.eval('{2,3,4,5,6}') - self.assertEqual([2, 3, 4, 5, 6], list(table.values())) # 1 - self.assertEqual([2, 3, 4, 5, 6], list(table.values())) # 2 - self.assertEqual([2, 3, 4, 5, 6], list(table.values())) # 3 + self.assertEqual([2,3,4,5,6], list(table.values())) # 1 + self.assertEqual([2,3,4,5,6], list(table.values())) # 2 + self.assertEqual([2,3,4,5,6], list(table.values())) # 3 def test_iter_multiple_tables(self): count = 10 - table_values = [self.lua.eval('{%s}' % ','.join(map(str, range(2, count + 2)))).values() + table_values = [self.lua.eval('{%s}' % ','.join(map(str, range(2, count+2)))).values() for _ in range(4)] # round robin @@ -325,11 +322,11 @@ def test_iter_multiple_tables(self): for table in table_values: sublist.append(_next(table)) - self.assertEqual([[i] * len(table_values) for i in range(2, count + 2)], l) + self.assertEqual([[i]*len(table_values) for i in range(2, count+2)], l) def test_iter_table_repeat(self): count = 10 - table_values = [self.lua.eval('{%s}' % ','.join(map(str, range(2, count + 2)))).values() + table_values = [self.lua.eval('{%s}' % ','.join(map(str, range(2, count+2)))).values() for _ in range(4)] # one table after the other @@ -338,7 +335,7 @@ def test_iter_table_repeat(self): for sublist in l: sublist.append(_next(table)) - self.assertEqual([[i] * len(table_values) for i in range(2, count + 2)], l) + self.assertEqual([[i]*len(table_values) for i in range(2,count+2)], l) def test_iter_table_refcounting(self): lua_func = self.lua.eval(''' @@ -390,20 +387,20 @@ def test_iter_table_values_int_keys(self): table = self.lua.eval('{%s}' % ','.join('[%d]=%d' % (i, -i) for i in range(10))) l = list(table.values()) l.sort() - self.assertEqual(list(range(-9, 1)), l) + self.assertEqual(list(range(-9,1)), l) def test_iter_table_items(self): keys = list('abcdefg') table = self.lua.eval('{%s}' % ','.join('%s=%d' % (c, i) for i, c in enumerate(keys))) l = list(table.items()) l.sort() - self.assertEqual(list(zip(keys, range(len(keys)))), l) + self.assertEqual(list(zip(keys,range(len(keys)))), l) def test_iter_table_items_int_keys(self): table = self.lua.eval('{%s}' % ','.join('[%d]=%d' % (i, -i) for i in range(10))) l = list(table.items()) l.sort() - self.assertEqual(list(zip(range(10), range(0, -10, -1))), l) + self.assertEqual(list(zip(range(10), range(0,-10,-1))), l) def test_iter_table_values_mixed(self): keys = list('abcdefg') @@ -436,7 +433,7 @@ def test_string_values(self): def test_int_values(self): function = self.lua.eval('function(i) return i + 5 end') - self.assertEqual(3 + 5, function(3)) + self.assertEqual(3+5, function(3)) def test_long_values(self): try: @@ -444,11 +441,11 @@ def test_long_values(self): except NameError: _long = int function = self.lua.eval('function(i) return i + 5 end') - self.assertEqual(3 + 5, function(_long(3))) + self.assertEqual(3+5, function(_long(3))) def test_float_values(self): function = self.lua.eval('function(i) return i + 5 end') - self.assertEqual(float(3) + 5, function(float(3))) + self.assertEqual(float(3)+5, function(float(3))) def test_str_function(self): func = self.lua.eval('function() return 1 end') @@ -459,7 +456,7 @@ def test_str_table(self): self.assertEqual(' Date: Sun, 14 Aug 2022 08:17:27 +0000 Subject: [PATCH 03/15] make keyword arguments clear --- lupa/_lupa.pyx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index bd76e7a1..dc4ee9bf 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -452,12 +452,12 @@ cdef class LuaRuntime: elif isinstance(obj, Mapping): for key in obj: value = obj[key] - py_to_lua(self, L, key, False, recursive) - py_to_lua(self, L, value, False, recursive) + py_to_lua(self, L, key, wrap_none=False, recursive=recursive) + py_to_lua(self, L, value, wrap_none=False, recursive=recursive) lua.lua_rawset(L, -3) else: for arg in obj: - py_to_lua(self, L, arg, False, recursive) + py_to_lua(self, L, arg, wrap_none=False, recursive=recursive) lua.lua_rawseti(L, -2, i) i += 1 return py_from_lua(self, L, -1) @@ -513,12 +513,12 @@ cdef class LuaRuntime: luaL_openlib(L, "python", py_lib, 0) # lib lua.lua_pushlightuserdata(L, self) # lib udata lua.lua_pushcclosure(L, py_args, 1) # lib function - lua.lua_setfield(L, -2, "args") # lib + lua.lua_setfield(L, -2, "args") # lib # register our own object metatable lua.luaL_newmetatable(L, POBJECT) # lib metatbl luaL_openlib(L, NULL, py_object_lib, 0) - lua.lua_pop(L, 1) # lib + lua.lua_pop(L, 1) # lib # create and store the python references table lua.lua_newtable(L) # lib tbl @@ -1822,6 +1822,7 @@ cdef bint call_python(LuaRuntime runtime, lua_State *L, py_object* py_obj) excep else: args = () kwargs = {} + for i in range(nargs): arg = py_from_lua(runtime, L, i+2) if isinstance(arg, _PyArguments): From 8623bbba6e24e55df0be57eb579ccf30ad6c1a2e Mon Sep 17 00:00:00 2001 From: synodriver <624805065@qq.com> Date: Sun, 14 Aug 2022 08:27:20 +0000 Subject: [PATCH 04/15] add overflow check --- lupa/_lupa.pyx | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index dc4ee9bf..d9aac75c 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -157,7 +157,7 @@ def lua_type(obj): lua.lua_settop(L, old_top) unlock_runtime(lua_object._runtime) -cdef int _len_as_int(Py_ssize_t obj) except -1: +cdef inline int _len_as_int(Py_ssize_t obj) except -1: if obj > LONG_MAX: raise OverflowError return obj @@ -1140,9 +1140,7 @@ cdef object resume_lua_thread(_LuaThread thread, tuple args): # already terminated raise StopIteration if args: - if len(args) > LONG_MAX: - raise OverflowError - nargs = len(args) + nargs = _len_as_int(len(args)) push_lua_arguments(thread._runtime, co, args) with nogil: status = lua.lua_resume(co, L, nargs, &nres) @@ -1388,7 +1386,7 @@ cdef py_object* unpack_userdata(lua_State *L, int n) nogil: cdef int py_function_result_to_lua(LuaRuntime runtime, lua_State *L, object o) except -1: if runtime._unpack_returned_tuples and isinstance(o, tuple): push_lua_arguments(runtime, L, o) - return len(o) + return _len_as_int(len(o)) check_lua_stack(L, 1) return py_to_lua(runtime, L, o) @@ -1474,13 +1472,13 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa o = (<_PyProtocolWrapper> o)._obj pushed_values_count = py_to_lua_custom(runtime, L, o, type_flags) elif recursive and isinstance(o, Sequence): - lua.lua_createtable(L, len(o), 0) # create a table at the top of stack, with narr already known + lua.lua_createtable(L, _len_as_int(len(o)), 0) # create a table at the top of stack, with narr already known for i, v in enumerate(o): py_to_lua(runtime, L, v, wrap_none, recursive) lua.lua_rawseti(L, -2, i+1) pushed_values_count = 1 elif recursive and isinstance(o, Mapping): - lua.lua_createtable(L, 0, len(o)) # create a table at the top of stack, with nrec already known + lua.lua_createtable(L, 0, _len_as_int(len(o))) # create a table at the top of stack, with nrec already known for key, value in o.items(): py_to_lua(runtime, L, key, wrap_none, recursive) py_to_lua(runtime, L, value, wrap_none, recursive) From e2c5faa1afb66e8d99d245fb135e483b0f21ff30 Mon Sep 17 00:00:00 2001 From: synodriver <624805065@qq.com> Date: Sun, 14 Aug 2022 13:11:07 +0000 Subject: [PATCH 05/15] better isinstance check --- lupa/_lupa.pyx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index d9aac75c..38b9d057 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -158,7 +158,7 @@ def lua_type(obj): unlock_runtime(lua_object._runtime) cdef inline int _len_as_int(Py_ssize_t obj) except -1: - if obj > LONG_MAX: + if obj > INT_MAX: raise OverflowError return obj @@ -1471,13 +1471,13 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa type_flags = (<_PyProtocolWrapper> o)._type_flags o = (<_PyProtocolWrapper> o)._obj pushed_values_count = py_to_lua_custom(runtime, L, o, type_flags) - elif recursive and isinstance(o, Sequence): + elif recursive and isinstance(o, (list, Sequence)): lua.lua_createtable(L, _len_as_int(len(o)), 0) # create a table at the top of stack, with narr already known - for i, v in enumerate(o): + for i, v in enumerate(o, 1): py_to_lua(runtime, L, v, wrap_none, recursive) - lua.lua_rawseti(L, -2, i+1) + lua.lua_rawseti(L, -2, i) pushed_values_count = 1 - elif recursive and isinstance(o, Mapping): + elif recursive and isinstance(o, (dict, Mapping)): lua.lua_createtable(L, 0, _len_as_int(len(o))) # create a table at the top of stack, with nrec already known for key, value in o.items(): py_to_lua(runtime, L, key, wrap_none, recursive) From 642b37598a921a252dfc4045fef0a185279e2f59 Mon Sep 17 00:00:00 2001 From: synodriver Date: Thu, 4 Jan 2024 19:27:44 +0800 Subject: [PATCH 06/15] add support for self-ref objects --- lupa/_lupa.pyx | 54 ++++++++++++++++++++++++++++------------------ lupa/tests/test.py | 24 +++++++++++++++------ 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 731ecf24..59a1fe9b 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -525,7 +525,7 @@ cdef class LuaRuntime: """ return self.table_from(items, kwargs) - def table_from(self, *args, int max_depth=1): + def table_from(self, *args, bint recursive=False): """Create a new table from Python mapping or iterable. table_from() accepts either a dict/mapping or an iterable with items. @@ -533,7 +533,7 @@ cdef class LuaRuntime: are placed in the table in order. Nested mappings / iterables are passed to Lua as userdata - (wrapped Python objects) if recursive depth is greater than `max_depth`, they are not converted to Lua tables. + (wrapped Python objects) if `recursive` is False, they are not converted to Lua tables. """ assert self._state is not NULL cdef lua_State *L = self._state @@ -547,8 +547,8 @@ cdef class LuaRuntime: for obj in args: if isinstance(obj, dict): for key, value in obj.iteritems(): - py_to_lua(self, L, key, wrap_none=True, max_depth=max_depth) - py_to_lua(self, L, value, wrap_none=False, max_depth=max_depth) + py_to_lua(self, L, key, True, recursive) + py_to_lua(self, L, value, False, recursive) lua.lua_rawset(L, -3) elif isinstance(obj, _LuaTable): @@ -564,12 +564,12 @@ cdef class LuaRuntime: elif isinstance(obj, Mapping): for key in obj: value = obj[key] - py_to_lua(self, L, key, wrap_none=True, max_depth=max_depth) - py_to_lua(self, L, value, wrap_none=False, max_depth=max_depth) + py_to_lua(self, L, key, True, recursive) + py_to_lua(self, L, value, False, recursive) lua.lua_rawset(L, -3) else: for arg in obj: - py_to_lua(self, L, arg, wrap_none=False, max_depth=max_depth) + py_to_lua(self, L, arg, False, recursive) lua.lua_rawseti(L, -2, i) i += 1 return py_from_lua(self, L, -1) @@ -1550,7 +1550,7 @@ cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) e lua.lua_settop(L, old_top) raise -cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False, int max_depth=1, int current_depth=0) except -1: +cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False, bint recursive=False, dict mapped_objs = None) except -1: """Converts Python object to Lua Preconditions: 1 extra slot in the Lua stack @@ -1559,8 +1559,6 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa Returns 0 if cannot convert Python object to Lua Returns 1 if the Python object was converted successfully and pushed onto the stack """ - if current_depth >= max_depth: - raise ValueError("max recursive depth reached") cdef int pushed_values_count = 0 cdef int type_flags = 0 @@ -1608,18 +1606,32 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa type_flags = (<_PyProtocolWrapper> o)._type_flags o = (<_PyProtocolWrapper> o)._obj pushed_values_count = py_to_lua_custom(runtime, L, o, type_flags) - elif max_depth>1 and isinstance(o, (list, Sequence)): - lua.lua_createtable(L, _len_as_int(len(o)), 0) # create a table at the top of stack, with narr already known - for i, v in enumerate(o, 1): - py_to_lua(runtime, L, v, wrap_none, max_depth, current_depth+1) - lua.lua_rawseti(L, -2, i) + elif recursive and isinstance(o, (list, Sequence)): + if mapped_objs is None: + mapped_objs = {} + if id(o) not in mapped_objs: + lua.lua_createtable(L, _len_as_int(len(o)), 0) # create a table at the top of stack, with narr already known + mapped_objs[id(o)] = lua.lua_gettop(L) + for i, v in enumerate(o, 1): + py_to_lua(runtime, L, v, wrap_none, recursive, mapped_objs) + lua.lua_rawseti(L, -2, i) + else: # self-reference detected + idx = mapped_objs[id(o)] + lua.lua_pushvalue(L, idx) pushed_values_count = 1 - elif max_depth>1 and isinstance(o, (dict, Mapping)): - lua.lua_createtable(L, 0, _len_as_int(len(o))) # create a table at the top of stack, with nrec already known - for key, value in o.items(): - py_to_lua(runtime, L, key, wrap_none, max_depth, current_depth+1) - py_to_lua(runtime, L, value, wrap_none, max_depth, current_depth+1) - lua.lua_rawset(L, -3) + elif recursive and isinstance(o, (dict, Mapping)): + if mapped_objs is None: + mapped_objs = {} + if id(o) not in mapped_objs: + lua.lua_createtable(L, 0, _len_as_int(len(o))) # create a table at the top of stack, with nrec already known + mapped_objs[id(o)] = lua.lua_gettop(L) + for key, value in o.items(): + py_to_lua(runtime, L, key, wrap_none, recursive, mapped_objs) + py_to_lua(runtime, L, value, wrap_none, recursive, mapped_objs) + lua.lua_rawset(L, -3) + else: # self-reference detected + idx = mapped_objs[id(o)] + lua.lua_pushvalue(L, idx) pushed_values_count = 1 else: # prefer __getitem__ over __getattr__ by default diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 5b73bdc1..e87e1b49 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -634,7 +634,7 @@ def test_table_from_table_iter_indirect(self): def test_table_from_nested_dict(self): data = {"a": {"a": "foo"}, "b": {"b": "bar"}} - table = self.lua.table_from(data, max_depth=10) + table = self.lua.table_from(data, recursive=True) self.assertEqual(table["a"]["a"], "foo") self.assertEqual(table["b"]["b"], "bar") self.lua.globals()["data"] = table @@ -659,7 +659,7 @@ def test_table_from_nested_dict(self): def test_table_from_nested_list(self): data = {"a": {"a": "foo"}, "b": [1, 2, 3]} - table = self.lua.table_from(data, max_depth=10) + table = self.lua.table_from(data, recursive=True) self.assertEqual(table["a"]["a"], "foo") self.assertEqual(table["b"][1], 1) self.assertEqual(table["b"][2], 2) @@ -686,7 +686,7 @@ def test_table_from_nested_list(self): def test_table_from_nested_list_bad(self): data = {"a": {"a": "foo"}, "b": [1, 2, 3]} - table = self.lua.table_from(data, max_depth=10) # in this case, lua will get userdata instead of table + table = self.lua.table_from(data, recursive=True) # in this case, lua will get userdata instead of table self.assertEqual(table["a"]["a"], "foo") print(list(table["b"])) self.assertEqual(table["b"][1], 1) @@ -699,11 +699,23 @@ def test_table_from_nested_list_bad(self): del self.lua.globals()["data"] - def test_table_from_recursive_dict(self): + def test_table_from_self_ref_obj(self): data = {} data["key"] = data - with self.assertRaises(ValueError): - self.lua.table_from(data, max_depth=10) + l = [] + l.append(l) + data["list"] = l + table = self.lua.table_from(data, recursive=True) + self.lua.globals()["data"] = table + self.lua.eval("assert(type(data)=='table', '')") + self.lua.eval("assert(type(data['key'])=='table', '')") + self.lua.eval("assert(type(data['list'])=='table', '')") + self.lua.eval("assert(data['list']==data['list'][1], 'wrong self-ref list')") + self.lua.eval("assert(type(data['key']['key']['key']['key'])=='table', 'wrong self-ref map')") + self.lua.eval("assert(type(data['key']['key']['key']['key']['list'])=='table', 'wrong self-ref map')") + # self.assertEqual(table["key"], table) + # self.assertEqual(table["list"], table["list"][0]) + del self.lua.globals()["data"] # FIXME: it segfaults # def test_table_from_generator_calling_lua_functions(self): From 0684e26082642dba5bf0a6795b45ab5021e76eac Mon Sep 17 00:00:00 2001 From: synodriver Date: Sat, 6 Jan 2024 22:56:18 +0800 Subject: [PATCH 07/15] better recursive call --- lupa/_lupa.pyx | 55 ++++++++++++++++++++++---------------------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index efd7641a..6c669d7d 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -1572,32 +1572,18 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa type_flags = (<_PyProtocolWrapper> o)._type_flags o = (<_PyProtocolWrapper> o)._obj pushed_values_count = py_to_lua_custom(runtime, L, o, type_flags) - elif recursive and isinstance(o, (list, Sequence)): + elif recursive and isinstance(o, (list, dict, Sequence, Mapping)): if mapped_objs is None: mapped_objs = {} - if id(o) not in mapped_objs: - lua.lua_createtable(L, _len_as_int(len(o)), 0) # create a table at the top of stack, with narr already known - mapped_objs[id(o)] = lua.lua_gettop(L) - for i, v in enumerate(o, 1): - py_to_lua(runtime, L, v, wrap_none, recursive, mapped_objs) - lua.lua_rawseti(L, -2, i) - else: # self-reference detected - idx = mapped_objs[id(o)] - lua.lua_pushvalue(L, idx) - pushed_values_count = 1 - elif recursive and isinstance(o, (dict, Mapping)): - if mapped_objs is None: - mapped_objs = {} - if id(o) not in mapped_objs: - lua.lua_createtable(L, 0, _len_as_int(len(o))) # create a table at the top of stack, with nrec already known - mapped_objs[id(o)] = lua.lua_gettop(L) - for key, value in o.items(): - py_to_lua(runtime, L, key, wrap_none, recursive, mapped_objs) - py_to_lua(runtime, L, value, wrap_none, recursive, mapped_objs) - lua.lua_rawset(L, -3) - else: # self-reference detected - idx = mapped_objs[id(o)] - lua.lua_pushvalue(L, idx) + table = py_to_lua_table(runtime, L, (o,), recursive, mapped_objs) + (<_LuaObject> table).push_lua_object(L) + # if id(o) not in mapped_objs: + # table = py_to_lua_table(runtime, L, (o,), recursive, mapped_objs) + # (<_LuaObject> table).push_lua_object(L) + # mapped_objs[id(o)] = lua.lua_gettop(L) + # else: # self-reference detected + # idx = mapped_objs[id(o)] + # lua.lua_pushvalue(L, idx) pushed_values_count = 1 else: # prefer __getitem__ over __getattr__ by default @@ -1687,7 +1673,7 @@ cdef bytes _asciiOrNone(s): return s -cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, object items, bint recursive=False): +cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bint recursive=False, dict mapped_objs=None): """ Create a new Lua table and add different kinds of values from the sequence 'items' to it. @@ -1699,13 +1685,20 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, object items, b old_top = lua.lua_gettop(L) lua.lua_newtable(L) # FIXME: how to check for failure? - + if recursive and mapped_objs is None: + mapped_objs = {} try: for obj in items: + if recursive: + if id(obj) not in mapped_objs: + mapped_objs[id(obj)] = lua.lua_gettop(L) + else: + idx = mapped_objs[id(obj)] + return new_lua_table(runtime, L, idx) if isinstance(obj, dict): for key, value in (obj).items(): - py_to_lua(runtime, L, key, True, recursive) - py_to_lua(runtime, L, value, False, recursive) + py_to_lua(runtime, L, key, True, recursive, mapped_objs) + py_to_lua(runtime, L, value, False, recursive, mapped_objs) lua.lua_rawset(L, -3) elif isinstance(obj, _LuaTable): @@ -1721,13 +1714,13 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, object items, b elif isinstance(obj, Mapping): for key in obj: value = obj[key] - py_to_lua(runtime, L, key, True, recursive) - py_to_lua(runtime, L, value, False, recursive) + py_to_lua(runtime, L, key, True, recursive, mapped_objs) + py_to_lua(runtime, L, value, False, recursive, mapped_objs) lua.lua_rawset(L, -3) else: for arg in obj: - py_to_lua(runtime, L, arg, False, recursive) + py_to_lua(runtime, L, arg, False, recursive, mapped_objs) lua.lua_rawseti(L, -2, i) i += 1 From 56f6dac46f82fc1e8d35a6a9e9275d0bec5e8b7e Mon Sep 17 00:00:00 2001 From: synodriver Date: Sat, 6 Jan 2024 23:04:45 +0800 Subject: [PATCH 08/15] clean up annotations --- lupa/_lupa.pyx | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 6c669d7d..86fa9d1e 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -533,7 +533,7 @@ cdef class LuaRuntime: are placed in the table in order. Nested mappings / iterables are passed to Lua as userdata - (wrapped Python objects) if `recursive` is False, they are not converted to Lua tables. + (wrapped Python objects). If `recursive` is False, they are not converted to Lua tables. """ assert self._state is not NULL cdef lua_State *L = self._state @@ -1577,13 +1577,6 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa mapped_objs = {} table = py_to_lua_table(runtime, L, (o,), recursive, mapped_objs) (<_LuaObject> table).push_lua_object(L) - # if id(o) not in mapped_objs: - # table = py_to_lua_table(runtime, L, (o,), recursive, mapped_objs) - # (<_LuaObject> table).push_lua_object(L) - # mapped_objs[id(o)] = lua.lua_gettop(L) - # else: # self-reference detected - # idx = mapped_objs[id(o)] - # lua.lua_pushvalue(L, idx) pushed_values_count = 1 else: # prefer __getitem__ over __getattr__ by default From 3c321ca51134def700ad403b04697a08fa942b41 Mon Sep 17 00:00:00 2001 From: synodriver Date: Sun, 7 Jan 2024 01:16:59 +0800 Subject: [PATCH 09/15] better tests --- lupa/tests/test.py | 78 ++++++++++++++-------------------------------- 1 file changed, 24 insertions(+), 54 deletions(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index e87e1b49..83171706 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -104,6 +104,9 @@ def get_attr(obj, name): class TestLuaRuntime(SetupLuaRuntimeMixin, LupaTestCase): + def assertLuaResult(self, lua_expression, result): + self.assertEqual(self.lua.eval(lua_expression), result) + def test_lua_version(self): version = self.lua.lua_version self.assertEqual(tuple, type(version)) @@ -638,24 +641,10 @@ def test_table_from_nested_dict(self): self.assertEqual(table["a"]["a"], "foo") self.assertEqual(table["b"]["b"], "bar") self.lua.globals()["data"] = table - self.lua.eval("assert(data.a.a=='foo', 'failed')") - self.lua.eval("assert(data.b.b=='bar', 'failed')") - self.lua.eval("assert(type(data.a)=='table', 'failed, expect table, got '..type(data.a))") - self.lua.eval("assert(type(data.b)=='table', 'failed, expect table, got '..type(data.b))") - self.lua.execute("""function itertable(table) - for k,v in pairs(table) do - print(k) - if type(v) == "table" then - itertable(v) - else - print(v) - end - end - end - print('\\n') - itertable(data) - """) - del self.lua.globals()["data"] + self.assertLuaResult("data.a.a", "foo") + self.assertLuaResult("data.b.b", "bar") + self.assertLuaResult("type(data.a)", "table") + self.assertLuaResult("type(data.b)", "table") def test_table_from_nested_list(self): data = {"a": {"a": "foo"}, "b": [1, 2, 3]} @@ -665,39 +654,23 @@ def test_table_from_nested_list(self): self.assertEqual(table["b"][2], 2) self.assertEqual(table["b"][3], 3) self.lua.globals()["data"] = table - self.lua.eval("assert(data.a.a=='foo', 'failed')") + self.assertLuaResult("data.a.a", "foo") + self.assertLuaResult("#data.b", 3) self.lua.eval("assert(#data.b==3, 'failed')") - self.lua.eval("assert(type(data.a)=='table', 'failed, expect table, got '..type(data.a))") - self.lua.eval("assert(type(data.b)=='table', 'failed, expect table, got '..type(data.b))") - self.lua.execute("""function itertable(table) - for k,v in pairs(table) do - print(k) - if type(v) == "table" then - itertable(v) - else - print(v) - end - end - end - print('\\n') - itertable(data) - """) - del self.lua.globals()["data"] + self.assertLuaResult("type(data.a)", "table") + self.assertLuaResult("type(data.b)", "table") def test_table_from_nested_list_bad(self): data = {"a": {"a": "foo"}, "b": [1, 2, 3]} - table = self.lua.table_from(data, recursive=True) # in this case, lua will get userdata instead of table + table = self.lua.table_from(data) # in this case, lua will get userdata instead of table self.assertEqual(table["a"]["a"], "foo") - print(list(table["b"])) - self.assertEqual(table["b"][1], 1) - self.assertEqual(table["b"][2], 2) - self.assertEqual(table["b"][3], 3) + self.assertEqual(list(table["b"]), [1, 2, 3]) + self.assertEqual(table["b"][0], 1) + self.assertEqual(table["b"][1], 2) + self.assertEqual(table["b"][2], 3) self.lua.globals()["data"] = table - - self.lua.eval("assert(type(data.a)=='table', 'failed, expect table, got '..type(data.a))") - self.lua.eval("assert(type(data.b)=='table', 'failed, expect table, got '..type(data.b))") - - del self.lua.globals()["data"] + self.assertLuaResult("type(data.a)", "userdata") + self.assertLuaResult("type(data.b)", "userdata") def test_table_from_self_ref_obj(self): data = {} @@ -707,15 +680,12 @@ def test_table_from_self_ref_obj(self): data["list"] = l table = self.lua.table_from(data, recursive=True) self.lua.globals()["data"] = table - self.lua.eval("assert(type(data)=='table', '')") - self.lua.eval("assert(type(data['key'])=='table', '')") - self.lua.eval("assert(type(data['list'])=='table', '')") - self.lua.eval("assert(data['list']==data['list'][1], 'wrong self-ref list')") - self.lua.eval("assert(type(data['key']['key']['key']['key'])=='table', 'wrong self-ref map')") - self.lua.eval("assert(type(data['key']['key']['key']['key']['list'])=='table', 'wrong self-ref map')") - # self.assertEqual(table["key"], table) - # self.assertEqual(table["list"], table["list"][0]) - del self.lua.globals()["data"] + self.assertLuaResult("type(data)", 'table') + self.assertLuaResult("type(data['key'])",'table') + self.assertLuaResult("type(data['list'])",'table') + self.assertLuaResult("data['list']==data['list'][1]", True) + self.assertLuaResult("type(data['key']['key']['key']['key'])", 'table') + self.assertLuaResult("type(data['key']['key']['key']['key']['list'])", 'table') # FIXME: it segfaults # def test_table_from_generator_calling_lua_functions(self): From b21907f3b139c7b7aead726072f8d9b846a3d6a4 Mon Sep 17 00:00:00 2001 From: synodriver Date: Sun, 7 Jan 2024 13:43:27 +0800 Subject: [PATCH 10/15] add python list with same value --- lupa/tests/test.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 83171706..fba16d26 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -601,10 +601,16 @@ def test_table_from_bad(self): self.assertRaises(TypeError, self.lua.table_from, None) self.assertRaises(TypeError, self.lua.table_from, {"a": 5}, 123) - # def test_table_from_nested(self): - # table = self.lua.table_from({"obj": {"foo": "bar"}}) - # lua_type = self.lua.eval("type") - # self.assertEqual(lua_type(table["obj"]), "table") + def test_table_from_nested(self): + table = self.lua.table_from([[3, 3, 3]], recursive=True) + self.lua.globals()["data"] = table + self.assertLuaResult("data[1][1]", 3) + self.assertLuaResult("data[1][2]", 3) + self.assertLuaResult("data[1][3]", 3) + self.assertLuaResult("type(data)", "table") + self.assertLuaResult("type(data[1])", "table") + self.assertLuaResult("#data", 1) + self.assertLuaResult("#data[1]", 3) def test_table_from_table(self): table1 = self.lua.eval("{3, 4, foo='bar'}") From 31edf83668a25627f84cc9d12d0432008934de7d Mon Sep 17 00:00:00 2001 From: synodriver Date: Mon, 5 Feb 2024 18:55:57 +0800 Subject: [PATCH 11/15] use suggested changes from maintainer --- lupa/__init__.py | 6 ++---- lupa/_lupa.pyx | 24 ++++++++++++++---------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/lupa/__init__.py b/lupa/__init__.py index d23019bc..7b05b6e8 100644 --- a/lupa/__init__.py +++ b/lupa/__init__.py @@ -63,10 +63,8 @@ def _import_newest_lib(): raise RuntimeError("Failed to import Lupa binary module.") # prefer Lua over LuaJIT and high versions over low versions. module_name = max(modules, key=lambda m: (m[1] == 'lua', tuple(map(int, m[2] or '0')))) - try: - _newest_lib = __import__(module_name[0], level=1, fromlist="*", globals=globals()) - except ModuleNotFoundError: - _newest_lib = __import__(module_name[1], level=1, fromlist="*", globals=globals()) + + _newest_lib = __import__(module_name[0], level=1, fromlist="*", globals=globals()) return _newest_lib diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 86fa9d1e..058d356b 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -533,13 +533,14 @@ cdef class LuaRuntime: are placed in the table in order. Nested mappings / iterables are passed to Lua as userdata - (wrapped Python objects). If `recursive` is False, they are not converted to Lua tables. + (wrapped Python objects). If `recursive` is False (the default), + they are not converted to Lua tables. """ assert self._state is not NULL cdef lua_State *L = self._state lock_runtime(self) try: - return py_to_lua_table(self, L, args, recursive) + return py_to_lua_table(self, L, args, recursive=recursive) finally: unlock_runtime(self) @@ -1516,7 +1517,7 @@ cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) e lua.lua_settop(L, old_top) raise -cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False, bint recursive=False, dict mapped_objs = None) except -1: +cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False, bint recursive=False, dict mapped_objs=None) except -1: """Converts Python object to Lua Preconditions: 1 extra slot in the Lua stack @@ -1575,7 +1576,7 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa elif recursive and isinstance(o, (list, dict, Sequence, Mapping)): if mapped_objs is None: mapped_objs = {} - table = py_to_lua_table(runtime, L, (o,), recursive, mapped_objs) + table = py_to_lua_table(runtime, L, (o,), recursive=recursive, mapped_objs=mapped_objs) (<_LuaObject> table).push_lua_object(L) pushed_values_count = 1 else: @@ -1677,6 +1678,7 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bi check_lua_stack(L, 5) old_top = lua.lua_gettop(L) lua.lua_newtable(L) + cdef int lua_table_ref = lua.lua_gettop(L) # the index of the lua table which we are filling # FIXME: how to check for failure? if recursive and mapped_objs is None: mapped_objs = {} @@ -1684,14 +1686,16 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bi for obj in items: if recursive: if id(obj) not in mapped_objs: - mapped_objs[id(obj)] = lua.lua_gettop(L) + # this object is never seen before, we should cache it + mapped_objs[id(obj)] = lua_table_ref else: + # this object has been cached, just get the corresponding lua table's index idx = mapped_objs[id(obj)] return new_lua_table(runtime, L, idx) if isinstance(obj, dict): for key, value in (obj).items(): - py_to_lua(runtime, L, key, True, recursive, mapped_objs) - py_to_lua(runtime, L, value, False, recursive, mapped_objs) + py_to_lua(runtime, L, key, wrap_none=True, recursive=recursive, mapped_objs=mapped_objs) + py_to_lua(runtime, L, value, wrap_none=False, recursive=recursive, mapped_objs=mapped_objs) lua.lua_rawset(L, -3) elif isinstance(obj, _LuaTable): @@ -1707,13 +1711,13 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bi elif isinstance(obj, Mapping): for key in obj: value = obj[key] - py_to_lua(runtime, L, key, True, recursive, mapped_objs) - py_to_lua(runtime, L, value, False, recursive, mapped_objs) + py_to_lua(runtime, L, key, wrap_none=True, recursive=recursive, mapped_objs=mapped_objs) + py_to_lua(runtime, L, value, wrap_none=False, recursive=recursive, mapped_objs=mapped_objs) lua.lua_rawset(L, -3) else: for arg in obj: - py_to_lua(runtime, L, arg, False, recursive, mapped_objs) + py_to_lua(runtime, L, arg, wrap_none=False, recursive=recursive, mapped_objs=mapped_objs) lua.lua_rawseti(L, -2, i) i += 1 From 56d50a64ecf2d7f3f42f8e5910b9f8505fd97264 Mon Sep 17 00:00:00 2001 From: synodriver Date: Mon, 5 Feb 2024 19:58:01 +0800 Subject: [PATCH 12/15] rename mapped_objs to mapped_tables --- lupa/_lupa.pyx | 34 +++++++++++++++++----------------- lupa/tests/test.py | 7 +++++++ 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 058d356b..8ee16793 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -1517,7 +1517,7 @@ cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) e lua.lua_settop(L, old_top) raise -cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False, bint recursive=False, dict mapped_objs=None) except -1: +cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False, bint recursive=False, dict mapped_tables=None) except -1: """Converts Python object to Lua Preconditions: 1 extra slot in the Lua stack @@ -1574,9 +1574,9 @@ cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=Fa o = (<_PyProtocolWrapper> o)._obj pushed_values_count = py_to_lua_custom(runtime, L, o, type_flags) elif recursive and isinstance(o, (list, dict, Sequence, Mapping)): - if mapped_objs is None: - mapped_objs = {} - table = py_to_lua_table(runtime, L, (o,), recursive=recursive, mapped_objs=mapped_objs) + if mapped_tables is None: + mapped_tables = {} + table = py_to_lua_table(runtime, L, (o,), recursive=recursive, mapped_tables=mapped_tables) (<_LuaObject> table).push_lua_object(L) pushed_values_count = 1 else: @@ -1667,7 +1667,7 @@ cdef bytes _asciiOrNone(s): return s -cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bint recursive=False, dict mapped_objs=None): +cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bint recursive=False, dict mapped_tables=None): """ Create a new Lua table and add different kinds of values from the sequence 'items' to it. @@ -1678,24 +1678,24 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bi check_lua_stack(L, 5) old_top = lua.lua_gettop(L) lua.lua_newtable(L) - cdef int lua_table_ref = lua.lua_gettop(L) # the index of the lua table which we are filling - # FIXME: how to check for failure? - if recursive and mapped_objs is None: - mapped_objs = {} + # FIXME: handle allocation errors + cdef int lua_table_ref = lua.lua_gettop(L) # the index of the lua table which we are filling + if recursive and mapped_tables is None: + mapped_tables = {} try: for obj in items: if recursive: - if id(obj) not in mapped_objs: + if id(obj) not in mapped_tables: # this object is never seen before, we should cache it - mapped_objs[id(obj)] = lua_table_ref + mapped_tables[id(obj)] = lua_table_ref else: # this object has been cached, just get the corresponding lua table's index - idx = mapped_objs[id(obj)] + idx = mapped_tables[id(obj)] return new_lua_table(runtime, L, idx) if isinstance(obj, dict): for key, value in (obj).items(): - py_to_lua(runtime, L, key, wrap_none=True, recursive=recursive, mapped_objs=mapped_objs) - py_to_lua(runtime, L, value, wrap_none=False, recursive=recursive, mapped_objs=mapped_objs) + py_to_lua(runtime, L, key, wrap_none=True, recursive=recursive, mapped_tables=mapped_tables) + py_to_lua(runtime, L, value, wrap_none=False, recursive=recursive, mapped_tables=mapped_tables) lua.lua_rawset(L, -3) elif isinstance(obj, _LuaTable): @@ -1711,13 +1711,13 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bi elif isinstance(obj, Mapping): for key in obj: value = obj[key] - py_to_lua(runtime, L, key, wrap_none=True, recursive=recursive, mapped_objs=mapped_objs) - py_to_lua(runtime, L, value, wrap_none=False, recursive=recursive, mapped_objs=mapped_objs) + py_to_lua(runtime, L, key, wrap_none=True, recursive=recursive, mapped_tables=mapped_tables) + py_to_lua(runtime, L, value, wrap_none=False, recursive=recursive, mapped_tables=mapped_tables) lua.lua_rawset(L, -3) else: for arg in obj: - py_to_lua(runtime, L, arg, wrap_none=False, recursive=recursive, mapped_objs=mapped_objs) + py_to_lua(runtime, L, arg, wrap_none=False, recursive=recursive, mapped_tables=mapped_tables) lua.lua_rawseti(L, -2, i) i += 1 diff --git a/lupa/tests/test.py b/lupa/tests/test.py index fba16d26..57b8734f 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -612,6 +612,13 @@ def test_table_from_nested(self): self.assertLuaResult("#data", 1) self.assertLuaResult("#data[1]", 3) + def test_table_from_nested2(self): + table2 = self.lua.table_from([{"a": "foo"}, {"b": 1}], recursive=True) + self.lua.globals()["data2"] = table2 + self.assertLuaResult("#data2", 2) + self.assertLuaResult("data2[1]['a']", "foo") + self.assertLuaResult("data2[2]['b']", 1) + def test_table_from_table(self): table1 = self.lua.eval("{3, 4, foo='bar'}") table2 = self.lua.table_from(table1) From b24166415676fb0eed1a4de91c09c6b80ec001f6 Mon Sep 17 00:00:00 2001 From: scoder Date: Tue, 20 Feb 2024 10:36:53 +0100 Subject: [PATCH 13/15] Improve docstring. --- lupa/_lupa.pyx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 8ee16793..48e7e7fe 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -533,8 +533,9 @@ cdef class LuaRuntime: are placed in the table in order. Nested mappings / iterables are passed to Lua as userdata - (wrapped Python objects). If `recursive` is False (the default), - they are not converted to Lua tables. + (wrapped Python objects) by default. If `recursive` is True, + they are converted to Lua tables recursively, handling loops + and duplicates via identity de-duplication. """ assert self._state is not NULL cdef lua_State *L = self._state From 7a69ee8c7abd071805906087cd4dd3edb7d0ece3 Mon Sep 17 00:00:00 2001 From: synodriver Date: Tue, 6 Feb 2024 00:31:39 +0800 Subject: [PATCH 14/15] add tests --- lupa/_lupa.pyx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 8ee16793..b6858816 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -533,8 +533,9 @@ cdef class LuaRuntime: are placed in the table in order. Nested mappings / iterables are passed to Lua as userdata - (wrapped Python objects). If `recursive` is False (the default), - they are not converted to Lua tables. + (wrapped Python objects) by default. If `recursive` is True, + they are converted to Lua tables recursively, handling loops + and duplicates via identity de-duplication. """ assert self._state is not NULL cdef lua_State *L = self._state @@ -1674,6 +1675,7 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bi Dicts, Mappings and Lua tables are unpacked into key-value pairs. Everything else is considered a sequence of plain values that get appended to the table. """ + print(f"in py_to_lua_table {items}") # todo del cdef int i = 1 check_lua_stack(L, 5) old_top = lua.lua_gettop(L) @@ -1688,8 +1690,10 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bi if id(obj) not in mapped_tables: # this object is never seen before, we should cache it mapped_tables[id(obj)] = lua_table_ref + print(f"caching {obj}") # todo del else: # this object has been cached, just get the corresponding lua table's index + print(f"using cache {obj}") # todo del idx = mapped_tables[id(obj)] return new_lua_table(runtime, L, idx) if isinstance(obj, dict): From c3ca11aa55bbc2b4cc244cfe252bf5f3567c9620 Mon Sep 17 00:00:00 2001 From: synodriver Date: Tue, 20 Feb 2024 22:00:07 +0800 Subject: [PATCH 15/15] remove print in py_to_lua_table --- lupa/_lupa.pyx | 3 --- 1 file changed, 3 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index b6858816..48e7e7fe 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -1675,7 +1675,6 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bi Dicts, Mappings and Lua tables are unpacked into key-value pairs. Everything else is considered a sequence of plain values that get appended to the table. """ - print(f"in py_to_lua_table {items}") # todo del cdef int i = 1 check_lua_stack(L, 5) old_top = lua.lua_gettop(L) @@ -1690,10 +1689,8 @@ cdef _LuaTable py_to_lua_table(LuaRuntime runtime, lua_State* L, tuple items, bi if id(obj) not in mapped_tables: # this object is never seen before, we should cache it mapped_tables[id(obj)] = lua_table_ref - print(f"caching {obj}") # todo del else: # this object has been cached, just get the corresponding lua table's index - print(f"using cache {obj}") # todo del idx = mapped_tables[id(obj)] return new_lua_table(runtime, L, idx) if isinstance(obj, dict):