From 8436f9cade01a8c05de1e603423fd27363b82f41 Mon Sep 17 00:00:00 2001 From: guidanoli Date: Fri, 16 Jul 2021 14:43:26 -0300 Subject: [PATCH 1/8] Context Handler for Lua stack --- lupa/_lupa.pyx | 258 +++++++++++++++++++++---------------------------- lupa/lua.pxd | 6 +- 2 files changed, 115 insertions(+), 149 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index cf172b4a..794c02cd 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -132,11 +132,9 @@ def lua_type(obj): return None lua_object = <_LuaObject>obj assert lua_object._runtime is not None - lock_runtime(lua_object._runtime) L = lua_object._state - old_top = lua.lua_gettop(L) cdef const char* lua_type_name - try: + with lua_object._runtime.stack(1): lua_object.push_lua_object(L) ltype = lua.lua_type(L, -1) if ltype == lua.LUA_TTABLE: @@ -150,9 +148,36 @@ def lua_type(obj): else: lua_type_name = lua.lua_typename(L, ltype) return lua_type_name if IS_PY2 else lua_type_name.decode('ascii') - finally: - lua.lua_settop(L, old_top) - unlock_runtime(lua_object._runtime) + + +cdef enum _LuaRuntimeStackRestorationPolicy: + # restoration policy for Lua stack context handler + RESTORE_NEVER = 0 + RESTORE_ALWAYS = 1 + RESTORE_ON_ERROR = 2 + +@cython.internal +@cython.no_gc_clear +@cython.freelist(16) +cdef class _LuaRuntimeStack: + """Context handler for the Lua runtime stack""" + cdef LuaRuntime _runtime + cdef int _top + cdef int _extra + cdef int _restore + + def __enter__(self): + lock_runtime(self._runtime) + check_lua_stack(self._runtime._state, self._extra) + self._top = lua.lua_gettop(self._runtime._state) + + def __exit__(self, *exc_info): + try: + if (self._restore == RESTORE_ALWAYS or + (self._restore == RESTORE_ON_ERROR and any(exc_info))): + lua.lua_settop(self._runtime._state, self._top) + finally: + unlock_runtime(self._runtime) @cython.no_gc_clear @@ -285,6 +310,20 @@ cdef class LuaRuntime: lua.lua_close(self._state) self._state = NULL + @cython.final + cdef _LuaRuntimeStack stack(self, int extra, int restore=RESTORE_ALWAYS): + """ + Context handler for managing the Lua stack + Ensures 'extra' slots in the stack + Employs 'restore' restoration policy + """ + cdef _LuaRuntimeStack ctx + ctx = _LuaRuntimeStack.__new__(_LuaRuntimeStack) + ctx._runtime = self + ctx._extra = extra + ctx._restore = restore + return ctx + @property def lua_version(self): """ @@ -346,10 +385,8 @@ cdef class LuaRuntime: if isinstance(lua_code, unicode): lua_code = (lua_code).encode(self._source_encoding) L = self._state - lock_runtime(self) - oldtop = lua.lua_gettop(L) cdef size_t size - try: + with self.stack(1): status = lua.luaL_loadbuffer(L, lua_code, len(lua_code), b'') if status == 0: return py_from_lua(self, L, -1) @@ -357,9 +394,6 @@ cdef class LuaRuntime: err = lua.lua_tolstring(L, -1, &size) error = err[:size] if self._encoding is None else err[:size].decode(self._encoding) raise LuaSyntaxError(error) - finally: - lua.lua_settop(L, oldtop) - unlock_runtime(self) def require(self, modulename): """Load a Lua library into the runtime. @@ -368,16 +402,11 @@ cdef class LuaRuntime: cdef lua_State *L = self._state if not isinstance(modulename, (bytes, unicode)): raise TypeError("modulename must be a string") - lock_runtime(self) - old_top = lua.lua_gettop(L) - try: + with self.stack(1): lua.lua_getglobal(L, 'require') if lua.lua_isnil(L, -1): raise LuaError("require is not defined") return call_lua(self, L, (modulename,)) - finally: - lua.lua_settop(L, old_top) - unlock_runtime(self) def globals(self): """Return the globals defined in this Lua runtime as a Lua @@ -385,16 +414,9 @@ cdef class LuaRuntime: """ assert self._state is not NULL cdef lua_State *L = self._state - lock_runtime(self) - old_top = lua.lua_gettop(L) - try: - lua.lua_getglobal(L, '_G') - if lua.lua_isnil(L, -1): - raise LuaError("globals not defined") + with self.stack(1): + lua.lua_pushglobaltable(L) return py_from_lua(self, L, -1) - finally: - lua.lua_settop(L, old_top) - unlock_runtime(self) def table(self, *items, **kwargs): """Create a new table with the provided items. Positional @@ -416,9 +438,7 @@ cdef class LuaRuntime: assert self._state is not NULL cdef lua_State *L = self._state cdef int i = 1 - lock_runtime(self) - old_top = lua.lua_gettop(L) - try: + with self.stack(5): lua.lua_newtable(L) # FIXME: how to check for failure? for obj in args: @@ -450,37 +470,30 @@ cdef class LuaRuntime: lua.lua_rawseti(L, -2, i) i += 1 return py_from_lua(self, L, -1) - finally: - lua.lua_settop(L, old_top) - unlock_runtime(self) def set_overflow_handler(self, overflow_handler): """Set the overflow handler function that is called on failures to pass large numbers to Lua. """ cdef lua_State *L = self._state - if overflow_handler is not None and not callable(overflow_handler): raise ValueError("overflow_handler must be callable") - - lua.lua_pushlstring(L, LUPAOFH, len(LUPAOFH)) - - if not py_to_lua(self, L, overflow_handler): - lua.lua_pop(L, 1) - raise LuaError("failed to convert overflow_handler") - - lua.lua_rawset(L, lua.LUA_REGISTRYINDEX) + with self.stack(2): + lua.lua_pushlstring(L, LUPAOFH, len(LUPAOFH)) # key + if not py_to_lua(self, L, overflow_handler): # key value + raise LuaError("failed to convert overflow_handler") + lua.lua_rawset(L, lua.LUA_REGISTRYINDEX) # @cython.final cdef int register_py_object(self, bytes cname, bytes pyname, object obj) except -1: cdef lua_State *L = self._state # tbl - lua.lua_pushlstring(L, cname, len(cname)) # tbl cname - if not py_to_lua_custom(self, L, obj, 0): # tbl cname obj - lua.lua_pop(L, 1) - raise LuaError("failed to convert %s object" % pyname) - lua.lua_pushlstring(L, pyname, len(pyname)) # tbl cname obj pyname - lua.lua_pushvalue(L, -2) # tbl cname obj pyname obj - lua.lua_rawset(L, -5) # tbl cname obj - lua.lua_rawset(L, lua.LUA_REGISTRYINDEX) # tbl + with self.stack(4): + lua.lua_pushlstring(L, cname, len(cname)) # tbl cname + if not py_to_lua_custom(self, L, obj, 0): # tbl cname obj + raise LuaError("failed to convert %s object" % pyname) + lua.lua_pushlstring(L, pyname, len(pyname)) # tbl cname obj pyname + lua.lua_pushvalue(L, -2) # tbl cname obj pyname obj + lua.lua_rawset(L, -5) # tbl cname obj + lua.lua_rawset(L, lua.LUA_REGISTRYINDEX) # tbl return 0 @cython.final @@ -608,10 +621,7 @@ cdef tuple unpack_lua_table(LuaRuntime runtime): cdef int old_top cdef Py_ssize_t index, length cdef lua_State* L = runtime._state - check_lua_stack(L, 2) - lock_runtime(runtime) - old_top = lua.lua_gettop(L) - try: + with runtime.stack(2): length = get_object_length(runtime, L, -1) args = cpython.tuple.PyTuple_New(length) lua.lua_pushnil(L) # nil (first key) @@ -634,9 +644,6 @@ cdef tuple unpack_lua_table(LuaRuntime runtime): else: raise TypeError("table key is neither an integer nor a string") lua.lua_pop(L, 1) # key - finally: - lua.lua_settop(L, old_top) - unlock_runtime(runtime) return args, kwargs @@ -728,15 +735,9 @@ cdef class _LuaObject: cdef Py_ssize_t _len(self) except -1: assert self._runtime is not None cdef lua_State* L = self._state - lock_runtime(self._runtime) - size = 0 - try: + with self._runtime.stack(1): self.push_lua_object(L) - size = get_object_length(self._runtime, L, -1) - lua.lua_pop(L, 1) - finally: - unlock_runtime(self._runtime) - return size + return get_object_length(self._runtime, L, -1) def __nonzero__(self): return True @@ -749,44 +750,28 @@ cdef class _LuaObject: assert self._runtime is not None cdef lua_State* L = self._state cdef bytes encoding = self._runtime._encoding or b'UTF-8' - lock_runtime(self._runtime) - try: + with self._runtime.stack(1): self.push_lua_object(L) return lua_object_repr(L, encoding) - finally: - lua.lua_pop(L, 1) - unlock_runtime(self._runtime) def __str__(self): assert self._runtime is not None cdef lua_State* L = self._state - cdef unicode py_string = None - cdef const char *s + cdef const char *string cdef size_t size = 0 cdef bytes encoding = self._runtime._encoding or b'UTF-8' - lock_runtime(self._runtime) - old_top = lua.lua_gettop(L) - try: - self.push_lua_object(L) - # lookup and call "__tostring" metatable method manually to catch any errors - if lua.lua_getmetatable(L, -1): - lua.lua_pushlstring(L, "__tostring", 10) - lua.lua_rawget(L, -2) - if not lua.lua_isnil(L, -1) and lua.lua_pcall(L, 1, 1, 0) == 0: - s = lua.lua_tolstring(L, -1, &size) - if s: + with self._runtime.stack(2): + self.push_lua_object(L) # obj + if lua.luaL_getmetafield(L, -1, "__tostring"): # obj tostr + lua.lua_insert(L, -2) # tostr obj + if lua.lua_pcall(L, 1, 1, 0) == 0: # str + string = lua.lua_tolstring(L, -1, &size) + if string: try: - py_string = s[:size].decode(encoding) + return string[:size].decode(encoding) except UnicodeDecodeError: - # safe 'decode' - py_string = s[:size].decode('ISO-8859-1') - if py_string is None: - lua.lua_settop(L, old_top + 1) - py_string = lua_object_repr(L, encoding) - finally: - lua.lua_settop(L, old_top) - unlock_runtime(self._runtime) - return py_string + return string[:size].decode('ISO-8859-1') + return repr(self) def __getattr__(self, name): assert self._runtime is not None @@ -805,21 +790,18 @@ cdef class _LuaObject: @cython.final cdef _getitem(self, name, bint is_attr_access): cdef lua_State* L = self._state - lock_runtime(self._runtime) - old_top = lua.lua_gettop(L) - try: - lua.lua_pushcfunction(L, get_from_lua_table) # func - self.push_lua_object(L) # func obj + cdef int lua_type + with self._runtime.stack(3): + # table[nil] fails, so map None -> python.none for Lua tables + lua.lua_pushcfunction(L, get_from_lua_table) # func + self.push_lua_object(L) # func obj lua_type = lua.lua_type(L, -1) if lua_type == lua.LUA_TFUNCTION or lua_type == lua.LUA_TTHREAD: raise (AttributeError if is_attr_access else TypeError)( "item/attribute access not supported on functions") # table[nil] fails, so map None -> python.none for Lua tables py_to_lua(self._runtime, L, name, wrap_none=(lua_type == lua.LUA_TTABLE)) # func obj key - return execute_lua_call(self._runtime, L, 2) # obj[key] - finally: - lua.lua_settop(L, old_top) # - unlock_runtime(self._runtime) + return execute_lua_call(self._runtime, L, 2) # obj[key] cdef _LuaObject new_lua_object(LuaRuntime runtime, lua_State* L, int n): @@ -900,17 +882,12 @@ cdef class _LuaTable(_LuaObject): @cython.final cdef int _setitem(self, name, value) except -1: cdef lua_State* L = self._state - lock_runtime(self._runtime) - old_top = lua.lua_gettop(L) - try: + with self._runtime.stack(3): self.push_lua_object(L) # table[nil] fails, so map None -> python.none for Lua tables py_to_lua(self._runtime, L, name, wrap_none=True) py_to_lua(self._runtime, L, value) lua.lua_settable(L, -3) - finally: - lua.lua_settop(L, old_top) - unlock_runtime(self._runtime) return 0 def __delattr__(self, item): @@ -931,16 +908,11 @@ cdef class _LuaTable(_LuaObject): @cython.final cdef _delitem(self, name): cdef lua_State* L = self._state - lock_runtime(self._runtime) - old_top = lua.lua_gettop(L) - try: + with self._runtime.stack(3): self.push_lua_object(L) py_to_lua(self._runtime, L, name, wrap_none=True) lua.lua_pushnil(L) lua.lua_settable(L, -3) - finally: - lua.lua_settop(L, old_top) - unlock_runtime(self._runtime) cdef _LuaTable new_lua_table(LuaRuntime runtime, lua_State* L, int n): @@ -962,9 +934,7 @@ cdef class _LuaFunction(_LuaObject): cdef lua_State* L = self._state cdef lua_State* co cdef _LuaThread thread - lock_runtime(self._runtime) - old_top = lua.lua_gettop(L) - try: + with self._runtime.stack(3): self.push_lua_object(L) if not lua.lua_isfunction(L, -1) or lua.lua_iscfunction(L, -1): raise TypeError("Lua object is not a function") @@ -977,9 +947,6 @@ cdef class _LuaFunction(_LuaObject): thread = new_lua_thread(self._runtime, L, -1) thread._arguments = args # always a tuple, not None ! return thread - finally: - lua.lua_settop(L, old_top) - unlock_runtime(self._runtime) cdef _LuaFunction new_lua_function(LuaRuntime runtime, lua_State* L, int n): cdef _LuaFunction obj = _LuaFunction.__new__(_LuaFunction) @@ -1078,9 +1045,7 @@ cdef object resume_lua_thread(_LuaThread thread, tuple args): cdef lua_State* co = thread._co_state cdef lua_State* L = thread._state cdef int status, i, nargs = 0, nres = 0 - lock_runtime(thread._runtime) - old_top = lua.lua_gettop(L) - try: + with thread._runtime.stack(1): if lua.lua_status(co) == 0 and lua.lua_gettop(co) == 0: # already terminated raise StopIteration @@ -1103,10 +1068,6 @@ cdef object resume_lua_thread(_LuaThread thread, tuple args): # it affects wrapped Lua functions returned to Python. lua.lua_xmove(co, L, nres) return unpack_lua_results(thread._runtime, L) - finally: - # FIXME: check that coroutine state is OK in case of errors? - lua.lua_settop(L, old_top) - unlock_runtime(thread._runtime) cdef enum: @@ -1159,9 +1120,7 @@ cdef class _LuaIter: if self._obj is None: raise StopIteration cdef lua_State* L = self._obj._state - lock_runtime(self._runtime) - old_top = lua.lua_gettop(L) - try: + with self._runtime.stack(3): if self._obj is None: raise StopIteration # iterable object @@ -1196,9 +1155,6 @@ cdef class _LuaIter: lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._refiter) self._refiter = 0 self._obj = None - finally: - lua.lua_settop(L, old_top) - unlock_runtime(self._runtime) raise StopIteration # type conversions and protocol adaptations @@ -1313,6 +1269,8 @@ cdef py_object* unpack_userdata(lua_State *L, int n) nogil: Like luaL_checkudata(), unpacks a userdata object and validates that it's a wrapped Python object. Returns NULL on failure. """ + if not lua.lua_checkstack(L, 2): + return NULL p = lua.lua_touserdata(L, n) if p and lua.lua_getmetatable(L, n): # found userdata with metatable - the one we expect? @@ -1327,6 +1285,7 @@ cdef int py_function_result_to_lua(LuaRuntime runtime, lua_State *L, object o) e if runtime._unpack_returned_tuples and isinstance(o, tuple): push_lua_arguments(runtime, L, o) return len(o) + check_lua_stack(L, 1) return py_to_lua(runtime, L, o) cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) except -1: @@ -1414,6 +1373,8 @@ cdef bint py_to_lua_custom(LuaRuntime runtime, lua_State *L, object o, int type_ cdef py_object* py_obj refkey = build_pyref_key(o, type_flags) cdef _PyReference pyref + if not lua.lua_checkstack(L, 3): + return 0 lua.lua_getfield(L, lua.LUA_REGISTRYINDEX, PYREFST) # tbl @@ -1507,17 +1468,11 @@ cdef build_lua_error_message(LuaRuntime runtime, lua_State* L, unicode err_messa cdef run_lua(LuaRuntime runtime, bytes lua_code, tuple args): # locks the runtime cdef lua_State* L = runtime._state - cdef bint result - lock_runtime(runtime) - old_top = lua.lua_gettop(L) - try: + with runtime.stack(1): if lua.luaL_loadbuffer(L, lua_code, len(lua_code), ''): raise LuaSyntaxError(build_lua_error_message( runtime, L, u"error loading code: %s", -1)) return call_lua(runtime, L, args) - finally: - lua.lua_settop(L, old_top) - unlock_runtime(runtime) cdef call_lua(LuaRuntime runtime, lua_State *L, tuple args): # does not lock the runtime! @@ -1554,15 +1509,22 @@ cdef object execute_lua_call(LuaRuntime runtime, lua_State *L, Py_ssize_t nargs) cdef int push_lua_arguments(LuaRuntime runtime, lua_State *L, tuple args, bint first_may_be_nil=True) except -1: - cdef int i + cdef int i, n + cdef Py_ssize_t nargs + cdef bint wrap_none = not first_may_be_nil if args: - old_top = lua.lua_gettop(L) - for i, arg in enumerate(args): - if not py_to_lua(runtime, L, arg, wrap_none=not first_may_be_nil): - lua.lua_settop(L, old_top) - raise TypeError("failed to convert argument at index %d" % i) - first_may_be_nil = True - return 0 + nargs = len(args) + if nargs > INT_MAX: + raise OverflowError("tuple too large to unpack") + n = nargs + with runtime.stack(n, RESTORE_ON_ERROR): + for i, arg in enumerate(args): + if not py_to_lua(runtime, L, arg, wrap_none=wrap_none): + raise TypeError("failed to convert argument at index %d" % i) + wrap_none = False + return n + else: + return 0 cdef inline object unpack_lua_results(LuaRuntime runtime, lua_State *L): cdef int nargs = lua.lua_gettop(L) diff --git a/lupa/lua.pxd b/lupa/lua.pxd index 262b3f20..92a989e9 100644 --- a/lupa/lua.pxd +++ b/lupa/lua.pxd @@ -456,11 +456,15 @@ cdef extern from * nogil: #else #error Lupa requires at least Lua 5.1 or LuaJIT 2.x #endif + + #if LUA_VERSION_NUM < 502 + #define lua_pushglobaltable(L) lua_pushvalue(L, LUA_GLOBALSINDEX) + #endif """ int read_lua_version(lua_State *L) int lua_isinteger(lua_State *L, int idx) lua_Integer lua_tointegerx (lua_State *L, int idx, int *isnum) - + void lua_pushglobaltable (lua_State *L) cdef extern from *: # Limits for Lua integers (in Lua<5.3: PTRDIFF_MIN, PTRDIFF_MAX) From 1d788387ece9f988c577dbc95a69cd9831f38d60 Mon Sep 17 00:00:00 2001 From: guidanoli Date: Fri, 16 Jul 2021 17:47:25 -0300 Subject: [PATCH 2/8] Made _LuaRuntimeStack safer * Make lock_runtime return boolean and not raise error directly * If check_lua_stack fails on _LuaRuntimeStack.__enter__, unlocks LuaRuntime, avoiding the resource to be acquired forever * Use _LuaRuntimeStack in py_to_lua_custom (unrolls stack on Python error) * Use _LuaRuntimeStack in py_to_lua_overflow (catches error on py_to_lua_custom) * Add preconditions and postconditions to important functions/methods * Add assertion for extra >= 0 on check_lua_stack (would violate Lua C API precondition and probably cause undefined behaviour) * Lock LuaRuntime instance on _fix_args_kwargs and ensure 1 Lua stack slot --- lupa/_lupa.pyx | 213 ++++++++++++++++++++++++++++--------------------- 1 file changed, 120 insertions(+), 93 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 794c02cd..622e40b3 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -160,18 +160,40 @@ cdef enum _LuaRuntimeStackRestorationPolicy: @cython.no_gc_clear @cython.freelist(16) cdef class _LuaRuntimeStack: - """Context handler for the Lua runtime stack""" + """Context handler for the Lua runtime stack + Invariants: + _runtime is not None + """ cdef LuaRuntime _runtime cdef int _top cdef int _extra cdef int _restore def __enter__(self): - lock_runtime(self._runtime) - check_lua_stack(self._runtime._state, self._extra) - self._top = lua.lua_gettop(self._runtime._state) + """Enter context handler + Preconditions: + _extra >= 0 + Postconditions: + _runtime is locked + Can push up to _extra values onto the stack + """ + if not lock_runtime(self._runtime): + raise RuntimeError("failed to acquire thread lock") + try: + check_lua_stack(self._runtime._state, self._extra) + self._top = lua.lua_gettop(self._runtime._state) + except: + unlock_runtime(self._runtime) + raise def __exit__(self, *exc_info): + """Exit context handler + Postconditions: + If _restore is RESTORE_ALWAYS, or + if _restore is RESTORE_ON_ERROR and an error was raised, + then restores stack top to when __enter__ was called + Unlocks _runtime + """ try: if (self._restore == RESTORE_ALWAYS or (self._restore == RESTORE_ON_ERROR and any(exc_info))): @@ -488,8 +510,7 @@ cdef class LuaRuntime: cdef lua_State *L = self._state # tbl with self.stack(4): lua.lua_pushlstring(L, cname, len(cname)) # tbl cname - if not py_to_lua_custom(self, L, obj, 0): # tbl cname obj - raise LuaError("failed to convert %s object" % pyname) + py_to_lua_custom(self, L, obj, 0) # tbl cname obj lua.lua_pushlstring(L, pyname, len(pyname)) # tbl cname obj pyname lua.lua_pushvalue(L, -2) # tbl cname obj pyname obj lua.lua_rawset(L, -5) # tbl cname obj @@ -578,6 +599,7 @@ cdef int check_lua_stack(lua_State* L, int extra) except -1: """Wrapper around lua_checkstack. On failure, a MemoryError is raised. """ + assert extra >= 0 if not lua.lua_checkstack(L, extra): raise MemoryError(f"could not reserve memory for {extra} free extra slots on the Lua stack") return 0 @@ -612,7 +634,7 @@ cdef Py_ssize_t get_object_length(LuaRuntime runtime, lua_State* L, int index) e cdef tuple unpack_lua_table(LuaRuntime runtime): """Unpacks the table at the top of the stack into a tuple of positional arguments - and a dictionary of keyword arguments. + and a dictionary of keyword arguments. """ assert runtime is not None cdef tuple args @@ -661,17 +683,16 @@ cdef tuple _fix_args_kwargs(tuple args): return args, {} cdef _LuaTable table = <_LuaTable>arg - table.push_lua_object(table._state) - return unpack_lua_table(table._runtime) + with table._runtime.stack(1): + table.push_lua_object(table._state) + return unpack_lua_table(table._runtime) ################################################################################ # fast, re-entrant runtime locking -cdef inline int lock_runtime(LuaRuntime runtime) except -1: - if not lock_lock(runtime._lock, pythread.PyThread_get_thread_ident(), True): - raise LuaError("Failed to acquire thread lock") - return 0 +cdef inline bint lock_runtime(LuaRuntime runtime) with gil: + return lock_lock(runtime._lock, pythread.PyThread_get_thread_ident(), True) cdef inline void unlock_runtime(LuaRuntime runtime) nogil: unlock_lock(runtime._lock) @@ -697,29 +718,30 @@ cdef class _LuaObject: if self._runtime is None: return cdef lua_State* L = self._state - locked = False - try: - lock_runtime(self._runtime) - locked = True - except: - pass - finally: + if lock_runtime(self._runtime): lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._ref) - if locked: - unlock_runtime(self._runtime) + unlock_runtime(self._runtime) @cython.final cdef inline int push_lua_object(self, lua_State* L) except -1: + """Pushes Lua object onto the stack + Preconditions: + 1 extra slot in the Lua stack + LuaRuntime is locked + Postconditions: + Lua object (not nil) is pushed onto of the stack + """ lua.lua_rawgeti(L, lua.LUA_REGISTRYINDEX, self._ref) if lua.lua_isnil(L, -1): lua.lua_pop(L, 1) raise LuaError("lost reference") - return 0 + return 1 def __call__(self, *args): assert self._runtime is not None cdef lua_State* L = self._state - lock_runtime(self._runtime) + if not lock_runtime(self._runtime): + raise RuntimeError("failed to acquire thread lock") try: lua.lua_settop(L, 0) self.push_lua_object(L) @@ -1099,16 +1121,9 @@ cdef class _LuaIter: if self._runtime is None: return cdef lua_State* L = self._state - if L is not NULL and self._refiter: - locked = False - try: - lock_runtime(self._runtime) - locked = True - except: - pass + if L is not NULL and self._refiter and lock_runtime(self._runtime): lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._refiter) - if locked: - unlock_runtime(self._runtime) + unlock_runtime(self._runtime) def __repr__(self): return u"LuaIter(%r)" % (self._obj) @@ -1204,9 +1219,9 @@ def as_itemgetter(obj): return wrap cdef object py_from_lua(LuaRuntime runtime, lua_State *L, int n): - """ - Convert a Lua object to a Python object by either mapping, wrapping - or unwrapping it. + """Convert a Lua object to a Python object by either mapping, wrapping or unwrapping it. + Preconditions: + Index n is valid """ cdef size_t size = 0 cdef const char *s @@ -1289,23 +1304,32 @@ cdef int py_function_result_to_lua(LuaRuntime runtime, lua_State *L, object o) e return py_to_lua(runtime, L, o) cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) except -1: - cdef int nargs - - lua.lua_pushlstring(L, LUPAOFH, len(LUPAOFH)) - lua.lua_rawget(L, lua.LUA_REGISTRYINDEX) - if lua.lua_isnil(L, -1): - lua.lua_pop(L, 1) - return 0 - nargs = py_to_lua_custom(runtime, L, o, 0) - if nargs <= 0: - lua.lua_pop(L, 1) - return 0 - if lua.lua_pcall(L, nargs, 1, 0): - lua.lua_pop(L, 1) - return 0 - return 1 + """Converts Python object to Lua via overflow handler + Postconditions: + Returns 0 if cannot convert Python object to Lua (handler not registered or failed) + Returns 1 if the Python object was converted successfully and pushed onto the stack + """ + with runtime.stack(2, RESTORE_ON_ERROR): + lua.lua_pushlstring(L, LUPAOFH, len(LUPAOFH)) + lua.lua_rawget(L, lua.LUA_REGISTRYINDEX) + if lua.lua_isnil(L, -1): + lua.lua_pop(L, 1) + return 0 + py_to_lua_custom(runtime, L, o, 0) + if lua.lua_pcall(L, 1, 1, 0): + lua.lua_pop(L, 1) + return 0 + return 1 cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False) except -1: + """Converts Python object to Lua + Preconditions: + 1 extra slot in the Lua stack + runtime is locked + Postconditions: + Returns 0 if cannot convert Python object to Lua + Returns 1 if the Python object was converted successfully and pushed onto the stack + """ cdef int pushed_values_count = 0 cdef int type_flags = 0 @@ -1370,47 +1394,46 @@ cdef inline tuple build_pyref_key(PyObject* o, int type_flags): cdef bint py_to_lua_custom(LuaRuntime runtime, lua_State *L, object o, int type_flags) except -1: + """Wrap Python object into a Lua userdatum with certain type flags + Postconditions: + Pushes wrapped Python object and returns 1 + """ cdef py_object* py_obj refkey = build_pyref_key(o, type_flags) cdef _PyReference pyref - if not lua.lua_checkstack(L, 3): - return 0 - - lua.lua_getfield(L, lua.LUA_REGISTRYINDEX, PYREFST) # tbl - - # check if Python object is already referenced in Lua - if refkey in runtime._pyrefs_in_lua: - pyref = <_PyReference>runtime._pyrefs_in_lua[refkey] - lua.lua_rawgeti(L, -1, pyref._ref) # tbl udata - py_obj = lua.lua_touserdata(L, -1) - if py_obj: - lua.lua_remove(L, -2) # udata - return 1 # values pushed - lua.lua_pop(L, 1) # tbl - - py_obj = lua.lua_newuserdata(L, sizeof(py_object)) - if not py_obj: - lua.lua_pop(L, 1) # - return 0 # values pushed - - py_obj.obj = o # tbl udata - py_obj.runtime = runtime - py_obj.type_flags = type_flags - lua.luaL_getmetatable(L, POBJECT) # tbl udata metatbl - lua.lua_setmetatable(L, -2) # tbl udata - lua.lua_pushvalue(L, -1) # tbl udata udata - pyref = _PyReference.__new__(_PyReference) - pyref._ref = lua.luaL_ref(L, -3) # tbl udata - pyref._obj = o - lua.lua_remove(L, -2) # udata - - # originally, we just used: - #cpython.ref.Py_INCREF(o) - # now, we store an owned reference in "runtime._pyrefs_in_lua" to keep it visible to Python - # and a borrowed reference in "py_obj.obj" for access from Lua - runtime._pyrefs_in_lua[refkey] = pyref + with runtime.stack(3, RESTORE_ON_ERROR): + lua.lua_getfield(L, lua.LUA_REGISTRYINDEX, PYREFST) # tbl - return 1 # values pushed + # check if Python object is already referenced in Lua + if refkey in runtime._pyrefs_in_lua: + pyref = <_PyReference>runtime._pyrefs_in_lua[refkey] + lua.lua_rawgeti(L, -1, pyref._ref) # tbl udata + py_obj = lua.lua_touserdata(L, -1) + if py_obj: + lua.lua_remove(L, -2) # udata + return 1 # values pushed + lua.lua_pop(L, 1) # tbl + + # create new wrapper for Python object + py_obj = lua.lua_newuserdata(L, sizeof(py_object)) + py_obj.obj = o # tbl udata + py_obj.runtime = runtime + py_obj.type_flags = type_flags + lua.luaL_getmetatable(L, POBJECT) # tbl udata metatbl + lua.lua_setmetatable(L, -2) # tbl udata + lua.lua_pushvalue(L, -1) # tbl udata udata + pyref = _PyReference.__new__(_PyReference) + pyref._ref = lua.luaL_ref(L, -3) # tbl udata + pyref._obj = o + lua.lua_remove(L, -2) # udata + + # originally, we just used: + #cpython.ref.Py_INCREF(o) + # now, we store an owned reference in "runtime._pyrefs_in_lua" to keep it visible to Python + # and a borrowed reference in "py_obj.obj" for access from Lua + runtime._pyrefs_in_lua[refkey] = pyref + + return 1 # values pushed cdef inline int _isascii(unsigned char* s): cdef unsigned char c = 0 @@ -1915,15 +1938,19 @@ cdef int py_iter_with_gil(lua_State* L, py_object* py_obj, int type_flags) with cdef int py_push_iterator(LuaRuntime runtime, lua_State* L, iterator, int type_flags, lua.lua_Integer initial_value) except -2: - # Lua needs three values: iterator C function + state + control variable (last iter) value - old_top = lua.lua_gettop(L) + """Pushes iterator function, invariant state variable and control variable + Preconditions: + 3 extra slots in the Lua stack + Postconditions: + Pushes py_iter_next, iterator and the control variable + Returns the number of pushed values + """ + # push the iterator function lua.lua_pushcfunction(L, py_iter_next) # push the wrapped iterator object as for-loop state object if runtime._unpack_returned_tuples: type_flags |= OBJ_UNPACK_TUPLE - if py_to_lua_custom(runtime, L, iterator, type_flags) < 1: - lua.lua_settop(L, old_top) - return -1 + py_to_lua_custom(runtime, L, iterator, type_flags) # push either enumerator index or nil as control variable value if type_flags & OBJ_ENUMERATOR: lua.lua_pushinteger(L, initial_value) From 0bdd832ddb7af141a123624dcffab50f6a1a88e7 Mon Sep 17 00:00:00 2001 From: guidanoli Date: Sat, 17 Jul 2021 00:53:20 -0300 Subject: [PATCH 3/8] Optimized stack management --- lupa/_lupa.pyx | 266 ++++++++++++++++++++++++++++++------------------- lupa/lua.pxd | 2 +- 2 files changed, 163 insertions(+), 105 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 622e40b3..cd274aaf 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -131,10 +131,11 @@ def lua_type(obj): if not isinstance(obj, _LuaObject): return None lua_object = <_LuaObject>obj - assert lua_object._runtime is not None L = lua_object._state - cdef const char* lua_type_name - with lua_object._runtime.stack(1): + runtime = lua_object._runtime + assert runtime is not None + top = getstacktop(runtime, L, 1) + try: lua_object.push_lua_object(L) ltype = lua.lua_type(L, -1) if ltype == lua.LUA_TTABLE: @@ -148,58 +149,8 @@ def lua_type(obj): else: lua_type_name = lua.lua_typename(L, ltype) return lua_type_name if IS_PY2 else lua_type_name.decode('ascii') - - -cdef enum _LuaRuntimeStackRestorationPolicy: - # restoration policy for Lua stack context handler - RESTORE_NEVER = 0 - RESTORE_ALWAYS = 1 - RESTORE_ON_ERROR = 2 - -@cython.internal -@cython.no_gc_clear -@cython.freelist(16) -cdef class _LuaRuntimeStack: - """Context handler for the Lua runtime stack - Invariants: - _runtime is not None - """ - cdef LuaRuntime _runtime - cdef int _top - cdef int _extra - cdef int _restore - - def __enter__(self): - """Enter context handler - Preconditions: - _extra >= 0 - Postconditions: - _runtime is locked - Can push up to _extra values onto the stack - """ - if not lock_runtime(self._runtime): - raise RuntimeError("failed to acquire thread lock") - try: - check_lua_stack(self._runtime._state, self._extra) - self._top = lua.lua_gettop(self._runtime._state) - except: - unlock_runtime(self._runtime) - raise - - def __exit__(self, *exc_info): - """Exit context handler - Postconditions: - If _restore is RESTORE_ALWAYS, or - if _restore is RESTORE_ON_ERROR and an error was raised, - then restores stack top to when __enter__ was called - Unlocks _runtime - """ - try: - if (self._restore == RESTORE_ALWAYS or - (self._restore == RESTORE_ON_ERROR and any(exc_info))): - lua.lua_settop(self._runtime._state, self._top) - finally: - unlock_runtime(self._runtime) + finally: + setstacktop(runtime, L, top) @cython.no_gc_clear @@ -332,20 +283,6 @@ cdef class LuaRuntime: lua.lua_close(self._state) self._state = NULL - @cython.final - cdef _LuaRuntimeStack stack(self, int extra, int restore=RESTORE_ALWAYS): - """ - Context handler for managing the Lua stack - Ensures 'extra' slots in the stack - Employs 'restore' restoration policy - """ - cdef _LuaRuntimeStack ctx - ctx = _LuaRuntimeStack.__new__(_LuaRuntimeStack) - ctx._runtime = self - ctx._extra = extra - ctx._restore = restore - return ctx - @property def lua_version(self): """ @@ -408,7 +345,8 @@ cdef class LuaRuntime: lua_code = (lua_code).encode(self._source_encoding) L = self._state cdef size_t size - with self.stack(1): + top = getstacktop(self, L, 1) + try: status = lua.luaL_loadbuffer(L, lua_code, len(lua_code), b'') if status == 0: return py_from_lua(self, L, -1) @@ -416,6 +354,8 @@ cdef class LuaRuntime: err = lua.lua_tolstring(L, -1, &size) error = err[:size] if self._encoding is None else err[:size].decode(self._encoding) raise LuaSyntaxError(error) + finally: + setstacktop(self, L, top) def require(self, modulename): """Load a Lua library into the runtime. @@ -424,11 +364,14 @@ cdef class LuaRuntime: cdef lua_State *L = self._state if not isinstance(modulename, (bytes, unicode)): raise TypeError("modulename must be a string") - with self.stack(1): + top = getstacktop(self, L, 1) + try: lua.lua_getglobal(L, 'require') if lua.lua_isnil(L, -1): raise LuaError("require is not defined") return call_lua(self, L, (modulename,)) + finally: + setstacktop(self, L, top) def globals(self): """Return the globals defined in this Lua runtime as a Lua @@ -436,9 +379,12 @@ cdef class LuaRuntime: """ assert self._state is not NULL cdef lua_State *L = self._state - with self.stack(1): + top = getstacktop(self, L, 1) + try: lua.lua_pushglobaltable(L) return py_from_lua(self, L, -1) + finally: + setstacktop(self, L, top) def table(self, *items, **kwargs): """Create a new table with the provided items. Positional @@ -460,8 +406,8 @@ cdef class LuaRuntime: assert self._state is not NULL cdef lua_State *L = self._state cdef int i = 1 - with self.stack(5): - lua.lua_newtable(L) + top = getstacktop(self, L, 5) + try: # FIXME: how to check for failure? for obj in args: if isinstance(obj, dict): @@ -492,6 +438,8 @@ cdef class LuaRuntime: lua.lua_rawseti(L, -2, i) i += 1 return py_from_lua(self, L, -1) + finally: + setstacktop(self, L, top) def set_overflow_handler(self, overflow_handler): """Set the overflow handler function that is called on failures to pass large numbers to Lua. @@ -499,23 +447,29 @@ cdef class LuaRuntime: cdef lua_State *L = self._state if overflow_handler is not None and not callable(overflow_handler): raise ValueError("overflow_handler must be callable") - with self.stack(2): + top = getstacktop(self, L, 2) + try: lua.lua_pushlstring(L, LUPAOFH, len(LUPAOFH)) # key if not py_to_lua(self, L, overflow_handler): # key value raise LuaError("failed to convert overflow_handler") lua.lua_rawset(L, lua.LUA_REGISTRYINDEX) # + finally: + setstacktop(self, L, top) @cython.final cdef int register_py_object(self, bytes cname, bytes pyname, object obj) except -1: cdef lua_State *L = self._state # tbl - with self.stack(4): + top = getstacktop(self, L, 4) + try: lua.lua_pushlstring(L, cname, len(cname)) # tbl cname py_to_lua_custom(self, L, obj, 0) # tbl cname obj lua.lua_pushlstring(L, pyname, len(pyname)) # tbl cname obj pyname lua.lua_pushvalue(L, -2) # tbl cname obj pyname obj lua.lua_rawset(L, -5) # tbl cname obj lua.lua_rawset(L, lua.LUA_REGISTRYINDEX) # tbl - return 0 + return 0 + finally: + setstacktop(self, L, top) @cython.final cdef int init_python_lib(self, bint register_eval, bint register_builtins) except -1: @@ -595,7 +549,44 @@ def unpacks_lua_table_method(meth): return wrapper -cdef int check_lua_stack(lua_State* L, int extra) except -1: +################################################################################ +# Lua stack management from Python + +cdef inline int getstacktop(LuaRuntime runtime, lua_State* L, int extra) except -1: + """Get stack top + Preconditions: + runtime is not None + L != NULL + extra >= 0 + Postconditions: + Acquires thread lock for runtime + Checks for extra slots in Lua stack + Returns Lua stack top (>= 0) + """ + if not lock_runtime(runtime): + raise RuntimeError("failed to acquire thread lock") + try: + check_lua_stack(L, extra) + return lua.lua_gettop(L) + except: + unlock_runtime(runtime) + raise + +cdef inline int setstacktop(LuaRuntime runtime, lua_State* L, int top): + """Set stack top + Preconditions: + runtime is not None + L != NULL + top >= 0 + Postconditions: + Releases thread lock for runtime + Sets Lua stack top to given index + """ + lua.lua_settop(L, top) + unlock_runtime(runtime) + + +cdef inline int check_lua_stack(lua_State* L, int extra) except -1: """Wrapper around lua_checkstack. On failure, a MemoryError is raised. """ @@ -632,7 +623,7 @@ cdef Py_ssize_t get_object_length(LuaRuntime runtime, lua_State* L, int index) e return length -cdef tuple unpack_lua_table(LuaRuntime runtime): +cdef tuple unpack_lua_table(LuaRuntime runtime, lua_State* L): """Unpacks the table at the top of the stack into a tuple of positional arguments and a dictionary of keyword arguments. """ @@ -642,8 +633,8 @@ cdef tuple unpack_lua_table(LuaRuntime runtime): cdef bytes source_encoding = runtime._source_encoding cdef int old_top cdef Py_ssize_t index, length - cdef lua_State* L = runtime._state - with runtime.stack(2): + top = getstacktop(runtime, L, 2) + try: length = get_object_length(runtime, L, -1) args = cpython.tuple.PyTuple_New(length) lua.lua_pushnil(L) # nil (first key) @@ -666,7 +657,9 @@ cdef tuple unpack_lua_table(LuaRuntime runtime): else: raise TypeError("table key is neither an integer nor a string") lua.lua_pop(L, 1) # key - return args, kwargs + return args, kwargs + finally: + setstacktop(runtime, L, top) cdef tuple _fix_args_kwargs(tuple args): @@ -683,9 +676,14 @@ cdef tuple _fix_args_kwargs(tuple args): return args, {} cdef _LuaTable table = <_LuaTable>arg - with table._runtime.stack(1): - table.push_lua_object(table._state) - return unpack_lua_table(table._runtime) + runtime = table._runtime + L = table._state + top = getstacktop(runtime, L, 1) + try: + table.push_lua_object(L) + return unpack_lua_table(runtime, L) + finally: + setstacktop(runtime, L, top) ################################################################################ @@ -757,9 +755,12 @@ cdef class _LuaObject: cdef Py_ssize_t _len(self) except -1: assert self._runtime is not None cdef lua_State* L = self._state - with self._runtime.stack(1): + top = getstacktop(self._runtime, L, 2) + try: self.push_lua_object(L) return get_object_length(self._runtime, L, -1) + finally: + setstacktop(self._runtime, L, top) def __nonzero__(self): return True @@ -772,9 +773,12 @@ cdef class _LuaObject: assert self._runtime is not None cdef lua_State* L = self._state cdef bytes encoding = self._runtime._encoding or b'UTF-8' - with self._runtime.stack(1): + top = getstacktop(self._runtime, L, 1) + try: self.push_lua_object(L) return lua_object_repr(L, encoding) + finally: + setstacktop(self._runtime, L, top) def __str__(self): assert self._runtime is not None @@ -782,7 +786,8 @@ cdef class _LuaObject: cdef const char *string cdef size_t size = 0 cdef bytes encoding = self._runtime._encoding or b'UTF-8' - with self._runtime.stack(2): + top = getstacktop(self._runtime, L, 2) + try: self.push_lua_object(L) # obj if lua.luaL_getmetafield(L, -1, "__tostring"): # obj tostr lua.lua_insert(L, -2) # tostr obj @@ -793,6 +798,8 @@ cdef class _LuaObject: return string[:size].decode(encoding) except UnicodeDecodeError: return string[:size].decode('ISO-8859-1') + finally: + setstacktop(self._runtime, L, top) return repr(self) def __getattr__(self, name): @@ -813,7 +820,8 @@ cdef class _LuaObject: cdef _getitem(self, name, bint is_attr_access): cdef lua_State* L = self._state cdef int lua_type - with self._runtime.stack(3): + top = getstacktop(self._runtime, L, 3) + try: # table[nil] fails, so map None -> python.none for Lua tables lua.lua_pushcfunction(L, get_from_lua_table) # func self.push_lua_object(L) # func obj @@ -824,6 +832,8 @@ cdef class _LuaObject: # table[nil] fails, so map None -> python.none for Lua tables py_to_lua(self._runtime, L, name, wrap_none=(lua_type == lua.LUA_TTABLE)) # func obj key return execute_lua_call(self._runtime, L, 2) # obj[key] + finally: + setstacktop(self._runtime, L, top) cdef _LuaObject new_lua_object(LuaRuntime runtime, lua_State* L, int n): @@ -904,13 +914,16 @@ cdef class _LuaTable(_LuaObject): @cython.final cdef int _setitem(self, name, value) except -1: cdef lua_State* L = self._state - with self._runtime.stack(3): + top = getstacktop(self._runtime, L, 3) + try: self.push_lua_object(L) # table[nil] fails, so map None -> python.none for Lua tables py_to_lua(self._runtime, L, name, wrap_none=True) py_to_lua(self._runtime, L, value) lua.lua_settable(L, -3) - return 0 + return 0 + finally: + setstacktop(self._runtime, L, top) def __delattr__(self, item): assert self._runtime is not None @@ -930,12 +943,14 @@ cdef class _LuaTable(_LuaObject): @cython.final cdef _delitem(self, name): cdef lua_State* L = self._state - with self._runtime.stack(3): + top = getstacktop(self._runtime, L, 1) + try: self.push_lua_object(L) py_to_lua(self._runtime, L, name, wrap_none=True) lua.lua_pushnil(L) lua.lua_settable(L, -3) - + finally: + setstacktop(self._runtime, L, top) cdef _LuaTable new_lua_table(LuaRuntime runtime, lua_State* L, int n): cdef _LuaTable obj = _LuaTable.__new__(_LuaTable) @@ -956,7 +971,8 @@ cdef class _LuaFunction(_LuaObject): cdef lua_State* L = self._state cdef lua_State* co cdef _LuaThread thread - with self._runtime.stack(3): + top = getstacktop(self._runtime, L, 3) + try: self.push_lua_object(L) if not lua.lua_isfunction(L, -1) or lua.lua_iscfunction(L, -1): raise TypeError("Lua object is not a function") @@ -969,6 +985,8 @@ cdef class _LuaFunction(_LuaObject): thread = new_lua_thread(self._runtime, L, -1) thread._arguments = args # always a tuple, not None ! return thread + finally: + setstacktop(self._runtime, L, top) cdef _LuaFunction new_lua_function(LuaRuntime runtime, lua_State* L, int n): cdef _LuaFunction obj = _LuaFunction.__new__(_LuaFunction) @@ -1067,7 +1085,8 @@ cdef object resume_lua_thread(_LuaThread thread, tuple args): cdef lua_State* co = thread._co_state cdef lua_State* L = thread._state cdef int status, i, nargs = 0, nres = 0 - with thread._runtime.stack(1): + top = getstacktop(thread._runtime, L, 1) + try: if lua.lua_status(co) == 0 and lua.lua_gettop(co) == 0: # already terminated raise StopIteration @@ -1090,6 +1109,8 @@ cdef object resume_lua_thread(_LuaThread thread, tuple args): # it affects wrapped Lua functions returned to Python. lua.lua_xmove(co, L, nres) return unpack_lua_results(thread._runtime, L) + finally: + setstacktop(thread._runtime, L, top) cdef enum: @@ -1135,7 +1156,8 @@ cdef class _LuaIter: if self._obj is None: raise StopIteration cdef lua_State* L = self._obj._state - with self._runtime.stack(3): + top = getstacktop(self._runtime, L, 3) + try: if self._obj is None: raise StopIteration # iterable object @@ -1170,6 +1192,8 @@ cdef class _LuaIter: lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._refiter) self._refiter = 0 self._obj = None + finally: + setstacktop(self._runtime, L, top) raise StopIteration # type conversions and protocol adaptations @@ -1305,11 +1329,15 @@ cdef int py_function_result_to_lua(LuaRuntime runtime, lua_State *L, object o) e cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) except -1: """Converts Python object to Lua via overflow handler + Preconditions: + Lua runtime is locked Postconditions: Returns 0 if cannot convert Python object to Lua (handler not registered or failed) Returns 1 if the Python object was converted successfully and pushed onto the stack """ - with runtime.stack(2, RESTORE_ON_ERROR): + top = lua.lua_gettop(L) + check_lua_stack(L, 2) + try: lua.lua_pushlstring(L, LUPAOFH, len(LUPAOFH)) lua.lua_rawget(L, lua.LUA_REGISTRYINDEX) if lua.lua_isnil(L, -1): @@ -1320,6 +1348,9 @@ cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) e lua.lua_pop(L, 1) return 0 return 1 + except: + lua.lua_settop(L, top) + raise cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False) except -1: """Converts Python object to Lua @@ -1395,13 +1426,17 @@ cdef inline tuple build_pyref_key(PyObject* o, int type_flags): cdef bint py_to_lua_custom(LuaRuntime runtime, lua_State *L, object o, int type_flags) except -1: """Wrap Python object into a Lua userdatum with certain type flags + Preconditions: + LuaRuntime is locked Postconditions: Pushes wrapped Python object and returns 1 """ cdef py_object* py_obj refkey = build_pyref_key(o, type_flags) cdef _PyReference pyref - with runtime.stack(3, RESTORE_ON_ERROR): + check_lua_stack(L, 3) + top = lua.lua_gettop(L) + try: lua.lua_getfield(L, lua.LUA_REGISTRYINDEX, PYREFST) # tbl # check if Python object is already referenced in Lua @@ -1434,6 +1469,9 @@ cdef bint py_to_lua_custom(LuaRuntime runtime, lua_State *L, object o, int type_ runtime._pyrefs_in_lua[refkey] = pyref return 1 # values pushed + except: + lua.lua_settop(L, top) + raise cdef inline int _isascii(unsigned char* s): cdef unsigned char c = 0 @@ -1489,17 +1527,25 @@ cdef build_lua_error_message(LuaRuntime runtime, lua_State* L, unicode err_messa # calling into Lua cdef run_lua(LuaRuntime runtime, bytes lua_code, tuple args): - # locks the runtime + """Run Lua code with arguments""" cdef lua_State* L = runtime._state - with runtime.stack(1): + top = getstacktop(runtime, L, 1) + try: if lua.luaL_loadbuffer(L, lua_code, len(lua_code), ''): raise LuaSyntaxError(build_lua_error_message( runtime, L, u"error loading code: %s", -1)) return call_lua(runtime, L, args) + finally: + setstacktop(runtime, L, top) cdef call_lua(LuaRuntime runtime, lua_State *L, tuple args): - # does not lock the runtime! - # does not clean up the stack! + """Call function on top of the stack with args + Preconditions: + Function is on top of the stack + LuaRuntime must be locked + Postconditions: + Pops function from the stack and pushes results + """ push_lua_arguments(runtime, L, args) return execute_lua_call(runtime, L, len(args)) @@ -1532,6 +1578,13 @@ cdef object execute_lua_call(LuaRuntime runtime, lua_State *L, Py_ssize_t nargs) cdef int push_lua_arguments(LuaRuntime runtime, lua_State *L, tuple args, bint first_may_be_nil=True) except -1: + """Push Python objects in tuple into Lua stack + Preconditions: + LuaRuntime is locked + Postconditions: + Pushes each value of the Python tuple into the Lua stack + Returns number of pushed values + """ cdef int i, n cdef Py_ssize_t nargs cdef bint wrap_none = not first_may_be_nil @@ -1540,12 +1593,17 @@ cdef int push_lua_arguments(LuaRuntime runtime, lua_State *L, if nargs > INT_MAX: raise OverflowError("tuple too large to unpack") n = nargs - with runtime.stack(n, RESTORE_ON_ERROR): + check_lua_stack(L, n) + top = lua.lua_gettop(L) + try: for i, arg in enumerate(args): if not py_to_lua(runtime, L, arg, wrap_none=wrap_none): raise TypeError("failed to convert argument at index %d" % i) wrap_none = False return n + except: + lua.lua_settop(L, top) + raise else: return 0 @@ -2009,7 +2067,7 @@ cdef int py_args_with_gil(PyObject* runtime_obj, lua_State* L) with gil: try: runtime = runtime_obj pyargs = _PyArguments.__new__(_PyArguments) - pyargs.args, pyargs.kwargs = unpack_lua_table(runtime) + pyargs.args, pyargs.kwargs = unpack_lua_table(runtime, L) return py_to_lua_custom(runtime, L, pyargs, 0) except: try: runtime.store_raised_exception(L, b'error while calling python.args()') diff --git a/lupa/lua.pxd b/lupa/lua.pxd index 92a989e9..a1958747 100644 --- a/lupa/lua.pxd +++ b/lupa/lua.pxd @@ -93,7 +93,7 @@ cdef extern from "lua.h" nogil: int lua_iscfunction (lua_State *L, int idx) int lua_isuserdata (lua_State *L, int idx) int lua_type (lua_State *L, int idx) - char *lua_typename (lua_State *L, int tp) + const char *lua_typename (lua_State *L, int tp) int lua_equal (lua_State *L, int idx1, int idx2) int lua_rawequal (lua_State *L, int idx1, int idx2) From 2f984dc728885af40c57cd15b36654c27b916d6b Mon Sep 17 00:00:00 2001 From: guidanoli Date: Sat, 17 Jul 2021 01:14:19 -0300 Subject: [PATCH 4/8] Fixed LuaRuntime.table --- lupa/_lupa.pyx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index cd274aaf..a952b632 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -409,6 +409,7 @@ cdef class LuaRuntime: top = getstacktop(self, L, 5) try: # FIXME: how to check for failure? + lua.lua_newtable(L) for obj in args: if isinstance(obj, dict): for key, value in obj.iteritems(): @@ -1335,8 +1336,8 @@ cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) e Returns 0 if cannot convert Python object to Lua (handler not registered or failed) Returns 1 if the Python object was converted successfully and pushed onto the stack """ - top = lua.lua_gettop(L) check_lua_stack(L, 2) + top = lua.lua_gettop(L) try: lua.lua_pushlstring(L, LUPAOFH, len(LUPAOFH)) lua.lua_rawget(L, lua.LUA_REGISTRYINDEX) @@ -1999,6 +2000,7 @@ cdef int py_push_iterator(LuaRuntime runtime, lua_State* L, iterator, int type_f """Pushes iterator function, invariant state variable and control variable Preconditions: 3 extra slots in the Lua stack + LuaRuntime is locked Postconditions: Pushes py_iter_next, iterator and the control variable Returns the number of pushed values From f3ebc5bdb4b5d8ea27fb757f33d0d87b14b25d13 Mon Sep 17 00:00:00 2001 From: guidanoli Date: Sun, 18 Jul 2021 10:33:25 -0300 Subject: [PATCH 5/8] Remove getstack/setstack --- lupa/_lupa.pyx | 224 ++++++++++++++++++++++++++++--------------------- 1 file changed, 127 insertions(+), 97 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index a952b632..e36977a9 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -134,8 +134,10 @@ def lua_type(obj): L = lua_object._state runtime = lua_object._runtime assert runtime is not None - top = getstacktop(runtime, L, 1) + lock_runtime(runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 1) lua_object.push_lua_object(L) ltype = lua.lua_type(L, -1) if ltype == lua.LUA_TTABLE: @@ -150,7 +152,8 @@ def lua_type(obj): lua_type_name = lua.lua_typename(L, ltype) return lua_type_name if IS_PY2 else lua_type_name.decode('ascii') finally: - setstacktop(runtime, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(runtime) @cython.no_gc_clear @@ -345,8 +348,10 @@ cdef class LuaRuntime: lua_code = (lua_code).encode(self._source_encoding) L = self._state cdef size_t size - top = getstacktop(self, L, 1) + lock_runtime(self) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 1) status = lua.luaL_loadbuffer(L, lua_code, len(lua_code), b'') if status == 0: return py_from_lua(self, L, -1) @@ -355,7 +360,8 @@ cdef class LuaRuntime: error = err[:size] if self._encoding is None else err[:size].decode(self._encoding) raise LuaSyntaxError(error) finally: - setstacktop(self, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(self) def require(self, modulename): """Load a Lua library into the runtime. @@ -364,14 +370,17 @@ cdef class LuaRuntime: cdef lua_State *L = self._state if not isinstance(modulename, (bytes, unicode)): raise TypeError("modulename must be a string") - top = getstacktop(self, L, 1) + lock_runtime(self) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 1) lua.lua_getglobal(L, 'require') if lua.lua_isnil(L, -1): raise LuaError("require is not defined") return call_lua(self, L, (modulename,)) finally: - setstacktop(self, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(self) def globals(self): """Return the globals defined in this Lua runtime as a Lua @@ -379,12 +388,15 @@ cdef class LuaRuntime: """ assert self._state is not NULL cdef lua_State *L = self._state - top = getstacktop(self, L, 1) + lock_runtime(self) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 1) lua.lua_pushglobaltable(L) return py_from_lua(self, L, -1) finally: - setstacktop(self, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(self) def table(self, *items, **kwargs): """Create a new table with the provided items. Positional @@ -406,10 +418,12 @@ cdef class LuaRuntime: assert self._state is not NULL cdef lua_State *L = self._state cdef int i = 1 - top = getstacktop(self, L, 5) + lock_runtime(self) + old_top = lua.lua_gettop(L) try: - # FIXME: how to check for failure? + check_lua_stack(L, 5) lua.lua_newtable(L) + # FIXME: how to check for failure? for obj in args: if isinstance(obj, dict): for key, value in obj.iteritems(): @@ -440,7 +454,8 @@ cdef class LuaRuntime: i += 1 return py_from_lua(self, L, -1) finally: - setstacktop(self, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(self) def set_overflow_handler(self, overflow_handler): """Set the overflow handler function that is called on failures to pass large numbers to Lua. @@ -448,20 +463,29 @@ cdef class LuaRuntime: cdef lua_State *L = self._state if overflow_handler is not None and not callable(overflow_handler): raise ValueError("overflow_handler must be callable") - top = getstacktop(self, L, 2) + lock_runtime(self) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 2) lua.lua_pushlstring(L, LUPAOFH, len(LUPAOFH)) # key if not py_to_lua(self, L, overflow_handler): # key value raise LuaError("failed to convert overflow_handler") lua.lua_rawset(L, lua.LUA_REGISTRYINDEX) # finally: - setstacktop(self, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(self) @cython.final cdef int register_py_object(self, bytes cname, bytes pyname, object obj) except -1: - cdef lua_State *L = self._state # tbl - top = getstacktop(self, L, 4) + """Register Python object 'obj' in the registry with cname and + in the library on top of the stack with pyname + Preconditions: + runtime must be locked + """ + cdef lua_State *L = self._state + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 4) # tbl lua.lua_pushlstring(L, cname, len(cname)) # tbl cname py_to_lua_custom(self, L, obj, 0) # tbl cname obj lua.lua_pushlstring(L, pyname, len(pyname)) # tbl cname obj pyname @@ -470,7 +494,7 @@ cdef class LuaRuntime: lua.lua_rawset(L, lua.LUA_REGISTRYINDEX) # tbl return 0 finally: - setstacktop(self, L, top) + lua.lua_settop(L, old_top) @cython.final cdef int init_python_lib(self, bint register_eval, bint register_builtins) except -1: @@ -550,50 +574,13 @@ def unpacks_lua_table_method(meth): return wrapper -################################################################################ -# Lua stack management from Python - -cdef inline int getstacktop(LuaRuntime runtime, lua_State* L, int extra) except -1: - """Get stack top - Preconditions: - runtime is not None - L != NULL - extra >= 0 - Postconditions: - Acquires thread lock for runtime - Checks for extra slots in Lua stack - Returns Lua stack top (>= 0) - """ - if not lock_runtime(runtime): - raise RuntimeError("failed to acquire thread lock") - try: - check_lua_stack(L, extra) - return lua.lua_gettop(L) - except: - unlock_runtime(runtime) - raise - -cdef inline int setstacktop(LuaRuntime runtime, lua_State* L, int top): - """Set stack top - Preconditions: - runtime is not None - L != NULL - top >= 0 - Postconditions: - Releases thread lock for runtime - Sets Lua stack top to given index - """ - lua.lua_settop(L, top) - unlock_runtime(runtime) - - -cdef inline int check_lua_stack(lua_State* L, int extra) except -1: +cdef int check_lua_stack(lua_State* L, int extra) except -1: """Wrapper around lua_checkstack. On failure, a MemoryError is raised. """ assert extra >= 0 if not lua.lua_checkstack(L, extra): - raise MemoryError(f"could not reserve memory for {extra} free extra slots on the Lua stack") + raise MemoryError return 0 @@ -605,9 +592,12 @@ cdef int get_object_length_from_lua(lua_State* L) nogil: cdef Py_ssize_t get_object_length(LuaRuntime runtime, lua_State* L, int index) except -1: """Obtains the length of the object at the given valid index. - - If Lua raises an error, a LuaError is raised. - If the object length doesn't fit into Py_ssize_t, an OverflowError is raised. + Preconditions: + runtime must be locked + index must be valid + Exceptions: + LuaError if Lua raises an error + OverflowError if length doesn't fit in a Py_ssize_t """ cdef int result cdef size_t length @@ -627,6 +617,8 @@ cdef Py_ssize_t get_object_length(LuaRuntime runtime, lua_State* L, int index) e cdef tuple unpack_lua_table(LuaRuntime runtime, lua_State* L): """Unpacks the table at the top of the stack into a tuple of positional arguments and a dictionary of keyword arguments. + Preconditions: + runtime must be locked """ assert runtime is not None cdef tuple args @@ -634,7 +626,8 @@ cdef tuple unpack_lua_table(LuaRuntime runtime, lua_State* L): cdef bytes source_encoding = runtime._source_encoding cdef int old_top cdef Py_ssize_t index, length - top = getstacktop(runtime, L, 2) + check_lua_stack(L, 2) + old_top = lua.lua_gettop(L) try: length = get_object_length(runtime, L, -1) args = cpython.tuple.PyTuple_New(length) @@ -660,7 +653,7 @@ cdef tuple unpack_lua_table(LuaRuntime runtime, lua_State* L): lua.lua_pop(L, 1) # key return args, kwargs finally: - setstacktop(runtime, L, top) + lua.lua_settop(L, old_top) cdef tuple _fix_args_kwargs(tuple args): @@ -677,14 +670,17 @@ cdef tuple _fix_args_kwargs(tuple args): return args, {} cdef _LuaTable table = <_LuaTable>arg - runtime = table._runtime - L = table._state - top = getstacktop(runtime, L, 1) + cdef LuaRuntime runtime = table._runtime + cdef lua_State* L = table._state + lock_runtime(runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 1) table.push_lua_object(L) return unpack_lua_table(runtime, L) finally: - setstacktop(runtime, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(runtime) ################################################################################ @@ -717,8 +713,9 @@ cdef class _LuaObject: if self._runtime is None: return cdef lua_State* L = self._state - if lock_runtime(self._runtime): - lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._ref) + locked = lock_runtime(self._runtime) + lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._ref) + if locked: unlock_runtime(self._runtime) @cython.final @@ -756,12 +753,15 @@ cdef class _LuaObject: cdef Py_ssize_t _len(self) except -1: assert self._runtime is not None cdef lua_State* L = self._state - top = getstacktop(self._runtime, L, 2) + lock_runtime(self._runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 1) self.push_lua_object(L) return get_object_length(self._runtime, L, -1) finally: - setstacktop(self._runtime, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(self._runtime) def __nonzero__(self): return True @@ -774,12 +774,15 @@ cdef class _LuaObject: assert self._runtime is not None cdef lua_State* L = self._state cdef bytes encoding = self._runtime._encoding or b'UTF-8' - top = getstacktop(self._runtime, L, 1) + lock_runtime(self._runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 1) self.push_lua_object(L) return lua_object_repr(L, encoding) finally: - setstacktop(self._runtime, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(self._runtime) def __str__(self): assert self._runtime is not None @@ -787,8 +790,10 @@ cdef class _LuaObject: cdef const char *string cdef size_t size = 0 cdef bytes encoding = self._runtime._encoding or b'UTF-8' - top = getstacktop(self._runtime, L, 2) + lock_runtime(self._runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 2) self.push_lua_object(L) # obj if lua.luaL_getmetafield(L, -1, "__tostring"): # obj tostr lua.lua_insert(L, -2) # tostr obj @@ -799,9 +804,10 @@ cdef class _LuaObject: return string[:size].decode(encoding) except UnicodeDecodeError: return string[:size].decode('ISO-8859-1') + return repr(self) finally: - setstacktop(self._runtime, L, top) - return repr(self) + lua.lua_settop(L, old_top) + unlock_runtime(self._runtime) def __getattr__(self, name): assert self._runtime is not None @@ -820,9 +826,10 @@ cdef class _LuaObject: @cython.final cdef _getitem(self, name, bint is_attr_access): cdef lua_State* L = self._state - cdef int lua_type - top = getstacktop(self._runtime, L, 3) + lock_runtime(self._runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 3) # table[nil] fails, so map None -> python.none for Lua tables lua.lua_pushcfunction(L, get_from_lua_table) # func self.push_lua_object(L) # func obj @@ -834,7 +841,8 @@ cdef class _LuaObject: py_to_lua(self._runtime, L, name, wrap_none=(lua_type == lua.LUA_TTABLE)) # func obj key return execute_lua_call(self._runtime, L, 2) # obj[key] finally: - setstacktop(self._runtime, L, top) + lua.lua_settop(L, old_top) # + unlock_runtime(self._runtime) cdef _LuaObject new_lua_object(LuaRuntime runtime, lua_State* L, int n): @@ -915,8 +923,10 @@ cdef class _LuaTable(_LuaObject): @cython.final cdef int _setitem(self, name, value) except -1: cdef lua_State* L = self._state - top = getstacktop(self._runtime, L, 3) + lock_runtime(self._runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 3) self.push_lua_object(L) # table[nil] fails, so map None -> python.none for Lua tables py_to_lua(self._runtime, L, name, wrap_none=True) @@ -924,7 +934,8 @@ cdef class _LuaTable(_LuaObject): lua.lua_settable(L, -3) return 0 finally: - setstacktop(self._runtime, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(self._runtime) def __delattr__(self, item): assert self._runtime is not None @@ -944,14 +955,18 @@ cdef class _LuaTable(_LuaObject): @cython.final cdef _delitem(self, name): cdef lua_State* L = self._state - top = getstacktop(self._runtime, L, 1) + lock_runtime(self._runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 3) self.push_lua_object(L) py_to_lua(self._runtime, L, name, wrap_none=True) lua.lua_pushnil(L) lua.lua_settable(L, -3) finally: - setstacktop(self._runtime, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(self._runtime) + cdef _LuaTable new_lua_table(LuaRuntime runtime, lua_State* L, int n): cdef _LuaTable obj = _LuaTable.__new__(_LuaTable) @@ -972,8 +987,10 @@ cdef class _LuaFunction(_LuaObject): cdef lua_State* L = self._state cdef lua_State* co cdef _LuaThread thread - top = getstacktop(self._runtime, L, 3) + lock_runtime(self._runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 3) self.push_lua_object(L) if not lua.lua_isfunction(L, -1) or lua.lua_iscfunction(L, -1): raise TypeError("Lua object is not a function") @@ -987,7 +1004,8 @@ cdef class _LuaFunction(_LuaObject): thread._arguments = args # always a tuple, not None ! return thread finally: - setstacktop(self._runtime, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(self._runtime) cdef _LuaFunction new_lua_function(LuaRuntime runtime, lua_State* L, int n): cdef _LuaFunction obj = _LuaFunction.__new__(_LuaFunction) @@ -1086,8 +1104,10 @@ cdef object resume_lua_thread(_LuaThread thread, tuple args): cdef lua_State* co = thread._co_state cdef lua_State* L = thread._state cdef int status, i, nargs = 0, nres = 0 - top = getstacktop(thread._runtime, L, 1) + lock_runtime(thread._runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 1) if lua.lua_status(co) == 0 and lua.lua_gettop(co) == 0: # already terminated raise StopIteration @@ -1111,7 +1131,9 @@ cdef object resume_lua_thread(_LuaThread thread, tuple args): lua.lua_xmove(co, L, nres) return unpack_lua_results(thread._runtime, L) finally: - setstacktop(thread._runtime, L, top) + # FIXME: check that coroutine state is OK in case of errors? + lua.lua_settop(L, old_top) + unlock_runtime(thread._runtime) cdef enum: @@ -1143,9 +1165,11 @@ cdef class _LuaIter: if self._runtime is None: return cdef lua_State* L = self._state - if L is not NULL and self._refiter and lock_runtime(self._runtime): + if L is not NULL and self._refiter: + locked = lock_runtime(self._runtime) lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._refiter) - unlock_runtime(self._runtime) + if locked: + unlock_runtime(self._runtime) def __repr__(self): return u"LuaIter(%r)" % (self._obj) @@ -1157,8 +1181,10 @@ cdef class _LuaIter: if self._obj is None: raise StopIteration cdef lua_State* L = self._obj._state - top = getstacktop(self._runtime, L, 3) + lock_runtime(self._runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 3) if self._obj is None: raise StopIteration # iterable object @@ -1193,9 +1219,10 @@ cdef class _LuaIter: lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._refiter) self._refiter = 0 self._obj = None + raise StopIteration finally: - setstacktop(self._runtime, L, top) - raise StopIteration + lua.lua_settop(L, old_top) + unlock_runtime(self._runtime) # type conversions and protocol adaptations @@ -1337,7 +1364,7 @@ cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) e Returns 1 if the Python object was converted successfully and pushed onto the stack """ check_lua_stack(L, 2) - top = lua.lua_gettop(L) + old_top = lua.lua_gettop(L) try: lua.lua_pushlstring(L, LUPAOFH, len(LUPAOFH)) lua.lua_rawget(L, lua.LUA_REGISTRYINDEX) @@ -1350,7 +1377,7 @@ cdef int py_to_lua_handle_overflow(LuaRuntime runtime, lua_State *L, object o) e return 0 return 1 except: - lua.lua_settop(L, top) + lua.lua_settop(L, old_top) raise cdef int py_to_lua(LuaRuntime runtime, lua_State *L, object o, bint wrap_none=False) except -1: @@ -1436,7 +1463,7 @@ cdef bint py_to_lua_custom(LuaRuntime runtime, lua_State *L, object o, int type_ refkey = build_pyref_key(o, type_flags) cdef _PyReference pyref check_lua_stack(L, 3) - top = lua.lua_gettop(L) + old_top = lua.lua_gettop(L) try: lua.lua_getfield(L, lua.LUA_REGISTRYINDEX, PYREFST) # tbl @@ -1471,7 +1498,7 @@ cdef bint py_to_lua_custom(LuaRuntime runtime, lua_State *L, object o, int type_ return 1 # values pushed except: - lua.lua_settop(L, top) + lua.lua_settop(L, old_top) raise cdef inline int _isascii(unsigned char* s): @@ -1530,14 +1557,17 @@ cdef build_lua_error_message(LuaRuntime runtime, lua_State* L, unicode err_messa cdef run_lua(LuaRuntime runtime, bytes lua_code, tuple args): """Run Lua code with arguments""" cdef lua_State* L = runtime._state - top = getstacktop(runtime, L, 1) + lock_runtime(runtime) + old_top = lua.lua_gettop(L) try: + check_lua_stack(L, 1) if lua.luaL_loadbuffer(L, lua_code, len(lua_code), ''): raise LuaSyntaxError(build_lua_error_message( runtime, L, u"error loading code: %s", -1)) return call_lua(runtime, L, args) finally: - setstacktop(runtime, L, top) + lua.lua_settop(L, old_top) + unlock_runtime(runtime) cdef call_lua(LuaRuntime runtime, lua_State *L, tuple args): """Call function on top of the stack with args @@ -1595,7 +1625,7 @@ cdef int push_lua_arguments(LuaRuntime runtime, lua_State *L, raise OverflowError("tuple too large to unpack") n = nargs check_lua_stack(L, n) - top = lua.lua_gettop(L) + old_top = lua.lua_gettop(L) try: for i, arg in enumerate(args): if not py_to_lua(runtime, L, arg, wrap_none=wrap_none): @@ -1603,7 +1633,7 @@ cdef int push_lua_arguments(LuaRuntime runtime, lua_State *L, wrap_none = False return n except: - lua.lua_settop(L, top) + lua.lua_settop(L, old_top) raise else: return 0 From 9e7dfc2a342844388d2dcf4eeeace2ee52a0f005 Mon Sep 17 00:00:00 2001 From: guidanoli Date: Sun, 18 Jul 2021 10:45:42 -0300 Subject: [PATCH 6/8] Minimized overall diff --- lupa/_lupa.pyx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index e36977a9..20af2a5b 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -131,11 +131,11 @@ def lua_type(obj): if not isinstance(obj, _LuaObject): return None lua_object = <_LuaObject>obj + assert lua_object._runtime is not None + lock_runtime(lua_object._runtime) L = lua_object._state - runtime = lua_object._runtime - assert runtime is not None - lock_runtime(runtime) old_top = lua.lua_gettop(L) + cdef const char* lua_type_name try: check_lua_stack(L, 1) lua_object.push_lua_object(L) @@ -153,7 +153,7 @@ def lua_type(obj): return lua_type_name if IS_PY2 else lua_type_name.decode('ascii') finally: lua.lua_settop(L, old_top) - unlock_runtime(runtime) + unlock_runtime(lua_object._runtime) @cython.no_gc_clear @@ -347,9 +347,9 @@ cdef class LuaRuntime: if isinstance(lua_code, unicode): lua_code = (lua_code).encode(self._source_encoding) L = self._state - cdef size_t size lock_runtime(self) old_top = lua.lua_gettop(L) + cdef size_t size try: check_lua_stack(L, 1) status = lua.luaL_loadbuffer(L, lua_code, len(lua_code), b'') From 8f9d9c360de02c4f36825ef1f6adbae43c3bbadc Mon Sep 17 00:00:00 2001 From: guidanoli Date: Sun, 18 Jul 2021 13:11:24 -0300 Subject: [PATCH 7/8] Assertions, LUA_REFNIL and LUA_NOREF --- lupa/_lupa.pyx | 67 +++++++++++++++++++++++++++++++++------------- lupa/lua.pxd | 5 ++++ lupa/tests/test.py | 27 +++++++++++++++++++ 3 files changed, 81 insertions(+), 18 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 20af2a5b..1cf7d5eb 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -132,6 +132,7 @@ def lua_type(obj): return None lua_object = <_LuaObject>obj assert lua_object._runtime is not None + assert lua_object._runtime._state is not NULL lock_runtime(lua_object._runtime) L = lua_object._state old_top = lua.lua_gettop(L) @@ -291,6 +292,7 @@ cdef class LuaRuntime: """ The Lua runtime/language version as tuple, e.g. (5, 3) for Lua 5.3. """ + assert self._state is not NULL cdef int version = lua.read_lua_version(self._state) return (version // 100, version % 100) @@ -343,6 +345,7 @@ cdef class LuaRuntime: def compile(self, lua_code): """Compile a Lua program into a callable Lua function. """ + assert self._state is not NULL cdef const char *err if isinstance(lua_code, unicode): lua_code = (lua_code).encode(self._source_encoding) @@ -460,6 +463,7 @@ cdef class LuaRuntime: def set_overflow_handler(self, overflow_handler): """Set the overflow handler function that is called on failures to pass large numbers to Lua. """ + assert self._state is not NULL cdef lua_State *L = self._state if overflow_handler is not None and not callable(overflow_handler): raise ValueError("overflow_handler must be callable") @@ -670,6 +674,8 @@ cdef tuple _fix_args_kwargs(tuple args): return args, {} cdef _LuaTable table = <_LuaTable>arg + assert table._runtime is not None + assert table._runtime._state is not NULL cdef LuaRuntime runtime = table._runtime cdef lua_State* L = table._state lock_runtime(runtime) @@ -706,6 +712,9 @@ cdef class _LuaObject: cdef lua_State* _state cdef int _ref + def __cinit__(self): + self._ref = lua.LUA_NOREF + def __init__(self): raise TypeError("Type cannot be instantiated manually") @@ -713,10 +722,14 @@ cdef class _LuaObject: if self._runtime is None: return cdef lua_State* L = self._state - locked = lock_runtime(self._runtime) - lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._ref) - if locked: - unlock_runtime(self._runtime) + if L is not NULL and self._ref != lua.LUA_NOREF: + locked = lock_runtime(self._runtime) + lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._ref) + self._ref = lua.LUA_NOREF + runtime = self._runtime + self._runtime = None + if locked: + unlock_runtime(runtime) @cython.final cdef inline int push_lua_object(self, lua_State* L) except -1: @@ -727,6 +740,8 @@ cdef class _LuaObject: Postconditions: Lua object (not nil) is pushed onto of the stack """ + if self._ref == lua.LUA_NOREF: + raise LuaError("lost reference") lua.lua_rawgeti(L, lua.LUA_REGISTRYINDEX, self._ref) if lua.lua_isnil(L, -1): lua.lua_pop(L, 1) @@ -797,14 +812,19 @@ cdef class _LuaObject: self.push_lua_object(L) # obj if lua.luaL_getmetafield(L, -1, "__tostring"): # obj tostr lua.lua_insert(L, -2) # tostr obj - if lua.lua_pcall(L, 1, 1, 0) == 0: # str + status = lua.lua_pcall(L, 1, 1, 0) + if status == 0: # str string = lua.lua_tolstring(L, -1, &size) - if string: - try: - return string[:size].decode(encoding) - except UnicodeDecodeError: - return string[:size].decode('ISO-8859-1') - return repr(self) + if string is NULL: + raise TypeError("__tostring returned non-string object") + try: + return string[:size].decode(encoding) + except UnicodeDecodeError: + return string[:size].decode('ISO-8859-1') + else: + raise_lua_error(self._runtime, L, status) + else: + return lua_object_repr(L, encoding) finally: lua.lua_settop(L, old_top) unlock_runtime(self._runtime) @@ -825,6 +845,7 @@ cdef class _LuaObject: @cython.final cdef _getitem(self, name, bint is_attr_access): + assert self._runtime is not None cdef lua_State* L = self._state lock_runtime(self._runtime) old_top = lua.lua_gettop(L) @@ -922,6 +943,7 @@ cdef class _LuaTable(_LuaObject): @cython.final cdef int _setitem(self, name, value) except -1: + assert self._runtime is not None cdef lua_State* L = self._state lock_runtime(self._runtime) old_top = lua.lua_gettop(L) @@ -954,6 +976,7 @@ cdef class _LuaTable(_LuaObject): @cython.final cdef _delitem(self, name): + assert self._runtime is not None cdef lua_State* L = self._state lock_runtime(self._runtime) old_top = lua.lua_gettop(L) @@ -1104,6 +1127,7 @@ cdef object resume_lua_thread(_LuaThread thread, tuple args): cdef lua_State* co = thread._co_state cdef lua_State* L = thread._state cdef int status, i, nargs = 0, nres = 0 + assert thread._runtime is not None lock_runtime(thread._runtime) old_top = lua.lua_gettop(L) try: @@ -1158,18 +1182,21 @@ cdef class _LuaIter: self._runtime = obj._runtime self._obj = obj self._state = obj._state - self._refiter = 0 + self._refiter = lua.LUA_REFNIL self._what = what def __dealloc__(self): if self._runtime is None: return cdef lua_State* L = self._state - if L is not NULL and self._refiter: + if L is not NULL and self._refiter != lua.LUA_NOREF: locked = lock_runtime(self._runtime) lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._refiter) + self._refiter = lua.LUA_NOREF + runtime = self._runtime + self._runtime = None if locked: - unlock_runtime(self._runtime) + unlock_runtime(runtime) def __repr__(self): return u"LuaIter(%r)" % (self._obj) @@ -1181,6 +1208,7 @@ cdef class _LuaIter: if self._obj is None: raise StopIteration cdef lua_State* L = self._obj._state + assert self._runtime is not None lock_runtime(self._runtime) old_top = lua.lua_gettop(L) try: @@ -1191,7 +1219,10 @@ cdef class _LuaIter: self._obj.push_lua_object(L) if not lua.lua_istable(L, -1): raise TypeError("cannot iterate over non-table (found %r)" % self._obj) - if not self._refiter: + if self._refiter == lua.LUA_NOREF: + # no key + raise StopIteration + elif self._refiter == lua.LUA_REFNIL: # initial key lua.lua_pushnil(L) else: @@ -1209,15 +1240,15 @@ cdef class _LuaIter: # pop value lua.lua_pop(L, 1) # pop and store key - if not self._refiter: + if self._refiter == lua.LUA_REFNIL: self._refiter = lua.luaL_ref(L, lua.LUA_REGISTRYINDEX) else: lua.lua_rawseti(L, lua.LUA_REGISTRYINDEX, self._refiter) return retval # iteration done, clean up - if self._refiter: + if self._refiter != lua.LUA_REFNIL: lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._refiter) - self._refiter = 0 + self._refiter = lua.LUA_NOREF self._obj = None raise StopIteration finally: diff --git a/lupa/lua.pxd b/lupa/lua.pxd index a1958747..35d4b6e1 100644 --- a/lupa/lua.pxd +++ b/lupa/lua.pxd @@ -278,6 +278,11 @@ cdef extern from "lauxlib.h" nogil: enum: LUA_ERRFILE # (LUA_ERRERR+1) + # pre-defined references + enum: + LUA_NOREF # -2 + LUA_REFNIL # -1 + ctypedef struct luaL_Reg: char *name lua_CFunction func diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 8d95ab51..d8706919 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -417,6 +417,16 @@ def test_error_iter_function(self): func = self.lua.eval('function() return 1 end') self.assertRaises(TypeError, list, func) + def test_iter_table_exaust(self): + table = self.lua.table(1, 2, 3) + tableiter = iter(table) + self.assertEqual(next(tableiter), 1) + self.assertEqual(next(tableiter), 2) + self.assertEqual(next(tableiter), 3) + self.assertRaises(StopIteration, next, tableiter) + self.assertRaises(StopIteration, next, tableiter) + self.assertRaises(StopIteration, next, tableiter) + def test_string_values(self): function = self.lua.eval('function(s) return s .. "abc" end') self.assertEqual('ABCabc', function('ABC')) @@ -2946,6 +2956,23 @@ def test_functions(self): self.testmissingref({}, lupa.as_attrgetter) # attribute getter protocol +################################################################################ +# test Lua object __str__ method + +class TestLuaObjectString(SetupLuaRuntimeMixin, unittest.TestCase): + def test_normal_string(self): + self.assertIn('Lua table', str(self.lua.eval('{}'))) + self.assertIn('Lua function', str(self.lua.eval('print'))) + self.assertIn('Lua thread', str(self.lua.execute('local t = coroutine.create(function() end); coroutine.resume(t); return t'))) + + def test_bad_tostring(self): + self.assertRaisesRegex(TypeError, '__tostring returned non-string object', + str, self.lua.eval('setmetatable({}, {__tostring = function() end})')) + + def test_tostring_err(self): + self.assertRaises(lupa.LuaError, str, self.lua.eval('setmetatable({}, {__tostring = function() error() end})')) + + if __name__ == '__main__': def print_version(): version = lupa.LuaRuntime().lua_implementation From 6d6c0e283b3b7b5a4ecda432a506db22d6c1611f Mon Sep 17 00:00:00 2001 From: guidanoli Date: Wed, 21 Jul 2021 17:07:34 -0300 Subject: [PATCH 8/8] Minimize diff --- lupa/_lupa.pyx | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 1cf7d5eb..9af27175 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -655,9 +655,9 @@ cdef tuple unpack_lua_table(LuaRuntime runtime, lua_State* L): else: raise TypeError("table key is neither an integer nor a string") lua.lua_pop(L, 1) # key - return args, kwargs finally: lua.lua_settop(L, old_top) + return args, kwargs cdef tuple _fix_args_kwargs(tuple args): @@ -851,7 +851,6 @@ cdef class _LuaObject: old_top = lua.lua_gettop(L) try: check_lua_stack(L, 3) - # table[nil] fails, so map None -> python.none for Lua tables lua.lua_pushcfunction(L, get_from_lua_table) # func self.push_lua_object(L) # func obj lua_type = lua.lua_type(L, -1) @@ -954,10 +953,10 @@ cdef class _LuaTable(_LuaObject): py_to_lua(self._runtime, L, name, wrap_none=True) py_to_lua(self._runtime, L, value) lua.lua_settable(L, -3) - return 0 finally: lua.lua_settop(L, old_top) unlock_runtime(self._runtime) + return 0 def __delattr__(self, item): assert self._runtime is not None @@ -1250,10 +1249,10 @@ cdef class _LuaIter: lua.luaL_unref(L, lua.LUA_REGISTRYINDEX, self._refiter) self._refiter = lua.LUA_NOREF self._obj = None - raise StopIteration finally: lua.lua_settop(L, old_top) unlock_runtime(self._runtime) + raise StopIteration # type conversions and protocol adaptations @@ -1496,16 +1495,15 @@ cdef bint py_to_lua_custom(LuaRuntime runtime, lua_State *L, object o, int type_ check_lua_stack(L, 3) old_top = lua.lua_gettop(L) try: - lua.lua_getfield(L, lua.LUA_REGISTRYINDEX, PYREFST) # tbl - # check if Python object is already referenced in Lua + lua.lua_getfield(L, lua.LUA_REGISTRYINDEX, PYREFST) # tbl if refkey in runtime._pyrefs_in_lua: pyref = <_PyReference>runtime._pyrefs_in_lua[refkey] lua.lua_rawgeti(L, -1, pyref._ref) # tbl udata py_obj = lua.lua_touserdata(L, -1) if py_obj: lua.lua_remove(L, -2) # udata - return 1 # values pushed + return 1 # values pushed lua.lua_pop(L, 1) # tbl # create new wrapper for Python object @@ -1526,12 +1524,12 @@ cdef bint py_to_lua_custom(LuaRuntime runtime, lua_State *L, object o, int type_ # now, we store an owned reference in "runtime._pyrefs_in_lua" to keep it visible to Python # and a borrowed reference in "py_obj.obj" for access from Lua runtime._pyrefs_in_lua[refkey] = pyref - - return 1 # values pushed except: lua.lua_settop(L, old_top) raise + return 1 # values pushed + cdef inline int _isascii(unsigned char* s): cdef unsigned char c = 0 while s[0]: