From 2669b6346b574d502cfbbcf2b0f79d3d3bf32d0b Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 14:00:49 +0200 Subject: [PATCH 01/55] add memory limit --- lupa/_lupa.pyx | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 9af27175..c08fbbed 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -9,6 +9,7 @@ from __future__ import absolute_import cimport cython from libc.string cimport strlen, strchr +from libc.stdlib cimport malloc, free, realloc from lupa cimport lua from .lua cimport lua_State @@ -246,8 +247,18 @@ cdef class LuaRuntime: def __cinit__(self, encoding='UTF-8', source_encoding=None, attribute_filter=None, attribute_handlers=None, bint register_eval=True, bint unpack_returned_tuples=False, - bint register_builtins=True, overflow_handler=None): - cdef lua_State* L = lua.luaL_newstate() + bint register_builtins=True, overflow_handler=None, + max_memory=None): + cdef lua_State* L + cdef size_t* ud + if max_memory is None: + L = lua.luaL_newstate() + else: + ud = malloc(sizeof(size_t)) + if ud is NULL: + raise LuaError("Failed to allocate Lua runtime memory limiter") + ud[0] = max_memory + L = lua.lua_newstate(&_lua_alloc_restricted, ud) if L is NULL: raise LuaError("Failed to initialise Lua runtime") self._state = L @@ -1609,6 +1620,31 @@ cdef call_lua(LuaRuntime runtime, lua_State *L, tuple args): push_lua_arguments(runtime, L, args) return execute_lua_call(runtime, L, len(args)) +# adapted from https://stackoverflow.com/a/9672205 +cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize) nogil: + cdef size_t* left = ud + + if ptr is NULL: + # : + # When ptr is NULL, osize encodes the kind of object that Lua is + # allocating. + # Since we don’t care about that, just mark it as 0. + osize = 0 + + if nsize == 0: + free(ptr) + left[0] += osize # add old size to available memory + return NULL + elif nsize == osize: + return ptr + else: + if nsize > osize and left[0] < nsize - osize: # reached the limit + return NULL + ptr = realloc(ptr, nsize) + if ptr: # reallocation successful? + left[0] -= nsize + osize + return ptr + cdef object execute_lua_call(LuaRuntime runtime, lua_State *L, Py_ssize_t nargs): cdef int result_status cdef object result From b58a05e2145de1d1b9fc30ff6d366247599e04ab Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 15:06:09 +0200 Subject: [PATCH 02/55] add panic handler --- lupa/_lupa.pyx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index c08fbbed..a7effc25 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -10,6 +10,7 @@ cimport cython from libc.string cimport strlen, strchr from libc.stdlib cimport malloc, free, realloc +from libc.stdio cimport fprintf, stderr, fflush from lupa cimport lua from .lua cimport lua_State @@ -287,9 +288,11 @@ cdef class LuaRuntime: raise ValueError("attribute_filter and attribute_handlers are mutually exclusive") self._attribute_getter, self._attribute_setter = getter, setter + if max_memory is not None: + # luaL_newstate already registers a panic handler + lua.lua_atpanic(L, &_lua_panic) lua.luaL_openlibs(L) self.init_python_lib(register_eval, register_builtins) - lua.lua_atpanic(L, 1) self.set_overflow_handler(overflow_handler) @@ -1645,6 +1648,15 @@ cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize left[0] -= nsize + osize return ptr +cdef int _lua_panic(lua_State *L) nogil: + cdef char* msg = lua.lua_tostring(L, -1) + if msg == NULL: + msg = "error object is not a string" + cdef char* message = "PANIC: unprotected error in call to Lua API (%s)\n" + fprintf(stderr, message, msg) + fflush(stderr) + return 0 # return to Lua to abort + cdef object execute_lua_call(LuaRuntime runtime, lua_State *L, Py_ssize_t nargs): cdef int result_status cdef object result From bbfaf16b14d0d25a7dcb28d9eb8c50b81fdd3756 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 15:06:38 +0200 Subject: [PATCH 03/55] allow accessing and changing max memory --- lupa/_lupa.pyx | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index a7effc25..99dbee86 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -244,6 +244,8 @@ cdef class LuaRuntime: cdef object _attribute_getter cdef object _attribute_setter cdef bint _unpack_returned_tuples + cdef size_t _max_memory + cdef size_t* _memory_left def __cinit__(self, encoding='UTF-8', source_encoding=None, attribute_filter=None, attribute_handlers=None, @@ -251,15 +253,15 @@ cdef class LuaRuntime: bint register_builtins=True, overflow_handler=None, max_memory=None): cdef lua_State* L - cdef size_t* ud + cdef size_t* memory_left if max_memory is None: L = lua.luaL_newstate() else: - ud = malloc(sizeof(size_t)) - if ud is NULL: + memory_left = malloc(sizeof(size_t)) + if memory_left is NULL: raise LuaError("Failed to allocate Lua runtime memory limiter") - ud[0] = max_memory - L = lua.lua_newstate(&_lua_alloc_restricted, ud) + memory_left[0] = max_memory + L = lua.lua_newstate(&_lua_alloc_restricted, memory_left) if L is NULL: raise LuaError("Failed to initialise Lua runtime") self._state = L @@ -271,6 +273,8 @@ cdef class LuaRuntime: raise ValueError("attribute_filter must be callable") self._attribute_filter = attribute_filter self._unpack_returned_tuples = unpack_returned_tuples + self._max_memory = max_memory + self._memory_left = memory_left if attribute_handlers: raise_error = False @@ -301,6 +305,10 @@ cdef class LuaRuntime: lua.lua_close(self._state) self._state = NULL + @property + def max_memory(self): + return self._max_memory + @property def lua_version(self): """ @@ -474,6 +482,13 @@ cdef class LuaRuntime: lua.lua_settop(L, old_top) unlock_runtime(self) + def set_max_memory(self, max_memory): + if self._max_memory is None: + raise RuntimeError("max_memory must be set on LuaRuntime creation") + used = self._max_memory - self._memory_left[0] + self._memory_left[0] = max_memory - used + self._max_memory = max_memory + def set_overflow_handler(self, overflow_handler): """Set the overflow handler function that is called on failures to pass large numbers to Lua. """ From 3d5cd61fb07fdf2386c03339e51217c78492a726 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 15:16:36 +0200 Subject: [PATCH 04/55] add docs --- CHANGES.rst | 3 +++ lupa/_lupa.pyx | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index b1905270..71f61fab 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,9 @@ Lupa change log 2.0a1 (2022-??-??) ------------------ +* GH#211: Option to limit memory usage + (patch by Leo Developer) + * GH#171: Python references in Lua are now more safely reference counted to prevent garbage collection glitches. (patch by Guilherme Dantas) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 99dbee86..9da6cca3 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -219,6 +219,10 @@ cdef class LuaRuntime: Normally, it should return the now well-behaved object that can be converted/wrapped to a Lua type. If the object cannot be precisely represented in Lua, it should raise an ``OverflowError``. + + * ``max_memory``: max memory usage this LuaRuntime can use in bytes. + Values below 40kb may cause lua panics. + (default: None, new in Lupa 2.0) Example usage:: From f57420628c89e359325276c8c63d3997fb47bbed Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 15:52:09 +0200 Subject: [PATCH 05/55] add LuaMemoryError --- lupa/_lupa.pyx | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 9da6cca3..1dc8da20 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -123,6 +123,11 @@ class LuaSyntaxError(LuaError): """ +class LuaMemoryError(LuaError): + """Memory error in Lua code. + """ + + def lua_type(obj): """ Return the Lua type name of a wrapped object as string, as provided @@ -387,6 +392,8 @@ cdef class LuaRuntime: else: err = lua.lua_tolstring(L, -1, &size) error = err[:size] if self._encoding is None else err[:size].decode(self._encoding) + if error == (b"not enough memory" if self._encoding is None else "not enough memory"): + raise LuaMemoryError(error) raise LuaSyntaxError(error) finally: lua.lua_settop(L, old_top) @@ -617,7 +624,7 @@ cdef int check_lua_stack(lua_State* L, int extra) except -1: """ assert extra >= 0 if not lua.lua_checkstack(L, extra): - raise MemoryError + raise LuaMemoryError return 0 @@ -1591,11 +1598,11 @@ cdef int raise_lua_error(LuaRuntime runtime, lua_State* L, int result) except -1 if result == 0: return 0 elif result == lua.LUA_ERRMEM: - raise MemoryError() + raise LuaMemoryError() else: raise LuaError(build_lua_error_message(runtime, L, None, -1)) -cdef build_lua_error_message(LuaRuntime runtime, lua_State* L, unicode err_message, int n): +cdef build_lua_error_message(LuaRuntime runtime, lua_State* L, int n): """Removes the string at the given stack index ``n`` to build an error message. If ``err_message`` is provided, it is used as a %-format string to build the error message. """ @@ -1609,10 +1616,7 @@ cdef build_lua_error_message(LuaRuntime runtime, lua_State* L, unicode err_messa else: py_ustring = s[:size].decode('ISO-8859-1') lua.lua_remove(L, n) - if err_message is None: - return py_ustring - else: - return err_message % py_ustring + return py_ustring # calling into Lua @@ -1624,8 +1628,10 @@ cdef run_lua(LuaRuntime runtime, bytes lua_code, tuple args): try: check_lua_stack(L, 1) if lua.luaL_loadbuffer(L, lua_code, len(lua_code), ''): - raise LuaSyntaxError(build_lua_error_message( - runtime, L, u"error loading code: %s", -1)) + error = build_lua_error_message(runtiem, L, -1) + if error == "not enough memory": + raise LuaMemoryError(error) + raise LuaSyntaxError(u"error loading code: %s" % error) return call_lua(runtime, L, args) finally: lua.lua_settop(L, old_top) From b125d1ec3038d10463004c7ecdbbdb3c2dd943f0 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 16:09:40 +0200 Subject: [PATCH 06/55] fix compiler warning --- lupa/_lupa.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 1dc8da20..0753e835 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -1674,7 +1674,7 @@ cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize return ptr cdef int _lua_panic(lua_State *L) nogil: - cdef char* msg = lua.lua_tostring(L, -1) + cdef const char* msg = lua.lua_tostring(L, -1) if msg == NULL: msg = "error object is not a string" cdef char* message = "PANIC: unprotected error in call to Lua API (%s)\n" From bb69bcc88443f0d639ab3d0fb1122042d7e09c5f Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 16:10:10 +0200 Subject: [PATCH 07/55] use 0 to denote unlimited memory and always use our allocator --- lupa/_lupa.pyx | 60 +++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 0753e835..b2af8ed6 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -60,7 +60,7 @@ from functools import wraps __all__ = ['LUA_VERSION', 'LUA_MAXINTEGER', 'LUA_MININTEGER', - 'LuaRuntime', 'LuaError', 'LuaSyntaxError', + 'LuaRuntime', 'LuaError', 'LuaSyntaxError', 'LuaMemoryError', 'as_itemgetter', 'as_attrgetter', 'lua_type', 'unpacks_lua_table', 'unpacks_lua_table_method'] @@ -226,8 +226,8 @@ cdef class LuaRuntime: represented in Lua, it should raise an ``OverflowError``. * ``max_memory``: max memory usage this LuaRuntime can use in bytes. - Values below 40kb may cause lua panics. - (default: None, new in Lupa 2.0) + Builtins are not counted towards this limit. + (default: 0, new in Lupa 2.0) Example usage:: @@ -254,25 +254,21 @@ cdef class LuaRuntime: cdef object _attribute_setter cdef bint _unpack_returned_tuples cdef size_t _max_memory - cdef size_t* _memory_left + cdef size_t _memory_left def __cinit__(self, encoding='UTF-8', source_encoding=None, attribute_filter=None, attribute_handlers=None, bint register_eval=True, bint unpack_returned_tuples=False, bint register_builtins=True, overflow_handler=None, - max_memory=None): + max_memory=0): cdef lua_State* L - cdef size_t* memory_left - if max_memory is None: - L = lua.luaL_newstate() - else: - memory_left = malloc(sizeof(size_t)) - if memory_left is NULL: - raise LuaError("Failed to allocate Lua runtime memory limiter") - memory_left[0] = max_memory - L = lua.lua_newstate(&_lua_alloc_restricted, memory_left) + + self._memory_left = 0 + + L = lua.lua_newstate(&_lua_alloc_restricted, &self._memory_left) if L is NULL: raise LuaError("Failed to initialise Lua runtime") + self._state = L self._lock = FastRLock() self._pyrefs_in_lua = {} @@ -283,7 +279,6 @@ cdef class LuaRuntime: self._attribute_filter = attribute_filter self._unpack_returned_tuples = unpack_returned_tuples self._max_memory = max_memory - self._memory_left = memory_left if attribute_handlers: raise_error = False @@ -301,14 +296,16 @@ cdef class LuaRuntime: raise ValueError("attribute_filter and attribute_handlers are mutually exclusive") self._attribute_getter, self._attribute_setter = getter, setter - if max_memory is not None: - # luaL_newstate already registers a panic handler - lua.lua_atpanic(L, &_lua_panic) + lua.lua_atpanic(L, &_lua_panic) lua.luaL_openlibs(L) self.init_python_lib(register_eval, register_builtins) self.set_overflow_handler(overflow_handler) + # lupa init done, set real limit + if max_memory > 0: + self._memory_left = 1 + max_memory + def __dealloc__(self): if self._state is not NULL: lua.lua_close(self._state) @@ -317,6 +314,10 @@ cdef class LuaRuntime: @property def max_memory(self): return self._max_memory + + @property + def memory_left(self): # TODO: remove + return self._memory_left @property def lua_version(self): @@ -494,10 +495,14 @@ cdef class LuaRuntime: unlock_runtime(self) def set_max_memory(self, max_memory): - if self._max_memory is None: - raise RuntimeError("max_memory must be set on LuaRuntime creation") - used = self._max_memory - self._memory_left[0] - self._memory_left[0] = max_memory - used + if self._max_memory == 0 or max_memory == 0: + self._memory_left = 1 + max_memory if max_memory else 0 + else: + used = self._max_memory - self._memory_left + 1 + if used > max_memory: + self._memory_left = 1 + else: + self._memory_left = 1 + max_memory - used self._max_memory = max_memory def set_overflow_handler(self, overflow_handler): @@ -1600,7 +1605,7 @@ cdef int raise_lua_error(LuaRuntime runtime, lua_State* L, int result) except -1 elif result == lua.LUA_ERRMEM: raise LuaMemoryError() else: - raise LuaError(build_lua_error_message(runtime, L, None, -1)) + raise LuaError(build_lua_error_message(runtime, L, -1)) cdef build_lua_error_message(LuaRuntime runtime, lua_State* L, int n): """Removes the string at the given stack index ``n`` to build an error message. @@ -1628,7 +1633,7 @@ cdef run_lua(LuaRuntime runtime, bytes lua_code, tuple args): try: check_lua_stack(L, 1) if lua.luaL_loadbuffer(L, lua_code, len(lua_code), ''): - error = build_lua_error_message(runtiem, L, -1) + error = build_lua_error_message(runtime, L, -1) if error == "not enough memory": raise LuaMemoryError(error) raise LuaSyntaxError(u"error loading code: %s" % error) @@ -1661,15 +1666,16 @@ cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize if nsize == 0: free(ptr) - left[0] += osize # add old size to available memory + if left[0] > 0: + left[0] += osize # add old size to available memory return NULL elif nsize == osize: return ptr else: - if nsize > osize and left[0] < nsize - osize: # reached the limit + if left[0] > 0 and nsize > osize and left[0] <= nsize - osize: # reached the limit return NULL ptr = realloc(ptr, nsize) - if ptr: # reallocation successful? + if ptr and left[0] > 0: # reallocation successful? left[0] -= nsize + osize return ptr From d607896b20082cad47bcede94b82145b6a94b3ed Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 16:10:15 +0200 Subject: [PATCH 08/55] add tests --- lupa/tests/test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index d8706919..578717c9 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -2973,6 +2973,24 @@ def test_tostring_err(self): self.assertRaises(lupa.LuaError, str, self.lua.eval('setmetatable({}, {__tostring = function() error() end})')) +################################################################################ +# test LuaRuntime max_memory + +class TestMaxMemory(SetupLuaRuntimeMixin, unittest.TestCase): + def test_not_enough_memory(self): + self.lua.set_max_memory(10_000) + self.lua.eval("('a'):rep(50)") + with self.assertRaises(lupa.LuaMemoryError): + self.lua.eval("('a'):rep(50000)") + + def test_decrease_memory(self): + self.lua.set_max_memory(1_000_000) + self.lua.execute("local a = ('a'):rep(50000)") + self.lua.set_max_memory(10_000) + with self.assertRaises(lupa.LuaMemoryError): + self.lua.eval("'a'") + + if __name__ == '__main__': def print_version(): version = lupa.LuaRuntime().lua_implementation From 22e40814d62cd417d5cb5fd27294a0ce251be053 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 16:15:30 +0200 Subject: [PATCH 09/55] use python mem allocator and fix memory leak warning: lupa/_lupa.pyx:268:29: Casting a GIL-requiring function into a nogil function circumvents GIL validation --- lupa/_lupa.pyx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index b2af8ed6..18046a01 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -9,7 +9,7 @@ from __future__ import absolute_import cimport cython from libc.string cimport strlen, strchr -from libc.stdlib cimport malloc, free, realloc +from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free from libc.stdio cimport fprintf, stderr, fflush from lupa cimport lua from .lua cimport lua_State @@ -1654,7 +1654,7 @@ cdef call_lua(LuaRuntime runtime, lua_State *L, tuple args): return execute_lua_call(runtime, L, len(args)) # adapted from https://stackoverflow.com/a/9672205 -cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize) nogil: +cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize): cdef size_t* left = ud if ptr is NULL: @@ -1665,7 +1665,7 @@ cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize osize = 0 if nsize == 0: - free(ptr) + PyMem_Free(ptr) if left[0] > 0: left[0] += osize # add old size to available memory return NULL @@ -1674,10 +1674,12 @@ cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize else: if left[0] > 0 and nsize > osize and left[0] <= nsize - osize: # reached the limit return NULL - ptr = realloc(ptr, nsize) - if ptr and left[0] > 0: # reallocation successful? + new_ptr = PyMem_Realloc(ptr, nsize) + if new_ptr is NULL: + PyMem_Free(ptr) + elif left[0] > 0: left[0] -= nsize + osize - return ptr + return new_ptr cdef int _lua_panic(lua_State *L) nogil: cdef const char* msg = lua.lua_tostring(L, -1) From b0306de9b7e208e90efcf0e0a6e8fc9e1fb7e57e Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 16:32:52 +0200 Subject: [PATCH 10/55] fix test by using a global so lua doesnt gc it --- lupa/tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 578717c9..781893c5 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -2985,7 +2985,7 @@ def test_not_enough_memory(self): def test_decrease_memory(self): self.lua.set_max_memory(1_000_000) - self.lua.execute("local a = ('a'):rep(50000)") + self.lua.execute("a = ('a'):rep(50000)") self.lua.set_max_memory(10_000) with self.assertRaises(lupa.LuaMemoryError): self.lua.eval("'a'") From 41032205e5cf70fa2d37ccf7549bc9642b719339 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 16:37:46 +0200 Subject: [PATCH 11/55] revert to libc allocator --- lupa/_lupa.pyx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 18046a01..44312bd8 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -9,7 +9,7 @@ from __future__ import absolute_import cimport cython from libc.string cimport strlen, strchr -from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free +from libc.stdlib cimport malloc, free, realloc from libc.stdio cimport fprintf, stderr, fflush from lupa cimport lua from .lua cimport lua_State @@ -1665,7 +1665,7 @@ cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize osize = 0 if nsize == 0: - PyMem_Free(ptr) + free(ptr) if left[0] > 0: left[0] += osize # add old size to available memory return NULL @@ -1674,9 +1674,9 @@ cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize else: if left[0] > 0 and nsize > osize and left[0] <= nsize - osize: # reached the limit return NULL - new_ptr = PyMem_Realloc(ptr, nsize) + new_ptr = realloc(ptr, nsize) if new_ptr is NULL: - PyMem_Free(ptr) + free(ptr) elif left[0] > 0: left[0] -= nsize + osize return new_ptr From 158e9226292d1aaff863e2b61129e168ba537e22 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 16:38:54 +0200 Subject: [PATCH 12/55] remove memory_left property was using this for debugging --- lupa/_lupa.pyx | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 44312bd8..5c5c02b3 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -314,10 +314,6 @@ cdef class LuaRuntime: @property def max_memory(self): return self._max_memory - - @property - def memory_left(self): # TODO: remove - return self._memory_left @property def lua_version(self): From 007a91384bdb8b09bf1cd92e8d3bb5260b8fef4c Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 8 May 2022 16:56:20 +0200 Subject: [PATCH 13/55] add nogil flag --- lupa/_lupa.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 5c5c02b3..95a16ad9 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -1650,7 +1650,7 @@ cdef call_lua(LuaRuntime runtime, lua_State *L, tuple args): return execute_lua_call(runtime, L, len(args)) # adapted from https://stackoverflow.com/a/9672205 -cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize): +cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize) nogil: cdef size_t* left = ud if ptr is NULL: From 881d754c3415d60b3519d178a2af034dd7e2891b Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Fri, 20 May 2022 11:34:21 +0200 Subject: [PATCH 14/55] implement suggestions --- lupa/_lupa.pyx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 95a16ad9..c365e85c 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -227,7 +227,7 @@ cdef class LuaRuntime: * ``max_memory``: max memory usage this LuaRuntime can use in bytes. Builtins are not counted towards this limit. - (default: 0, new in Lupa 2.0) + (default: 0, i.e. no limitation. New in Lupa 2.0) Example usage:: @@ -1651,7 +1651,7 @@ cdef call_lua(LuaRuntime runtime, lua_State *L, tuple args): # adapted from https://stackoverflow.com/a/9672205 cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize) nogil: - cdef size_t* left = ud + cdef size_t* memory_left = ud if ptr is NULL: # : @@ -1662,19 +1662,19 @@ cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize if nsize == 0: free(ptr) - if left[0] > 0: - left[0] += osize # add old size to available memory + if memory_left[0] > 0: + memory_left[0] += osize # add old size to available memory return NULL elif nsize == osize: return ptr else: - if left[0] > 0 and nsize > osize and left[0] <= nsize - osize: # reached the limit + if memory_left[0] > 0 and nsize > osize and memory_left[0] <= nsize - osize: # reached the limit return NULL new_ptr = realloc(ptr, nsize) if new_ptr is NULL: free(ptr) - elif left[0] > 0: - left[0] -= nsize + osize + elif memory_left[0] > 0: + memory_left[0] -= nsize + osize return new_ptr cdef int _lua_panic(lua_State *L) nogil: From ef0fb32fab351921ea150df331ac0b8c63876525 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 22 May 2022 13:16:40 +0200 Subject: [PATCH 15/55] use lua allocator; add strict for set_max_memory; add docs --- lupa/_lupa.pyx | 47 ++++++++++++++++++++++++++++++++++++++++------ lupa/tests/test.py | 19 ++++++++++++++----- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index c365e85c..94189c50 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -226,7 +226,9 @@ cdef class LuaRuntime: represented in Lua, it should raise an ``OverflowError``. * ``max_memory``: max memory usage this LuaRuntime can use in bytes. - Builtins are not counted towards this limit. + If max_memory is 0, the default lua allocator is used and calls to + ``set_max_memory(limit)`` will fail. + Note: Not supported on 64bit LuaJIT. (default: 0, i.e. no limitation. New in Lupa 2.0) Example usage:: @@ -260,12 +262,15 @@ cdef class LuaRuntime: attribute_filter=None, attribute_handlers=None, bint register_eval=True, bint unpack_returned_tuples=False, bint register_builtins=True, overflow_handler=None, - max_memory=0): + max_memory=None): cdef lua_State* L self._memory_left = 0 - L = lua.lua_newstate(&_lua_alloc_restricted, &self._memory_left) + if max_memory is None: + L = lua.luaL_newstate() + else: + L = lua.lua_newstate(&_lua_alloc_restricted, &self._memory_left) if L is NULL: raise LuaError("Failed to initialise Lua runtime") @@ -278,7 +283,7 @@ cdef class LuaRuntime: raise ValueError("attribute_filter must be callable") self._attribute_filter = attribute_filter self._unpack_returned_tuples = unpack_returned_tuples - self._max_memory = max_memory + self._max_memory = 0 if max_memory is None else max_memory if attribute_handlers: raise_error = False @@ -303,7 +308,14 @@ cdef class LuaRuntime: self.set_overflow_handler(overflow_handler) # lupa init done, set real limit - if max_memory > 0: + if max_memory is None: + # memory_left is either 0 to indicate infinite memory + # or 1 + the limit + # since a limit of 0 is 0 instead of 1, 1 is unused + # and used as a flag for differentiating between max_memory=None + # and max_memory=0 + self._memory_left = 1 + elif max_memory > 0: self._memory_left = 1 + max_memory def __dealloc__(self): @@ -313,6 +325,13 @@ cdef class LuaRuntime: @property def max_memory(self): + """ + Maximum memory allowed to be used by this LuaRuntime. + 0 indicates no limit meanwhile None indicates that the default lua + allocator is being used and ``set_max_memory()`` cannot be used. + """ + if self._max_memory == 0 and self._memory_left == 1: + return None return self._max_memory @property @@ -490,13 +509,29 @@ cdef class LuaRuntime: lua.lua_settop(L, old_top) unlock_runtime(self) - def set_max_memory(self, max_memory): + def set_max_memory(self, max_memory, strict=False): + """Set maximum allowed memory for this LuaRuntime. + + Setting max_memory to a value lower than currently in use, will set + max_memory to the current usage. Use strict=True to throw a + LuaMemoryError instead. + + If max_memory was set to None during creation, this will raise a + RuntimeError. + """ + if self._max_memory == 0 and self._memory_left == 1: + raise RuntimeError("max_memory must be set on LuaRuntime creation") if self._max_memory == 0 or max_memory == 0: self._memory_left = 1 + max_memory if max_memory else 0 else: used = self._max_memory - self._memory_left + 1 if used > max_memory: + if strict: + raise LuaMemoryError( + "setting max_memory to less than currently in use" + ) self._memory_left = 1 + max_memory = used else: self._memory_left = 1 + max_memory - used self._max_memory = max_memory diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 781893c5..a6f43ac4 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -2977,19 +2977,28 @@ def test_tostring_err(self): # test LuaRuntime max_memory class TestMaxMemory(SetupLuaRuntimeMixin, unittest.TestCase): + lua_runtime_kwargs = {"max_memory": 10_000} + def test_not_enough_memory(self): - self.lua.set_max_memory(10_000) self.lua.eval("('a'):rep(50)") - with self.assertRaises(lupa.LuaMemoryError): - self.lua.eval("('a'):rep(50000)") + self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "('a'):rep(50000)") def test_decrease_memory(self): self.lua.set_max_memory(1_000_000) self.lua.execute("a = ('a'):rep(50000)") self.lua.set_max_memory(10_000) - with self.assertRaises(lupa.LuaMemoryError): - self.lua.eval("'a'") + self.assertGreater(self.lua.max_memory, 10_000) + self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "1") + + def test_decrease_strict(self): + self.lua.set_max_memory(1_000_000) + self.lua.execute("a = ('a'):rep(50000)") + self.assertRaises(lupa.LuaMemoryError, self.lua.set_max_memory, 10_000, True) + +class TestMaxMemoryWithoutSettingIt(SetupLuaRuntimeMixin, unittest.TestCase): + def test_set_max(self): + self.assertRaises(RuntimeError, self.lua.set_max_memory, 10_000) if __name__ == '__main__': def print_version(): From 6d3463f83de4a77209afa3b4eba4d498fb5f731f Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 22 May 2022 13:16:47 +0200 Subject: [PATCH 16/55] add more tests --- lupa/tests/test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index a6f43ac4..d1494050 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -2979,6 +2979,11 @@ def test_tostring_err(self): class TestMaxMemory(SetupLuaRuntimeMixin, unittest.TestCase): lua_runtime_kwargs = {"max_memory": 10_000} + def test_property(self): + self.assertEqual(self.lua.max_memory, 10_000) + self.lua.set_max_memory(1_000_000) + self.assertEqual(self.lua.max_memory, 1_000_000) + def test_not_enough_memory(self): self.lua.eval("('a'):rep(50)") self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "('a'):rep(50000)") @@ -2989,7 +2994,7 @@ def test_decrease_memory(self): self.lua.set_max_memory(10_000) self.assertGreater(self.lua.max_memory, 10_000) self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "1") - + def test_decrease_strict(self): self.lua.set_max_memory(1_000_000) self.lua.execute("a = ('a'):rep(50000)") @@ -2997,6 +3002,9 @@ def test_decrease_strict(self): class TestMaxMemoryWithoutSettingIt(SetupLuaRuntimeMixin, unittest.TestCase): + def test_property(self): + self.assertEqual(self.lua.max_memory, None) + def test_set_max(self): self.assertRaises(RuntimeError, self.lua.set_max_memory, 10_000) From 5b4dc02a817902c18f4618c6e1a702d289950f97 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 22 May 2022 13:27:58 +0200 Subject: [PATCH 17/55] update docs --- lupa/_lupa.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 94189c50..22656680 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -226,10 +226,10 @@ cdef class LuaRuntime: represented in Lua, it should raise an ``OverflowError``. * ``max_memory``: max memory usage this LuaRuntime can use in bytes. - If max_memory is 0, the default lua allocator is used and calls to + If max_memory is None, the default lua allocator is used and calls to ``set_max_memory(limit)`` will fail. Note: Not supported on 64bit LuaJIT. - (default: 0, i.e. no limitation. New in Lupa 2.0) + (default: None, i.e. no limitation. New in Lupa 2.0) Example usage:: From 46e35070dda10ddd162b8ba99d0b7101a590426e Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 22 May 2022 15:26:23 +0200 Subject: [PATCH 18/55] add memory_used --- lupa/_lupa.pyx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 22656680..8480d9b7 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -334,6 +334,17 @@ cdef class LuaRuntime: return None return self._max_memory + @property + def memory_used(self): + """" + Memory currently in use. + This is None if the default lua allocator is used and 0 if + ``max_memory`` is 0. + """ + if self._max_memory == 0: + return None if self._memory_left == 1 else 0 + return self._max_memory - self._memory_left + 1 + @property def lua_version(self): """ From 68aad666869b60f65d7f3724facfec509b3e1fa1 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 22 May 2022 15:26:47 +0200 Subject: [PATCH 19/55] welp --- lupa/_lupa.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 8480d9b7..cf677b11 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -336,7 +336,7 @@ cdef class LuaRuntime: @property def memory_used(self): - """" + """ Memory currently in use. This is None if the default lua allocator is used and 0 if ``max_memory`` is 0. From d8231524fb652350a459e089386b6998b2ebaf21 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Wed, 3 Aug 2022 21:14:16 +0200 Subject: [PATCH 20/55] update test --- lupa/tests/test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index d1494050..c2e47a59 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -2992,8 +2992,9 @@ def test_decrease_memory(self): self.lua.set_max_memory(1_000_000) self.lua.execute("a = ('a'):rep(50000)") self.lua.set_max_memory(10_000) - self.assertGreater(self.lua.max_memory, 10_000) - self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "1") + self.assertGreaterEqual(self.lua.max_memory, 10_000) + self.assertGreaterEqual(self.lua.memory_used, 50_000) + self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "'1'") def test_decrease_strict(self): self.lua.set_max_memory(1_000_000) From 00a81fe9aebd8663f58ed4ab65b0c7ee206b2fbd Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Wed, 3 Aug 2022 21:16:53 +0200 Subject: [PATCH 21/55] revert this change --- lupa/tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index c2e47a59..5ca46357 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -2992,7 +2992,7 @@ def test_decrease_memory(self): self.lua.set_max_memory(1_000_000) self.lua.execute("a = ('a'):rep(50000)") self.lua.set_max_memory(10_000) - self.assertGreaterEqual(self.lua.max_memory, 10_000) + self.assertGreater(self.lua.max_memory, 10_000) self.assertGreaterEqual(self.lua.memory_used, 50_000) self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "'1'") From 3aa4d1d5d1d1a7324da55ade4a661f9a71f4ba9e Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 7 Aug 2022 20:53:21 +0200 Subject: [PATCH 22/55] Minor code cleanups. --- lupa/_lupa.pyx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 2d6b2eb0..e135ee08 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -538,9 +538,7 @@ cdef class LuaRuntime: used = self._max_memory - self._memory_left + 1 if used > max_memory: if strict: - raise LuaMemoryError( - "setting max_memory to less than currently in use" - ) + raise LuaMemoryError("setting max_memory to less than currently in use") self._memory_left = 1 max_memory = used else: @@ -1717,7 +1715,7 @@ cdef run_lua(LuaRuntime runtime, bytes lua_code, tuple args): error = build_lua_error_message(runtime, L) if error == "not enough memory": raise LuaMemoryError(error) - raise LuaSyntaxError(u"error loading code: %s" % error) + raise LuaSyntaxError(u"error loading code: " + error) return call_lua(runtime, L, args) finally: lua.lua_settop(L, old_top) From d907edf95be6759a85ef4b3f27d89bcde238a237 Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 7 Aug 2022 20:55:09 +0200 Subject: [PATCH 23/55] Improve docstring. --- lupa/_lupa.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index e135ee08..69c27353 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -227,7 +227,7 @@ cdef class LuaRuntime: * ``max_memory``: max memory usage this LuaRuntime can use in bytes. If max_memory is None, the default lua allocator is used and calls to - ``set_max_memory(limit)`` will fail. + ``set_max_memory(limit)`` will fail with a ``LuaMemoryError``. Note: Not supported on 64bit LuaJIT. (default: None, i.e. no limitation. New in Lupa 2.0) From bde8543e68beb17fdf3b6240b63b04765b700da0 Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 7 Aug 2022 20:58:27 +0200 Subject: [PATCH 24/55] Fix tests. --- lupa/tests/test.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index f925b9a4..742cc669 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -3006,38 +3006,39 @@ def load_tests(loader, standard_tests, pattern): ################################################################################ # test LuaRuntime max_memory -class TestMaxMemory(SetupLuaRuntimeMixin, unittest.TestCase): - lua_runtime_kwargs = {"max_memory": 10_000} +class TestMaxMemory(SetupLuaRuntimeMixin, LupaTestCase): + lua_runtime_kwargs = {"max_memory": 10000} def test_property(self): - self.assertEqual(self.lua.max_memory, 10_000) - self.lua.set_max_memory(1_000_000) - self.assertEqual(self.lua.max_memory, 1_000_000) + self.assertEqual(self.lua.max_memory, 10000) + self.lua.set_max_memory(1000000) + self.assertEqual(self.lua.max_memory, 1000000) def test_not_enough_memory(self): self.lua.eval("('a'):rep(50)") self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "('a'):rep(50000)") def test_decrease_memory(self): - self.lua.set_max_memory(1_000_000) + self.lua.set_max_memory(1000000) self.lua.execute("a = ('a'):rep(50000)") - self.lua.set_max_memory(10_000) - self.assertGreater(self.lua.max_memory, 10_000) - self.assertGreaterEqual(self.lua.memory_used, 50_000) + self.lua.set_max_memory(10000) + self.assertGreater(self.lua.max_memory, 10000) + self.assertGreaterEqual(self.lua.memory_used, 50000) self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "'1'") def test_decrease_strict(self): - self.lua.set_max_memory(1_000_000) + self.lua.set_max_memory(1000000) self.lua.execute("a = ('a'):rep(50000)") - self.assertRaises(lupa.LuaMemoryError, self.lua.set_max_memory, 10_000, True) + self.assertRaises(lupa.LuaMemoryError, self.lua.set_max_memory, 10000, True) -class TestMaxMemoryWithoutSettingIt(SetupLuaRuntimeMixin, unittest.TestCase): +class TestMaxMemoryWithoutSettingIt(SetupLuaRuntimeMixin, LupaTestCase): def test_property(self): self.assertEqual(self.lua.max_memory, None) def test_set_max(self): - self.assertRaises(RuntimeError, self.lua.set_max_memory, 10_000) + self.assertRaises(RuntimeError, self.lua.set_max_memory, 10000) + if __name__ == '__main__': def print_version(): From 82502a663e529f9a02e70c446c00554197fef480 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 7 Aug 2022 21:40:10 +0200 Subject: [PATCH 25/55] ignore luajit --- lupa/tests/test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 742cc669..7269aa0f 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -3009,6 +3009,12 @@ def load_tests(loader, standard_tests, pattern): class TestMaxMemory(SetupLuaRuntimeMixin, LupaTestCase): lua_runtime_kwargs = {"max_memory": 10000} + def setUp(self): + # need to test in here because the creation of the LuaRuntime fails + if "luajit" in self.lupa.LuaRuntime().lua_implementation.lower(): + self.skipTest("not supported in LuaJIT") + return super().setUp() + def test_property(self): self.assertEqual(self.lua.max_memory, 10000) self.lua.set_max_memory(1000000) From b1806fcd1d3923c7794daeaf9537d1c18ee1001b Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 7 Aug 2022 21:43:13 +0200 Subject: [PATCH 26/55] fix tests --- lupa/tests/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 7269aa0f..4ae4dd94 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -3022,7 +3022,7 @@ def test_property(self): def test_not_enough_memory(self): self.lua.eval("('a'):rep(50)") - self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "('a'):rep(50000)") + self.assertRaises(self.lupa.LuaMemoryError, self.lua.eval, "('a'):rep(50000)") def test_decrease_memory(self): self.lua.set_max_memory(1000000) @@ -3030,12 +3030,12 @@ def test_decrease_memory(self): self.lua.set_max_memory(10000) self.assertGreater(self.lua.max_memory, 10000) self.assertGreaterEqual(self.lua.memory_used, 50000) - self.assertRaises(lupa.LuaMemoryError, self.lua.eval, "'1'") + self.assertRaises(self.lupa.LuaMemoryError, self.lua.eval, "('a'):rep(10)") def test_decrease_strict(self): self.lua.set_max_memory(1000000) self.lua.execute("a = ('a'):rep(50000)") - self.assertRaises(lupa.LuaMemoryError, self.lua.set_max_memory, 10000, True) + self.assertRaises(self.lupa.LuaMemoryError, self.lua.set_max_memory, 10000, True) class TestMaxMemoryWithoutSettingIt(SetupLuaRuntimeMixin, LupaTestCase): From a7386f3f39a12b812c91593f393b1f2ccad47fc0 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 7 Aug 2022 21:44:17 +0200 Subject: [PATCH 27/55] catch more memory errors --- lupa/_lupa.pyx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 69c27353..268edc20 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -419,7 +419,8 @@ cdef class LuaRuntime: else: err = lua.lua_tolstring(L, -1, &size) error = err[:size] if self._encoding is None else err[:size].decode(self._encoding) - if error == (b"not enough memory" if self._encoding is None else "not enough memory"): + not_enough_memory = (b"not enough memory" if self._encoding is None else "not enough memory") + if error.startswith(not_enough_memory): raise LuaMemoryError(error) raise LuaSyntaxError(error) finally: @@ -1645,7 +1646,10 @@ cdef int raise_lua_error(LuaRuntime runtime, lua_State* L, int result) except -1 elif result == lua.LUA_ERRMEM: raise LuaMemoryError() else: - raise LuaError(build_lua_error_message(runtime, L)) + error_message = build_lua_error_message(runtime, L) + if u"not enough memory" in error_message: + raise LuaMemoryError(error_message) + raise LuaError(error_message) cdef bint _looks_like_traceback_line(unicode line): @@ -1713,7 +1717,7 @@ cdef run_lua(LuaRuntime runtime, bytes lua_code, tuple args): check_lua_stack(L, 1) if lua.luaL_loadbuffer(L, lua_code, len(lua_code), ''): error = build_lua_error_message(runtime, L) - if error == "not enough memory": + if error.startswith("not enough memory"): raise LuaMemoryError(error) raise LuaSyntaxError(u"error loading code: " + error) return call_lua(runtime, L, args) From 4143da59682d1f22ce516efd50aa23929ffc63b0 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sun, 7 Aug 2022 22:04:06 +0200 Subject: [PATCH 28/55] inherit from MemoryError you shall --- lupa/_lupa.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 268edc20..d47ca5f2 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -123,7 +123,7 @@ class LuaSyntaxError(LuaError): """ -class LuaMemoryError(LuaError): +class LuaMemoryError(LuaError, MemoryError): """Memory error in Lua code. """ From c43fbb94a2f55286bde2c65a240b17ddf8f29cd9 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 8 Aug 2022 08:22:33 +0200 Subject: [PATCH 29/55] fix python2.x compat & actually skip initializing lua --- lupa/tests/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 4ae4dd94..2c6c5f50 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -3012,8 +3012,8 @@ class TestMaxMemory(SetupLuaRuntimeMixin, LupaTestCase): def setUp(self): # need to test in here because the creation of the LuaRuntime fails if "luajit" in self.lupa.LuaRuntime().lua_implementation.lower(): - self.skipTest("not supported in LuaJIT") - return super().setUp() + return self.skipTest("not supported in LuaJIT") + return super(TestMaxMemory, self).setUp() def test_property(self): self.assertEqual(self.lua.max_memory, 10000) From aedeb6e75a0a9b6b339a654d31bc26e348da546a Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 8 Aug 2022 08:27:52 +0200 Subject: [PATCH 30/55] add test for compile and fix memory error assert --- lupa/_lupa.pyx | 2 +- lupa/tests/test.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index d47ca5f2..6696b279 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -420,7 +420,7 @@ cdef class LuaRuntime: err = lua.lua_tolstring(L, -1, &size) error = err[:size] if self._encoding is None else err[:size].decode(self._encoding) not_enough_memory = (b"not enough memory" if self._encoding is None else "not enough memory") - if error.startswith(not_enough_memory): + if not_enough_memory in error: raise LuaMemoryError(error) raise LuaSyntaxError(error) finally: diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 2c6c5f50..7dc9a384 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -3037,6 +3037,10 @@ def test_decrease_strict(self): self.lua.execute("a = ('a'):rep(50000)") self.assertRaises(self.lupa.LuaMemoryError, self.lua.set_max_memory, 10000, True) + def test_compile_not_enough_memory(self): + self.lua.set_max_memory(10) + self.assertRaises(self.lupa.LuaMemoryError, self.lua.compile, "_G.a = function() return 'test abcdef' end") + class TestMaxMemoryWithoutSettingIt(SetupLuaRuntimeMixin, LupaTestCase): def test_property(self): From 93f61bea4d999660be4db80079611a4feb66cf39 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 8 Aug 2022 08:28:06 +0200 Subject: [PATCH 31/55] possible fix for flaky test on win? --- lupa/tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 7dc9a384..fb910793 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -3030,7 +3030,7 @@ def test_decrease_memory(self): self.lua.set_max_memory(10000) self.assertGreater(self.lua.max_memory, 10000) self.assertGreaterEqual(self.lua.memory_used, 50000) - self.assertRaises(self.lupa.LuaMemoryError, self.lua.eval, "('a'):rep(10)") + self.assertRaises(self.lupa.LuaMemoryError, self.lua.eval, "('b'):rep(10)") def test_decrease_strict(self): self.lua.set_max_memory(1000000) From 6d874d5df7d9533ffaf2feb225680c2d2c384827 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 8 Aug 2022 08:41:05 +0200 Subject: [PATCH 32/55] this should be at the bottom, no? --- lupa/tests/test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index fb910793..c7b992dc 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -2995,14 +2995,6 @@ def test_tostring_err(self): self.assertRaises(self.lupa.LuaError, str, self.lua.eval('setmetatable({}, {__tostring = function() error() end})')) - -################################################################################ -# Load tests for different Lua version modules - -def load_tests(loader, standard_tests, pattern): - return lupa.tests.build_suite_for_modules(loader, globals()) - - ################################################################################ # test LuaRuntime max_memory @@ -3050,6 +3042,14 @@ def test_set_max(self): self.assertRaises(RuntimeError, self.lua.set_max_memory, 10000) +################################################################################ +# Load tests for different Lua version modules + +def load_tests(loader, standard_tests, pattern): + return lupa.tests.build_suite_for_modules(loader, globals()) + + + if __name__ == '__main__': def print_version(): version = lupa.LuaRuntime().lua_implementation From 7ac2d461aacd113a2a576d36568dd16cd80352e4 Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 14 Aug 2022 10:01:32 +0200 Subject: [PATCH 33/55] Clean up code and fix a couple of issues. --- lupa/_lupa.pyx | 94 +++++++++++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index f7cb7797..f5bed5aa 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -343,7 +343,7 @@ cdef class LuaRuntime: """ if self._max_memory == 0: return None if self._memory_left == 1 else 0 - return self._max_memory - self._memory_left + 1 + return self._max_memory - (self._memory_left - 1) @property def lua_version(self): @@ -418,9 +418,13 @@ cdef class LuaRuntime: return py_from_lua(self, L, -1) else: err = lua.lua_tolstring(L, -1, &size) - error = err[:size] if self._encoding is None else err[:size].decode(self._encoding) - not_enough_memory = (b"not enough memory" if self._encoding is None else "not enough memory") - if not_enough_memory in error: + if self._encoding is None: + error = err[:size] # bytes + is_memory_error = b"not enough memory" in error + else: + error = err[:size].decode(self._encoding) + is_memory_error = u"not enough memory" in error + if is_memory_error: raise LuaMemoryError(error) raise LuaSyntaxError(error) finally: @@ -521,7 +525,7 @@ cdef class LuaRuntime: lua.lua_settop(L, old_top) unlock_runtime(self) - def set_max_memory(self, max_memory, strict=False): + def set_max_memory(self, size_t max_memory, bint strict=False): """Set maximum allowed memory for this LuaRuntime. Setting max_memory to a value lower than currently in use, will set @@ -531,19 +535,20 @@ cdef class LuaRuntime: If max_memory was set to None during creation, this will raise a RuntimeError. """ + cdef size_t used if self._max_memory == 0 and self._memory_left == 1: raise RuntimeError("max_memory must be set on LuaRuntime creation") if self._max_memory == 0 or max_memory == 0: self._memory_left = 1 + max_memory if max_memory else 0 else: - used = self._max_memory - self._memory_left + 1 + used = self._max_memory - (self._memory_left - 1) if used > max_memory: if strict: raise LuaMemoryError("setting max_memory to less than currently in use") self._memory_left = 1 max_memory = used else: - self._memory_left = 1 + max_memory - used + self._memory_left = 1 + (max_memory - used) self._max_memory = max_memory def set_overflow_handler(self, overflow_handler): @@ -1736,43 +1741,6 @@ cdef call_lua(LuaRuntime runtime, lua_State *L, tuple args): push_lua_arguments(runtime, L, args) return execute_lua_call(runtime, L, len(args)) -# adapted from https://stackoverflow.com/a/9672205 -cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t osize, size_t nsize) nogil: - cdef size_t* memory_left = ud - - if ptr is NULL: - # : - # When ptr is NULL, osize encodes the kind of object that Lua is - # allocating. - # Since we don’t care about that, just mark it as 0. - osize = 0 - - if nsize == 0: - free(ptr) - if memory_left[0] > 0: - memory_left[0] += osize # add old size to available memory - return NULL - elif nsize == osize: - return ptr - else: - if memory_left[0] > 0 and nsize > osize and memory_left[0] <= nsize - osize: # reached the limit - return NULL - new_ptr = realloc(ptr, nsize) - if new_ptr is NULL: - free(ptr) - elif memory_left[0] > 0: - memory_left[0] -= nsize + osize - return new_ptr - -cdef int _lua_panic(lua_State *L) nogil: - cdef const char* msg = lua.lua_tostring(L, -1) - if msg == NULL: - msg = "error object is not a string" - cdef char* message = "PANIC: unprotected error in call to Lua API (%s)\n" - fprintf(stderr, message, msg) - fflush(stderr) - return 0 # return to Lua to abort - cdef object execute_lua_call(LuaRuntime runtime, lua_State *L, Py_ssize_t nargs): cdef int result_status cdef object result @@ -1849,6 +1817,44 @@ cdef tuple unpack_multiple_lua_results(LuaRuntime runtime, lua_State *L, int nar return args +# bounded memory allocation + +cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t old_size, size_t new_size) nogil: + # adapted from https://stackoverflow.com/a/9672205 + cdef size_t* memory_left = ud + + if ptr is NULL: + # : + # When ptr is NULL, old_size encodes the kind of object that Lua is allocating. + # Since we don’t care about that, just mark it as 0. + old_size = 0 + + if new_size == 0: + free(ptr) + if memory_left[0] > 0: + memory_left[0] += old_size # add old size to available memory + return NULL + elif new_size == old_size: + return ptr + else: + if memory_left[0] > 0 and new_size > old_size and memory_left[0] <= new_size - old_size: # reached the limit + return NULL + new_ptr = realloc(ptr, new_size) + if new_ptr is not NULL and memory_left[0] > 0: + memory_left[0] += new_size - old_size + return new_ptr + + +cdef int _lua_panic(lua_State *L) nogil: + cdef const char* msg = lua.lua_tostring(L, -1) + if msg == NULL: + msg = "error object is not a string" + cdef char* message = "PANIC: unprotected error in call to Lua API (%s)\n" + fprintf(stderr, message, msg) + fflush(stderr) + return 0 # return to Lua to abort + + ################################################################################ # Python support in Lua From b47916070e993e44fc04b1583b7167570fbd8440 Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 14 Aug 2022 10:11:59 +0200 Subject: [PATCH 34/55] Improve wording in changelog. Co-authored-by: Leo Developer --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index fda7ba49..abfaf435 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,7 +11,7 @@ Lupa change log Lua 5.4, LuaJIT 2.0 and LuaJIT 2.1 beta. Note that this is build specific and may depend on the platform. A normal Python import cascade can be used. -* GH#211: Option to limit memory usage +* GH#211: A new option `max_memory` allows to limit the memory usage of Lua code. (patch by Leo Developer) * GH#171: Python references in Lua are now more safely reference counted From 89540d02e23fa1e8b115165f74bc07cede875c05 Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 14 Aug 2022 10:34:19 +0200 Subject: [PATCH 35/55] Clarify and fix memory calculation logic in allocation function. --- lupa/_lupa.pyx | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index f5bed5aa..d94e8bca 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -1821,7 +1821,9 @@ cdef tuple unpack_multiple_lua_results(LuaRuntime runtime, lua_State *L, int nar cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t old_size, size_t new_size) nogil: # adapted from https://stackoverflow.com/a/9672205 - cdef size_t* memory_left = ud + cdef size_t* memory_left_status = ud + cdef size_t memory_left = memory_left_status[0] + memory_is_counted = memory_left > 0 if ptr is NULL: # : @@ -1829,20 +1831,22 @@ cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t old_size, size_t ne # Since we don’t care about that, just mark it as 0. old_size = 0 + cdef void* new_ptr if new_size == 0: free(ptr) - if memory_left[0] > 0: - memory_left[0] += old_size # add old size to available memory - return NULL + if memory_is_counted: + memory_left += old_size # add deallocated old size to available memory + new_ptr = NULL elif new_size == old_size: - return ptr + new_ptr = ptr else: - if memory_left[0] > 0 and new_size > old_size and memory_left[0] <= new_size - old_size: # reached the limit + if memory_is_counted and new_size > old_size and memory_left <= new_size - old_size: # reached the limit return NULL new_ptr = realloc(ptr, new_size) - if new_ptr is not NULL and memory_left[0] > 0: - memory_left[0] += new_size - old_size - return new_ptr + + if new_ptr is not NULL and memory_is_counted: + memory_left_status[0] = memory_left - (new_size - old_size) + return new_ptr cdef int _lua_panic(lua_State *L) nogil: From 215a262dab973024d1f9ace8fba089f2a113f24e Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sat, 17 Sep 2022 15:58:59 +0200 Subject: [PATCH 36/55] use struct --- lupa/_lupa.pyx | 109 +++++++++++++++++++++++---------------------- lupa/tests/test.py | 28 +++++++----- 2 files changed, 73 insertions(+), 64 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index d94e8bca..2b831526 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -9,7 +9,7 @@ from __future__ import absolute_import cimport cython from libc.string cimport strlen, strchr -from libc.stdlib cimport malloc, free, realloc +from libc.stdlib cimport malloc, calloc, free, realloc from libc.stdio cimport fprintf, stderr, fflush from . cimport luaapi as lua from .luaapi cimport lua_State @@ -255,8 +255,7 @@ cdef class LuaRuntime: cdef object _attribute_getter cdef object _attribute_setter cdef bint _unpack_returned_tuples - cdef size_t _max_memory - cdef size_t _memory_left + cdef MemoryStatus* _memory_status def __cinit__(self, encoding='UTF-8', source_encoding=None, attribute_filter=None, attribute_handlers=None, @@ -265,12 +264,15 @@ cdef class LuaRuntime: max_memory=None): cdef lua_State* L - self._memory_left = 0 - if max_memory is None: L = lua.luaL_newstate() else: - L = lua.lua_newstate(&_lua_alloc_restricted, &self._memory_left) + memory_status = calloc(1, sizeof(MemoryStatus)) + self._memory_status = memory_status + self._memory_status.used = 0 + self._memory_status.base_usage = 0 + self._memory_status.limit = 0 + L = lua.lua_newstate(&_lua_alloc_restricted, self._memory_status) if L is NULL: raise LuaError("Failed to initialise Lua runtime") @@ -283,7 +285,6 @@ cdef class LuaRuntime: raise ValueError("attribute_filter must be callable") self._attribute_filter = attribute_filter self._unpack_returned_tuples = unpack_returned_tuples - self._max_memory = 0 if max_memory is None else max_memory if attribute_handlers: raise_error = False @@ -308,42 +309,47 @@ cdef class LuaRuntime: self.set_overflow_handler(overflow_handler) # lupa init done, set real limit - if max_memory is None: - # memory_left is either 0 to indicate infinite memory - # or 1 + the limit - # since a limit of 0 is 0 instead of 1, 1 is unused - # and used as a flag for differentiating between max_memory=None - # and max_memory=0 - self._memory_left = 1 - elif max_memory > 0: - self._memory_left = 1 + max_memory + if max_memory is not None: + self._memory_status.base_usage = self._memory_status.used + self._memory_status.limit = self._memory_status.base_usage + max_memory def __dealloc__(self): if self._state is not NULL: lua.lua_close(self._state) self._state = NULL + if self._memory_status is not NULL: + free(self._memory_status) + self._memory_status = NULL - @property - def max_memory(self): + def get_max_memory(self, count_base=False): """ Maximum memory allowed to be used by this LuaRuntime. 0 indicates no limit meanwhile None indicates that the default lua allocator is being used and ``set_max_memory()`` cannot be used. + + If ``count_base`` is True, the base memory used by the lua runtime + will be included in the limit. """ - if self._max_memory == 0 and self._memory_left == 1: + if self._memory_status is NULL: return None - return self._max_memory + elif count_base: + return self._memory_status.limit + return self._memory_status.limit - self._memory_status.base_usage - @property - def memory_used(self): + def get_memory_used(self, count_base=False): """ Memory currently in use. This is None if the default lua allocator is used and 0 if ``max_memory`` is 0. + + If ``count_base`` is True, the base memory used by the lua runtime + will be included. """ - if self._max_memory == 0: - return None if self._memory_left == 1 else 0 - return self._max_memory - (self._memory_left - 1) + if self._memory_status is NULL: + return None + elif count_base: + return self._memory_status.used + return self._memory_status.used - self._memory_status.base_usage @property def lua_version(self): @@ -525,7 +531,7 @@ cdef class LuaRuntime: lua.lua_settop(L, old_top) unlock_runtime(self) - def set_max_memory(self, size_t max_memory, bint strict=False): + def set_max_memory(self, size_t max_memory, count_base=False): """Set maximum allowed memory for this LuaRuntime. Setting max_memory to a value lower than currently in use, will set @@ -536,20 +542,12 @@ cdef class LuaRuntime: RuntimeError. """ cdef size_t used - if self._max_memory == 0 and self._memory_left == 1: + if self._memory_status is NULL: raise RuntimeError("max_memory must be set on LuaRuntime creation") - if self._max_memory == 0 or max_memory == 0: - self._memory_left = 1 + max_memory if max_memory else 0 + elif count_base: + self._memory_status.limit = max_memory else: - used = self._max_memory - (self._memory_left - 1) - if used > max_memory: - if strict: - raise LuaMemoryError("setting max_memory to less than currently in use") - self._memory_left = 1 - max_memory = used - else: - self._memory_left = 1 + (max_memory - used) - self._max_memory = max_memory + self._memory_status.limit = self._memory_status.base_usage + max_memory def set_overflow_handler(self, overflow_handler): """Set the overflow handler function that is called on failures to pass large numbers to Lua. @@ -1819,11 +1817,16 @@ cdef tuple unpack_multiple_lua_results(LuaRuntime runtime, lua_State *L, int nar # bounded memory allocation +cdef struct MemoryStatus: + size_t used + size_t base_usage + size_t limit + cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t old_size, size_t new_size) nogil: # adapted from https://stackoverflow.com/a/9672205 - cdef size_t* memory_left_status = ud - cdef size_t memory_left = memory_left_status[0] - memory_is_counted = memory_left > 0 + # print(ud, ptr, old_size, new_size) + cdef MemoryStatus* memory_status = ud + # print(" ", memory_status.used, memory_status.base_usage, memory_status.limit) if ptr is NULL: # : @@ -1834,21 +1837,21 @@ cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t old_size, size_t ne cdef void* new_ptr if new_size == 0: free(ptr) - if memory_is_counted: - memory_left += old_size # add deallocated old size to available memory - new_ptr = NULL + memory_status.used -= old_size # add deallocated old size to available memory + return NULL elif new_size == old_size: - new_ptr = ptr - else: - if memory_is_counted and new_size > old_size and memory_left <= new_size - old_size: # reached the limit - return NULL - new_ptr = realloc(ptr, new_size) - - if new_ptr is not NULL and memory_is_counted: - memory_left_status[0] = memory_left - (new_size - old_size) + return ptr + + if memory_status.limit > 0 and new_size > old_size and memory_status.limit <= memory_status.used + new_size - old_size: # reached the limit + # print("REACHED LIMIT") + return NULL + # print(" realloc()...") + new_ptr = realloc(ptr, new_size) + # print(" ", memory_status.used, new_size - old_size, memory_status.used + new_size - old_size) + if new_ptr is not NULL: + memory_status.used += new_size - old_size return new_ptr - cdef int _lua_panic(lua_State *L) nogil: cdef const char* msg = lua.lua_tostring(L, -1) if msg == NULL: diff --git a/lupa/tests/test.py b/lupa/tests/test.py index c7b992dc..595c6777 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -3007,10 +3007,19 @@ def setUp(self): return self.skipTest("not supported in LuaJIT") return super(TestMaxMemory, self).setUp() - def test_property(self): - self.assertEqual(self.lua.max_memory, 10000) + def test_getters(self): + self.assertEqual(self.lua.get_memory_used(), 0) + self.assertGreater(self.lua.get_memory_used(True), 0) + self.assertEqual(self.lua.get_max_memory(), 10000) + self.assertGreater(self.lua.get_max_memory(True), 10000) self.lua.set_max_memory(1000000) - self.assertEqual(self.lua.max_memory, 1000000) + self.assertEqual(self.lua.get_memory_used(), 0) + self.assertGreater(self.lua.get_memory_used(True), 0) + self.assertEqual(self.lua.get_max_memory(), 1000000) + self.assertGreater(self.lua.get_max_memory(True), 1000000) + self.lua.set_max_memory(1000000, True) + self.assertEqual(self.lua.get_max_memory(True), 1000000) + self.assertLess(self.lua.get_max_memory(), 1000000) def test_not_enough_memory(self): self.lua.eval("('a'):rep(50)") @@ -3020,14 +3029,11 @@ def test_decrease_memory(self): self.lua.set_max_memory(1000000) self.lua.execute("a = ('a'):rep(50000)") self.lua.set_max_memory(10000) - self.assertGreater(self.lua.max_memory, 10000) - self.assertGreaterEqual(self.lua.memory_used, 50000) + self.assertEqual(self.lua.get_max_memory(), 10000) + self.assertGreaterEqual(self.lua.get_memory_used(), 50000) self.assertRaises(self.lupa.LuaMemoryError, self.lua.eval, "('b'):rep(10)") - - def test_decrease_strict(self): - self.lua.set_max_memory(1000000) - self.lua.execute("a = ('a'):rep(50000)") - self.assertRaises(self.lupa.LuaMemoryError, self.lua.set_max_memory, 10000, True) + del self.lua.globals()["a"] + self.lua.eval("('b'):rep(10)") def test_compile_not_enough_memory(self): self.lua.set_max_memory(10) @@ -3036,7 +3042,7 @@ def test_compile_not_enough_memory(self): class TestMaxMemoryWithoutSettingIt(SetupLuaRuntimeMixin, LupaTestCase): def test_property(self): - self.assertEqual(self.lua.max_memory, None) + self.assertEqual(self.lua.get_max_memory(), None) def test_set_max(self): self.assertRaises(RuntimeError, self.lua.set_max_memory, 10000) From 86537ea9444029f59ddacca3b913ec979e133415 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Sat, 17 Sep 2022 21:29:32 +0200 Subject: [PATCH 37/55] dont need these --- lupa/_lupa.pyx | 3 --- 1 file changed, 3 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 2b831526..e459f270 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -269,9 +269,6 @@ cdef class LuaRuntime: else: memory_status = calloc(1, sizeof(MemoryStatus)) self._memory_status = memory_status - self._memory_status.used = 0 - self._memory_status.base_usage = 0 - self._memory_status.limit = 0 L = lua.lua_newstate(&_lua_alloc_restricted, self._memory_status) if L is NULL: raise LuaError("Failed to initialise Lua runtime") From 17e8c65bdca4a94fe2971558e886258b4d8eb903 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Fri, 23 Sep 2022 17:38:57 +0200 Subject: [PATCH 38/55] update set_max_memory docstring --- lupa/_lupa.pyx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index e459f270..71280af2 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -531,9 +531,8 @@ cdef class LuaRuntime: def set_max_memory(self, size_t max_memory, count_base=False): """Set maximum allowed memory for this LuaRuntime. - Setting max_memory to a value lower than currently in use, will set - max_memory to the current usage. Use strict=True to throw a - LuaMemoryError instead. + If ``count_base`` is True, the base memory used by the LuaRuntime itself + will be included in the memory limit. If max_memory was set to None during creation, this will raise a RuntimeError. From 437c58dccf4512d72d7f7e72e40751d759023490 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Fri, 23 Sep 2022 17:40:01 +0200 Subject: [PATCH 39/55] use keyword arguments for better readability --- lupa/tests/test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 595c6777..7d039dd6 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -3009,16 +3009,16 @@ def setUp(self): def test_getters(self): self.assertEqual(self.lua.get_memory_used(), 0) - self.assertGreater(self.lua.get_memory_used(True), 0) + self.assertGreater(self.lua.get_memory_used(count_base=True), 0) self.assertEqual(self.lua.get_max_memory(), 10000) - self.assertGreater(self.lua.get_max_memory(True), 10000) + self.assertGreater(self.lua.get_max_memory(count_base=True), 10000) self.lua.set_max_memory(1000000) self.assertEqual(self.lua.get_memory_used(), 0) - self.assertGreater(self.lua.get_memory_used(True), 0) + self.assertGreater(self.lua.get_memory_used(count_base=True), 0) self.assertEqual(self.lua.get_max_memory(), 1000000) - self.assertGreater(self.lua.get_max_memory(True), 1000000) - self.lua.set_max_memory(1000000, True) - self.assertEqual(self.lua.get_max_memory(True), 1000000) + self.assertGreater(self.lua.get_max_memory(count_base=True), 1000000) + self.lua.set_max_memory(1000000, count_base=True) + self.assertEqual(self.lua.get_max_memory(count_base=True), 1000000) self.assertLess(self.lua.get_max_memory(), 1000000) def test_not_enough_memory(self): From 95eb2a5f53c61d73946a6079f76d6aed97ea28ef Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Fri, 23 Sep 2022 17:46:32 +0200 Subject: [PATCH 40/55] add support for unlimited memory (0) --- lupa/_lupa.pyx | 6 +++++- lupa/tests/test.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 71280af2..3dfee04d 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -308,7 +308,8 @@ cdef class LuaRuntime: # lupa init done, set real limit if max_memory is not None: self._memory_status.base_usage = self._memory_status.used - self._memory_status.limit = self._memory_status.base_usage + max_memory + if max_memory > 0: + self._memory_status.limit = self._memory_status.base_usage + max_memory def __dealloc__(self): if self._state is not NULL: @@ -531,6 +532,7 @@ cdef class LuaRuntime: def set_max_memory(self, size_t max_memory, count_base=False): """Set maximum allowed memory for this LuaRuntime. + If `max_memory` is 0, there will be no limit. If ``count_base`` is True, the base memory used by the LuaRuntime itself will be included in the memory limit. @@ -540,6 +542,8 @@ cdef class LuaRuntime: cdef size_t used if self._memory_status is NULL: raise RuntimeError("max_memory must be set on LuaRuntime creation") + elif max_memory <= 0: + self._memory_status.limit = 0 elif count_base: self._memory_status.limit = max_memory else: diff --git a/lupa/tests/test.py b/lupa/tests/test.py index 7d039dd6..a5b7aef2 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -3039,6 +3039,10 @@ def test_compile_not_enough_memory(self): self.lua.set_max_memory(10) self.assertRaises(self.lupa.LuaMemoryError, self.lua.compile, "_G.a = function() return 'test abcdef' end") + def test_unlimited_memory(self): + self.lua.set_max_memory(0) + self.lua.execute("a = ('a'):rep(50000)") + class TestMaxMemoryWithoutSettingIt(SetupLuaRuntimeMixin, LupaTestCase): def test_property(self): From 4066683466711569059f8028464693c973b6826e Mon Sep 17 00:00:00 2001 From: scoder Date: Mon, 26 Sep 2022 09:18:58 +0200 Subject: [PATCH 41/55] Try to get by without dynamically allocating the memory state. --- lupa/_lupa.pyx | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 3dfee04d..0a7bc05f 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -9,7 +9,7 @@ from __future__ import absolute_import cimport cython from libc.string cimport strlen, strchr -from libc.stdlib cimport malloc, calloc, free, realloc +from libc.stdlib cimport malloc, free, realloc from libc.stdio cimport fprintf, stderr, fflush from . cimport luaapi as lua from .luaapi cimport lua_State @@ -255,7 +255,7 @@ cdef class LuaRuntime: cdef object _attribute_getter cdef object _attribute_setter cdef bint _unpack_returned_tuples - cdef MemoryStatus* _memory_status + cdef MemoryStatus _memory_status def __cinit__(self, encoding='UTF-8', source_encoding=None, attribute_filter=None, attribute_handlers=None, @@ -266,10 +266,9 @@ cdef class LuaRuntime: if max_memory is None: L = lua.luaL_newstate() + self._memory_status.limit = -1 else: - memory_status = calloc(1, sizeof(MemoryStatus)) - self._memory_status = memory_status - L = lua.lua_newstate(&_lua_alloc_restricted, self._memory_status) + L = lua.lua_newstate(&_lua_alloc_restricted, &self._memory_status) if L is NULL: raise LuaError("Failed to initialise Lua runtime") @@ -315,9 +314,6 @@ cdef class LuaRuntime: if self._state is not NULL: lua.lua_close(self._state) self._state = NULL - if self._memory_status is not NULL: - free(self._memory_status) - self._memory_status = NULL def get_max_memory(self, count_base=False): """ @@ -328,7 +324,7 @@ cdef class LuaRuntime: If ``count_base`` is True, the base memory used by the lua runtime will be included in the limit. """ - if self._memory_status is NULL: + if self._memory_status.limit >= 0: return None elif count_base: return self._memory_status.limit @@ -540,7 +536,7 @@ cdef class LuaRuntime: RuntimeError. """ cdef size_t used - if self._memory_status is NULL: + if self._memory_status.limit < 0: raise RuntimeError("max_memory must be set on LuaRuntime creation") elif max_memory <= 0: self._memory_status.limit = 0 From f7659e4c18f077077007df2ff7cbff4ac307893c Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 26 Sep 2022 09:24:37 +0200 Subject: [PATCH 42/55] count_base -> total --- lupa/_lupa.pyx | 18 +++++++++--------- lupa/tests/test.py | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 0a7bc05f..a0837126 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -315,33 +315,33 @@ cdef class LuaRuntime: lua.lua_close(self._state) self._state = NULL - def get_max_memory(self, count_base=False): + def get_max_memory(self, total=False): """ Maximum memory allowed to be used by this LuaRuntime. 0 indicates no limit meanwhile None indicates that the default lua allocator is being used and ``set_max_memory()`` cannot be used. - If ``count_base`` is True, the base memory used by the lua runtime + If ``total`` is True, the base memory used by the lua runtime will be included in the limit. """ if self._memory_status.limit >= 0: return None - elif count_base: + elif total: return self._memory_status.limit return self._memory_status.limit - self._memory_status.base_usage - def get_memory_used(self, count_base=False): + def get_memory_used(self, total=False): """ Memory currently in use. This is None if the default lua allocator is used and 0 if ``max_memory`` is 0. - If ``count_base`` is True, the base memory used by the lua runtime + If ``total`` is True, the base memory used by the lua runtime will be included. """ if self._memory_status is NULL: return None - elif count_base: + elif total: return self._memory_status.used return self._memory_status.used - self._memory_status.base_usage @@ -525,11 +525,11 @@ cdef class LuaRuntime: lua.lua_settop(L, old_top) unlock_runtime(self) - def set_max_memory(self, size_t max_memory, count_base=False): + def set_max_memory(self, size_t max_memory, total=False): """Set maximum allowed memory for this LuaRuntime. If `max_memory` is 0, there will be no limit. - If ``count_base`` is True, the base memory used by the LuaRuntime itself + If ``total`` is True, the base memory used by the LuaRuntime itself will be included in the memory limit. If max_memory was set to None during creation, this will raise a @@ -540,7 +540,7 @@ cdef class LuaRuntime: raise RuntimeError("max_memory must be set on LuaRuntime creation") elif max_memory <= 0: self._memory_status.limit = 0 - elif count_base: + elif total: self._memory_status.limit = max_memory else: self._memory_status.limit = self._memory_status.base_usage + max_memory diff --git a/lupa/tests/test.py b/lupa/tests/test.py index a5b7aef2..65db0c2c 100644 --- a/lupa/tests/test.py +++ b/lupa/tests/test.py @@ -3009,16 +3009,16 @@ def setUp(self): def test_getters(self): self.assertEqual(self.lua.get_memory_used(), 0) - self.assertGreater(self.lua.get_memory_used(count_base=True), 0) + self.assertGreater(self.lua.get_memory_used(total=True), 0) self.assertEqual(self.lua.get_max_memory(), 10000) - self.assertGreater(self.lua.get_max_memory(count_base=True), 10000) + self.assertGreater(self.lua.get_max_memory(total=True), 10000) self.lua.set_max_memory(1000000) self.assertEqual(self.lua.get_memory_used(), 0) - self.assertGreater(self.lua.get_memory_used(count_base=True), 0) + self.assertGreater(self.lua.get_memory_used(total=True), 0) self.assertEqual(self.lua.get_max_memory(), 1000000) - self.assertGreater(self.lua.get_max_memory(count_base=True), 1000000) - self.lua.set_max_memory(1000000, count_base=True) - self.assertEqual(self.lua.get_max_memory(count_base=True), 1000000) + self.assertGreater(self.lua.get_max_memory(total=True), 1000000) + self.lua.set_max_memory(1000000, total=True) + self.assertEqual(self.lua.get_max_memory(total=True), 1000000) self.assertLess(self.lua.get_max_memory(), 1000000) def test_not_enough_memory(self): From 4287c239555b990f848cc5d90a9d0f9adace46ad Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 26 Sep 2022 09:25:34 +0200 Subject: [PATCH 43/55] limit fixes --- lupa/_lupa.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index a0837126..42575a3e 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -324,7 +324,7 @@ cdef class LuaRuntime: If ``total`` is True, the base memory used by the lua runtime will be included in the limit. """ - if self._memory_status.limit >= 0: + if self._memory_status.limit < 0: return None elif total: return self._memory_status.limit @@ -339,7 +339,7 @@ cdef class LuaRuntime: If ``total`` is True, the base memory used by the lua runtime will be included. """ - if self._memory_status is NULL: + if self._memory_status.limit < 0: return None elif total: return self._memory_status.used From 5e2e67386f4a1dbcd76bcb199e6ca300f14a6f32 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 26 Sep 2022 09:57:54 +0200 Subject: [PATCH 44/55] attempt to document in readme --- README.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/README.rst b/README.rst index 2c53f0fc..5e486f02 100644 --- a/README.rst +++ b/README.rst @@ -913,6 +913,52 @@ setter function implementations for a ``LuaRuntime``: AttributeError: not allowed to write attribute "noway" +Restricting Lua Memory Usage +---------------------------- + +Lupa provides a simple mechanism to control the maximum memory +usage of the Lua Runtime since version 2.0. +By default Lupa does not interfere with Lua's memory allocation, to opt-in +you must set the ``max_memory`` when creating the LuaRuntime. + +Lupa provides three helper functions for controlling and getting the +memory usage of the LuaRuntime: + +1. ``get_memory_used(total=False)`` to get the current memory + usage of the LuaRuntime. + +2. ``get_max_memory(total=False)`` to get the current memory limit. + ``0`` means there is no memory limitation. + +3. ``set_max_memory(max_memory, total=False)`` to change the memory limit. + Values below or equal to 0 mean no limit. + +There is always some memory used by the LuaRuntime itself (around ~20KiB, +depends on your lua version and other factors) which is excluded from all +calculations unless specifying ``total=True``. + +.. code:: python + + >>> lua = LuaRuntime(max_memory=0) # 0 for unlimited, default is None + >>> lua.get_memory_used() # memory used by your code + 0 + >>> lua.get_memory_used(total=True) # includes memory used by the runtime itself + 21358 # this will depend on your lua version and other factors + + +Lua code hitting the memory limit will receive memory errors: + +.. code:: python + + >>> lua.set_max_memory(100) + >>> lua.eval("string.rep('a', 1000)") + Traceback (most recent call last): + ... + lupa.LuaMemoryError: not enough memory + +``LuaMemoryError`` inherits from ``LuaError`` and ``MemoryError``. + + Importing Lua binary modules ---------------------------- From 48af7ac3905a7b4c5c4972840df28a3e46ec51b1 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 26 Sep 2022 10:06:51 +0200 Subject: [PATCH 45/55] Update README.rst Co-authored-by: scoder --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 5e486f02..9169de26 100644 --- a/README.rst +++ b/README.rst @@ -921,8 +921,8 @@ usage of the Lua Runtime since version 2.0. By default Lupa does not interfere with Lua's memory allocation, to opt-in you must set the ``max_memory`` when creating the LuaRuntime. -Lupa provides three helper functions for controlling and getting the -memory usage of the LuaRuntime: +The ``LuaRuntime`` provides three methods for controlling and reading the +memory usage: 1. ``get_memory_used(total=False)`` to get the current memory usage of the LuaRuntime. From 0fc83c2e02bc5dffa1aa9ae5481f5c04348ac42f Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 26 Sep 2022 10:06:58 +0200 Subject: [PATCH 46/55] Update README.rst Co-authored-by: scoder --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 9169de26..e6d6bf2d 100644 --- a/README.rst +++ b/README.rst @@ -934,8 +934,8 @@ memory usage: Values below or equal to 0 mean no limit. There is always some memory used by the LuaRuntime itself (around ~20KiB, -depends on your lua version and other factors) which is excluded from all -calculations unless specifying ``total=True``. +depending on your lua version and other factors) which is excluded from all +calculations unless you specify ``total=True``. .. code:: python From 7b49c69675a4597825fc2464944616254ed8e7f3 Mon Sep 17 00:00:00 2001 From: scoder Date: Mon, 26 Sep 2022 10:07:39 +0200 Subject: [PATCH 47/55] Move struct declaration before first usage --- lupa/_lupa.pyx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 42575a3e..368a5d6f 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -113,6 +113,12 @@ else: # probably not smaller LUA_MININTEGER, LUA_MAXINTEGER = (CHAR_MIN, CHAR_MAX) +cdef struct MemoryStatus: + size_t used + size_t base_usage + size_t limit + + class LuaError(Exception): """Base class for errors in the Lua runtime. """ @@ -1813,11 +1819,6 @@ cdef tuple unpack_multiple_lua_results(LuaRuntime runtime, lua_State *L, int nar # bounded memory allocation -cdef struct MemoryStatus: - size_t used - size_t base_usage - size_t limit - cdef void* _lua_alloc_restricted(void* ud, void* ptr, size_t old_size, size_t new_size) nogil: # adapted from https://stackoverflow.com/a/9672205 # print(ud, ptr, old_size, new_size) From 396e9ddbc556ee39f34900b06fb1fce3d7f5db75 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 26 Sep 2022 10:15:06 +0200 Subject: [PATCH 48/55] Update README.rst Co-authored-by: scoder --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e6d6bf2d..ed0933d9 100644 --- a/README.rst +++ b/README.rst @@ -942,8 +942,8 @@ calculations unless you specify ``total=True``. >>> lua = LuaRuntime(max_memory=0) # 0 for unlimited, default is None >>> lua.get_memory_used() # memory used by your code 0 - >>> lua.get_memory_used(total=True) # includes memory used by the runtime itself - 21358 # this will depend on your lua version and other factors + >>> total_lua_memory = lua.get_memory_used(total=True) # includes memory used by the runtime itself + >>> assert total_lua_memory > 0 # exact amount depends on your lua version and other factors Lua code hitting the memory limit will receive memory errors: From 21f29457357156691a3c613a6714529970ab0d4a Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 26 Sep 2022 10:20:08 +0200 Subject: [PATCH 49/55] fix issues --- lupa/_lupa.pyx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 368a5d6f..97496b1d 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -270,10 +270,13 @@ cdef class LuaRuntime: max_memory=None): cdef lua_State* L + self._memory_status.used = 0 # these need to be initiliazed + self._memory_status.base_usage = 0 if max_memory is None: L = lua.luaL_newstate() self._memory_status.limit = -1 else: + self._memory_status.limit = 0 L = lua.lua_newstate(&_lua_alloc_restricted, &self._memory_status) if L is NULL: raise LuaError("Failed to initialise Lua runtime") @@ -330,7 +333,7 @@ cdef class LuaRuntime: If ``total`` is True, the base memory used by the lua runtime will be included in the limit. """ - if self._memory_status.limit < 0: + if self._memory_status.limit == -1: return None elif total: return self._memory_status.limit @@ -345,7 +348,7 @@ cdef class LuaRuntime: If ``total`` is True, the base memory used by the lua runtime will be included. """ - if self._memory_status.limit < 0: + if self._memory_status.limit == -1: return None elif total: return self._memory_status.used @@ -542,7 +545,7 @@ cdef class LuaRuntime: RuntimeError. """ cdef size_t used - if self._memory_status.limit < 0: + if self._memory_status.limit == -1: raise RuntimeError("max_memory must be set on LuaRuntime creation") elif max_memory <= 0: self._memory_status.limit = 0 From 40cf327ea77d8ae399188e78cb169f2a796d9a50 Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 26 Sep 2022 10:27:20 +0200 Subject: [PATCH 50/55] Update README.rst Co-authored-by: scoder --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ed0933d9..24738f7b 100644 --- a/README.rst +++ b/README.rst @@ -951,10 +951,10 @@ Lua code hitting the memory limit will receive memory errors: .. code:: python >>> lua.set_max_memory(100) - >>> lua.eval("string.rep('a', 1000)") + >>> lua.eval("string.rep('a', 1000)") # doctest: +ELLIPSIS Traceback (most recent call last): ... - lupa.LuaMemoryError: not enough memory + lupa...LuaMemoryError: not enough memory ``LuaMemoryError`` inherits from ``LuaError`` and ``MemoryError``. From 07a6b84fa372d606b4aa8464173a1414ebda1283 Mon Sep 17 00:00:00 2001 From: scoder Date: Mon, 26 Sep 2022 11:15:36 +0200 Subject: [PATCH 51/55] Remove unnecessary initialisations --- lupa/_lupa.pyx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 97496b1d..ea404398 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -270,13 +270,10 @@ cdef class LuaRuntime: max_memory=None): cdef lua_State* L - self._memory_status.used = 0 # these need to be initiliazed - self._memory_status.base_usage = 0 if max_memory is None: L = lua.luaL_newstate() - self._memory_status.limit = -1 + self._memory_status.limit = -1 else: - self._memory_status.limit = 0 L = lua.lua_newstate(&_lua_alloc_restricted, &self._memory_status) if L is NULL: raise LuaError("Failed to initialise Lua runtime") From 8390d1f19668099ce0d91a26b21fa2e94c48f44e Mon Sep 17 00:00:00 2001 From: Leo Developer Date: Mon, 26 Sep 2022 12:23:44 +0200 Subject: [PATCH 52/55] Update README.rst Co-authored-by: scoder --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 24738f7b..8465e324 100644 --- a/README.rst +++ b/README.rst @@ -954,7 +954,7 @@ Lua code hitting the memory limit will receive memory errors: >>> lua.eval("string.rep('a', 1000)") # doctest: +ELLIPSIS Traceback (most recent call last): ... - lupa...LuaMemoryError: not enough memory + ...LuaMemoryError: not enough memory ``LuaMemoryError`` inherits from ``LuaError`` and ``MemoryError``. From 5ae6a852eb4d500ed2ab2b9f1f7d86729ef6f2ab Mon Sep 17 00:00:00 2001 From: scoder Date: Tue, 27 Sep 2022 13:38:13 +0200 Subject: [PATCH 53/55] Ignore fully qualified exception name in doctest (Py2/3). --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 8465e324..40ec9a0c 100644 --- a/README.rst +++ b/README.rst @@ -951,10 +951,10 @@ Lua code hitting the memory limit will receive memory errors: .. code:: python >>> lua.set_max_memory(100) - >>> lua.eval("string.rep('a', 1000)") # doctest: +ELLIPSIS + >>> lua.eval("string.rep('a', 1000)") # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... - ...LuaMemoryError: not enough memory + lupa.LuaMemoryError: not enough memory ``LuaMemoryError`` inherits from ``LuaError`` and ``MemoryError``. From 2b9cd9623506e1a970b65527eff0b759baf3abd6 Mon Sep 17 00:00:00 2001 From: scoder Date: Wed, 16 Nov 2022 10:57:31 +0100 Subject: [PATCH 54/55] size_t cannot be < 0 --- lupa/_lupa.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index ea404398..1d5926ae 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -544,7 +544,7 @@ cdef class LuaRuntime: cdef size_t used if self._memory_status.limit == -1: raise RuntimeError("max_memory must be set on LuaRuntime creation") - elif max_memory <= 0: + elif max_memory == 0: self._memory_status.limit = 0 elif total: self._memory_status.limit = max_memory From 79608b8e9af4c96bd7ee29cd106afdb6ce64ca22 Mon Sep 17 00:00:00 2001 From: scoder Date: Wed, 16 Nov 2022 10:58:49 +0100 Subject: [PATCH 55/55] Prevent usage of our special limit value -1. --- lupa/_lupa.pyx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lupa/_lupa.pyx b/lupa/_lupa.pyx index 1d5926ae..4b2bf484 100644 --- a/lupa/_lupa.pyx +++ b/lupa/_lupa.pyx @@ -315,6 +315,9 @@ cdef class LuaRuntime: self._memory_status.base_usage = self._memory_status.used if max_memory > 0: self._memory_status.limit = self._memory_status.base_usage + max_memory + # Prevent accidental (or deliberate) usage of our special value. + if self._memory_status.limit == -1: + self._memory_status.limit -= 1 def __dealloc__(self): if self._state is not NULL: @@ -550,6 +553,9 @@ cdef class LuaRuntime: self._memory_status.limit = max_memory else: self._memory_status.limit = self._memory_status.base_usage + max_memory + # Prevent accidental (or deliberate) usage of our special value. + if self._memory_status.limit == -1: + self._memory_status.limit -= 1 def set_overflow_handler(self, overflow_handler): """Set the overflow handler function that is called on failures to pass large numbers to Lua.