Ship multiple Lua versions in separate extension modules (GH-207)

* Build multiple Lupa extension modules for the different bundled Lua versions.
* Test all Lupa variants with all Lua versions, not just the latest.
* setup.py: Make sure we always remove all known options from the command line, not just the ones that happen to be relevant.
* Disable Py27 builds in appveyor due to certificate issues with the old git version. We still have Windows builds in Github Actions.
* Improve output in case of LuaJIT build failures.
* Import the latest extension module only on demand from "import lupa" (on Py3.7 and later, which have PEP 562: module __getattr__).
* Update versions of dependencies and build matrix.

Closes #196

.github/workflows/ci.yml

@@ -10,9 +10,9 @@ jobs:
       fail-fast: false
 
       matrix:
-        os: [windows-2019, ubuntu-18.04, macos-10.15]
-        python-version: ["2.7", "3.10", "3.9", "3.8", "3.7", "3.6", "3.5", "pypy-2.7", "pypy-3.7", "pypy-3.8"]
-        lua-version: ["bundle", "lua5.2", "lua5.3", "luajit-5.1"]
+        os: [windows-2019, ubuntu-20.04, macos-11]
+        python-version: ["2.7", "3.10", "3.9", "3.8", "3.7", "3.6", "pypy-2.7", "pypy-3.7", "pypy-3.8"]
+        lua-version: ["bundle", "lua5.3", "lua5.2", "luajit-5.1"]
 
         exclude:
           - os: windows-2019
@@ -23,21 +23,27 @@ jobs:
             lua-version: lua5.2
           - os: windows-2019
             lua-version: lua5.3
+          - os: windows-2019
+            lua-version: lua5.4
           - os: windows-2019
             lua-version: luajit-5.1
-          - os: macos-10.15
+          - os: macos-11
             python-version: 2.7
-          - os: macos-10.15
+          - os: macos-11
             lua-version: lua5.2
-          - os: macos-10.15
+          - os: macos-11
             lua-version: lua5.3
-          - os: macos-10.15
+          - os: macos-11
+            lua-version: lua5.4
+          - os: macos-11
             lua-version: luajit-5.1
 
     runs-on: ${{ matrix.os }}
 
     env:
       CFLAGS_LTO: ${{ contains(matrix.lua-version, 'bundle')  && (contains(matrix.os, 'windows') && '/LTCG' || '-flto') || '' }}
+      CFLAGS: ${{ contains(matrix.os, 'windows') && '/O2' || '-O2 -fPIC' }} -g
+      MACOSX_DEPLOYMENT_TARGET: "10.15"
 
     steps:
     - uses: actions/checkout@v2
@@ -58,18 +64,18 @@ jobs:
       run: sudo apt-get install lib${{ matrix.lua-version }}-dev
 
     - name: Build wheel
-      run: python -m pip install -r requirements.txt && python setup.py sdist bdist_wheel
+      run: python -m pip install -r requirements.txt && python setup.py sdist ${{ contains(matrix.python-version, '3.') && 'build_ext -j5' || '' }} bdist_wheel
       env:
         SETUP_OPTIONS: ${{ !contains(matrix.lua-version, 'luajit') && (contains(matrix.lua-version, 'bundle') && '--use-bundle' || '--no-luajit') || '' }}
-        CFLAGS: ${{ env.CFLAGS_LTO }} -g
+        CFLAGS: ${{ env.CFLAGS }} ${{ env.CFLAGS_LTO }}
         LDFLAGS: ${{ env.CFLAGS_LTO }}
 
     - name: Run tests
       run: python setup.py test
       continue-on-error: ${{ contains(matrix.python-version, 'pypy') }}
       env:
         SETUP_OPTIONS: ${{ !contains(matrix.lua-version, 'luajit') && (contains(matrix.lua-version, 'bundle') && '--use-bundle' || '--no-luajit') || '' }}
-        CFLAGS: ${{ env.CFLAGS_LTO }} -g
+        CFLAGS: ${{ env.CFLAGS }} ${{ env.CFLAGS_LTO }}
         LDFLAGS: ${{ env.CFLAGS_LTO }}
 
     - name: Upload wheels

.github/workflows/wheels.yml

@@ -46,32 +46,42 @@ jobs:
 
       matrix:
         image:
-          - manylinux1_x86_64
-          - manylinux1_i686
-          #- manylinux2010_x86_64
-          #- manylinux2010_i686
-          - manylinux_2_24_x86_64
-          - manylinux_2_24_i686
-          - manylinux_2_24_aarch64
+          - manylinux2014_x86_64
+          - manylinux2014_i686
+          - manylinux_2_28_x86_64
+          - manylinux_2_28_i686
+          - manylinux_2_28_aarch64
           - musllinux_1_1_x86_64
+          - musllinux_1_1_aarch64
           #- manylinux_2_24_ppc64le
           #- manylinux_2_24_ppc64le
           #- manylinux_2_24_s390x
         pyversion: ["*"]
 
         exclude:
-          - image: manylinux_2_24_aarch64
+          - image: manylinux_2_28_aarch64
+            pyversion: "*"
+          - image: musllinux_1_1_aarch64
             pyversion: "*"
         include:
           - image: manylinux2014_aarch64
             pyversion: "cp36*"
-          - image: manylinux_2_24_aarch64
+          - image: manylinux_2_28_aarch64
             pyversion: "cp37*"
-          - image: manylinux_2_24_aarch64
+          - image: manylinux_2_28_aarch64
             pyversion: "cp38*"
-          - image: manylinux_2_24_aarch64
+          - image: manylinux_2_28_aarch64
             pyversion: "cp39*"
-          - image: manylinux_2_24_aarch64
+          - image: manylinux_2_28_aarch64
+            pyversion: "cp310*"
+
+          - image: musllinux_1_1_aarch64
+            pyversion: "cp37*"
+          - image: musllinux_1_1_aarch64
+            pyversion: "cp38*"
+          - image: musllinux_1_1_aarch64
+            pyversion: "cp39*"
+          - image: musllinux_1_1_aarch64
             pyversion: "cp310*"
 
     steps:
@@ -111,21 +121,19 @@ jobs:
       fail-fast: false
 
       matrix:
-        os: [macos-10.15, windows-latest]
-        #os: [macos-10.15, windows-latest, macOS-M1]
-        #os: [macos-10.15, macOS-M1]
-        #os: [macos-10.15]
-        python_version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.7-v7.3.3", "pypy-3.8-v7.3.7"]
+        os: [macos-11, windows-latest]
+        #os: [macos-11, windows-latest, macOS-M1]
+        #os: [macos-11, macOS-M1]
+        #os: [macos-11]
+        python_version: ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.7-v7.3.3", "pypy-3.8-v7.3.7"]
 
         exclude:
           # outdated compilers and probably not worth supporting anymore
           - os: windows-latest
             python_version: 2.7
-          - os: windows-latest
-            python_version: 3.5
 
     runs-on: ${{ matrix.os }}
-    env: { MACOSX_DEPLOYMENT_TARGET: 10.14 }
+    env: { MACOSX_DEPLOYMENT_TARGET: 10.15 }
 
     steps:
     - uses: actions/checkout@v2

.gitignore

@@ -14,6 +14,7 @@ MANIFEST
 *.patch
 wheel*/
 lupa/version.py
+lupa/lua*.pyx
 
 # Vim swapfiles
 *.swp

.gitmodules

@@ -1,12 +1,15 @@
+[submodule "third-party/lua54"]
+	path = third-party/lua54
+	url = https://github.com/lua/lua.git
 [submodule "third-party/lua53"]
 	path = third-party/lua53
 	url = https://github.com/lua/lua.git
-[submodule "third-party/lua54"]
-	path = third-party/lua54
+[submodule "third-party/lua52"]
+	path = third-party/lua52
 	url = https://github.com/lua/lua.git
-[submodule "third-party/luajit20"]
-	path = third-party/luajit20
-	url = https://luajit.org/git/luajit.git
 [submodule "third-party/luajit21"]
 	path = third-party/luajit21
 	url = https://luajit.org/git/luajit.git
+[submodule "third-party/luajit20"]
+	path = third-party/luajit20
+	url = https://luajit.org/git/luajit.git

Makefile

@@ -13,7 +13,8 @@ MANYLINUX_IMAGES= \
 	manylinux_2_24_aarch64 \
 	manylinux_2_24_ppc64le \
 	manylinux_2_24_s390x \
-	musllinux_1_1_x86_64
+	musllinux_1_1_x86_64 \
+	musllinux_1_1_aarch64
 
 .PHONY: all local sdist test clean realclean
 
@@ -29,7 +30,8 @@ test: local
 	PYTHONPATH=. $(PYTHON) -m lupa.tests.test
 
 clean:
-	rm -fr build lupa/_lupa.so
+	rm -fr build lupa/_lupa*.so lupa/lua*.pyx lupa/*.c
+	@for dir in third-party/*/; do $(MAKE) -C $${dir} clean; done
 
 realclean: clean
 	rm -fr lupa/_lupa.c

README.rst

@@ -37,8 +37,8 @@ Major features
 
 * tested with Python 2.7/3.5 and later
 
-* written for LuaJIT2 (tested with LuaJIT 2.0.2), but also works
-  with the normal Lua interpreter (5.1 and later)
+* ships with Lua 5.3 and 5.4 (works with Lua 5.1 and later)
+  as well as LuaJIT 2.0 and 2.1 on systems that support it.
 
 * easy to hack on and extend as it is written in Cython, not C
 
@@ -80,6 +80,31 @@ switching between the two languages at runtime, based on the tradeoff
 between simplicity and speed.
 
 
+Which Lua version?
+------------------
+
+The binary wheels include different Lua versions as well as LuaJIT, if supported.
+By default, ``import lupa`` uses the latest Lua version, but you can choose
+a specific one via import:
+
+.. code:: python
+
+    try:
+        import lupa.luajit20 as lupa
+    except ImportError:
+        try:
+            import lupa.lua54 as lupa
+        except ImportError:
+            try:
+                import lupa.lua53 as lupa
+            except ImportError:
+                import lupa
+
+    print(f"Using {lupa.LuaRuntime().lua_implementation} (compiled with {lupa.LUA_VERSION})")
+
+Note that LuaJIT 2.1 may also be included (as ``luajit21``) but is currently in Alpha state.
+
+
 Examples
 --------
 

appveyor.yml

@@ -3,10 +3,11 @@ image: Visual Studio 2019
 
 environment:
   matrix:
-  - python: 27
-    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013
-  - python: 27-x64
-    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013
+# Disable Py27 builds since they run into certificate issues when retrieving the LuaJIT git submodule.
+#  - python: 27
+#    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013
+#  - python: 27-x64
+#    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013
   - python: 310
   - python: 310-x64
   - python: 39

lupa/__init__.py

@@ -27,9 +27,54 @@ def _try_import_with_global_library_symbols():
 
 del _try_import_with_global_library_symbols
 
-# the following is all that should stay in the namespace:
 
-from lupa._lupa import *
+# Find the implementation with the latest Lua version available.
+_newest_lib = None
+
+
+def _import_newest_lib():
+    global _newest_lib
+    if _newest_lib is not None:
+        return _newest_lib
+
+    import os.path
+    import re
+
+    package_dir = os.path.dirname(__file__)
+    modules = [
+        match.groups() for match in (
+            re.match(r"((lua[a-z]*)([0-9]*))\..*", filename)
+            for filename in os.listdir(package_dir)
+        )
+        if match
+    ]
+    if not modules:
+        raise RuntimeError("Failed to import Lupa binary module.")
+    # prefer Lua over LuaJIT and high versions over low versions.
+    module_name = max(modules, key=lambda m: (m[1] == 'lua', tuple(map(int, m[2] or '0'))))
+    _newest_lib = __import__(module_name[0], level=1, fromlist="*", globals=globals())
+
+    return _newest_lib
+
+
+def __getattr__(name):
+    """
+    Get a name from the latest available Lua (or LuaJIT) module.
+    Imports the module as needed.
+    """
+    lua = _newest_lib if _newest_lib is not None else _import_newest_lib()
+    return getattr(lua, name)
+
+
+import sys
+if sys.version_info < (3, 7):
+    # Module level "__getattr__" requires Py3.7 or later => import latest Lua now
+    _import_newest_lib()
+    globals().update(
+        (name, getattr(_newest_lib, name))
+        for name in _newest_lib.__all__
+    )
+del sys
 
 try:
     from lupa.version import __version__

lupa/tests/__init__.py

@@ -9,6 +9,68 @@
 import lupa
 
 
+class LupaTestCase(unittest.TestCase):
+    """
+    Subclasses can use 'self.lupa' to get the test module, which build_suite_for_module() below will vary.
+    """
+    lupa = lupa
+
+
+def find_lua_modules():
+    modules = [lupa]
+    imported = set()
+    for filename in os.listdir(os.path.dirname(os.path.dirname(__file__))):
+        if not filename.startswith('lua'):
+            continue
+        module_name = "lupa." + filename.partition('.')[0]
+        if module_name in imported:
+            continue
+        try:
+            module = __import__(module_name, fromlist='*', level=0)
+        except ImportError:
+            pass
+        else:
+            imported.add(module_name)
+            modules.append(module)
+
+    return modules
+
+
+def build_suite_for_modules(loader, test_module_globals):
+    suite = unittest.TestSuite()
+    all_lua_modules = find_lua_modules()
+
+    for module in all_lua_modules[1:]:
+        suite.addTests(doctest.DocTestSuite(module))
+
+    def add_tests(cls):
+        tests = loader.loadTestsFromTestCase(cls)
+        suite.addTests(tests)
+
+    for name, test_class in test_module_globals.items():
+        if (not isinstance(test_class, type) or
+                not name.startswith('Test') or
+                not issubclass(test_class, unittest.TestCase)):
+            continue
+
+        if issubclass(test_class, LupaTestCase):
+            prefix = test_class.__name__ + "_"
+            qprefix = getattr(test_class, '__qualname__', test_class.__name__) + "_"
+
+            for module in all_lua_modules:
+                class TestClass(test_class):
+                    lupa = module
+
+                module_name = module.__name__.rpartition('.')[2]
+                TestClass.__name__ = prefix + module_name
+                TestClass.__qualname__ = qprefix + module_name
+                add_tests(TestClass)
+        else:
+            add_tests(test_class)
+
+    return suite
+
+
 def suite():
     test_dir = os.path.abspath(os.path.dirname(__file__))
 
@@ -18,7 +80,6 @@ def suite():
             tests.append('lupa.tests.'  + filename[:-3])
 
     suite = unittest.defaultTestLoader.loadTestsFromNames(tests)
-    suite.addTest(doctest.DocTestSuite(lupa._lupa))
 
     # Long version of
     # suite.addTest(doctest.DocFileSuite('../../README.rst'))