Build and test PyWavelets Pyodide wheels in CI (#701)

The main change is adding a Pyodide CI job. Other changes:

* Add some relevant files and venvs to ignore
* ensure file handles get closed and use importlib.resources instead of `__file__`
* cache data loading
* Convert `ecg` and `sst_nino3` data files from .npy to .npz (NumPy compressed archive)
  for Pyodide compatibility
* Properly skip tests for WASM and wherever threading isn't availables
* import `importlib.resources`, mark `/pywt/data/` as constant

Co-authored-by: Ralf Gommers <[email protected]>

.github/workflows/emscripten.yml

@@ -0,0 +1,60 @@
+name: Test Pyodide build for PyWavelets
+
+on:
+  push:
+    branches:
+      - master
+      - v1.**
+  pull_request:
+    branches:
+      - master
+      - v1.**
+
+env:
+  FORCE_COLOR: 3
+
+jobs:
+  build_wasm_emscripten:
+    name: Build PyWavelets for Pyodide
+    runs-on: ubuntu-latest
+    # Uncomment the following line to test changes on a fork
+    # if: github.repository == 'PyWavelets/pywt'
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        id: setup-python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.11.2'
+
+      - name: Install prerequisites
+        run: |
+          python -m pip install pyodide-build "pydantic<2"
+          echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV
+
+      - name: Set up Emscripten toolchain
+        uses: mymindstorm/setup-emscripten@v14
+        with:
+          version: ${{ env.EMSCRIPTEN_VERSION }}
+          actions-cache-folder: emsdk-cache
+
+      - name: Set up Node.js
+        uses: actions/[email protected]
+        with:
+          node-version: '18'
+
+      - name: Build PyWavelets
+        run: |
+          pyodide build
+
+      - name: Install and test wheel
+        run: |
+          pyodide venv .venv-pyodide
+          source .venv-pyodide/bin/activate
+          pip install dist/*.whl
+          pushd demo
+          pip install matplotlib pytest
+          python -c "import pywt; print(pywt.__version__)"
+          pytest --pyargs pywt

.gitignore

@@ -2,6 +2,8 @@ __pycache__
 *.py[co]
 *.pyd
 *.so
+.DS_Store
+.pytest_cache/
 
 # Packages
 *.egg
@@ -32,6 +34,12 @@ cythonize.dat
 pywt/version.py
 build.log
 
+# Virtual environments
+.env/
+env/
+venv/
+.venv/
+
 # asv files
 asv/env
 asv/html

README.rst

@@ -65,7 +65,7 @@ For more usage examples see the `demo`_ directory in the source package.
 Installation
 ------------
 
-PyWavelets supports `Python`_ >=3.7, and is only dependent on `NumPy`_
+PyWavelets supports `Python`_ >=3.9, and is only dependent on `NumPy`_
 (supported versions are currently ``>= 1.14.6``). To pass all of the tests,
 `Matplotlib`_ is also required. `SciPy`_ is also an optional dependency. When
 present, FFT-based continuous wavelet transforms will use FFTs from SciPy

demo/batch_processing.py

@@ -24,8 +24,7 @@
     from concurrent import futures
 except ImportError:
     raise ImportError(
-        "This demo requires concurrent.futures.  It can be installed for "
-        "for python 2.x via:  pip install futures")
+        "This demo requires concurrent.futures. If you are on WebAssembly, this is not available.")
 
 import numpy as np
 from numpy.testing import assert_array_equal

pywt/_pytest.py

@@ -1,6 +1,7 @@
 """common test-related code."""
 import os
 import sys
+import platform
 import multiprocessing
 import numpy as np
 import pytest
@@ -18,15 +19,18 @@
            ]
 
 try:
-    if sys.version_info[0] == 2:
-        import futures
-    else:
-        from concurrent import futures
+    from concurrent import futures
     max_workers = multiprocessing.cpu_count()
     futures_available = True
 except ImportError:
     futures_available = False
     futures = None
+    max_workers = 1
+
+# Check if running on Emscripten/WASM, and skip tests that require concurrency.
+# Relevant issue: https://github.com/pyodide/pyodide/issues/237
+IS_WASM = (sys.platform == "emscripten") or (platform.machine() in ["wasm32", "wasm64"])
+
 
 # check if pymatbridge + MATLAB tests should be run
 matlab_result_dict_dwt = None
@@ -57,7 +61,9 @@
     matlab_result_dict_dwt = np.load(matlab_data_file_dwt)
 
 uses_futures = pytest.mark.skipif(
-    not futures_available, reason='futures not available')
+    not futures_available or IS_WASM,
+    reason='futures is not available, or running via Pyodide/WASM.')
+    # not futures_available, reason='futures not available')
 uses_matlab = pytest.mark.skipif(
     matlab_missing, reason='pymatbridge and/or Matlab not available')
 uses_pymatbridge = pytest.mark.skipif(

pywt/data/_readers.py

@@ -1,8 +1,14 @@
+import functools
+import importlib.resources
 import os
 
 import numpy as np
 
 
+_DATADIR = importlib.resources.files('pywt.data')
+
+
+@functools.cache
 def ascent():
     """
     Get an 8-bit grayscale bit-depth, 512 x 512 derived image for
@@ -36,11 +42,13 @@ def ascent():
     >>> plt.show() # doctest: +SKIP
 
     """
-    fname = os.path.join(os.path.dirname(__file__), 'ascent.npz')
-    ascent = np.load(fname)['data']
+    with importlib.resources.as_file(_DATADIR.joinpath('ascent.npz')) as f:
+        ascent = np.load(f)['data']
+
     return ascent
 
 
+@functools.cache
 def aero():
     """
     Get an 8-bit grayscale bit-depth, 512 x 512 derived image for
@@ -71,11 +79,13 @@ def aero():
     >>> plt.show() # doctest: +SKIP
 
     """
-    fname = os.path.join(os.path.dirname(__file__), 'aero.npz')
-    aero = np.load(fname)['data']
+    with importlib.resources.as_file(_DATADIR.joinpath('aero.npz')) as f:
+        aero = np.load(f)['data']
+
     return aero
 
 
+@functools.cache
 def camera():
     """
     Get an 8-bit grayscale bit-depth, 512 x 512 derived image for
@@ -117,11 +127,13 @@ def camera():
     >>> plt.show() # doctest: +SKIP
 
     """
-    fname = os.path.join(os.path.dirname(__file__), 'camera.npz')
-    camera = np.load(fname)['data']
+    with importlib.resources.as_file(_DATADIR.joinpath('camera.npz')) as f:
+        camera = np.load(f)['data']
+
     return camera
 
 
+@functools.cache
 def ecg():
     """
     Get 1024 points of an ECG timeseries.
@@ -147,11 +159,13 @@ def ecg():
     [<matplotlib.lines.Line2D object at ...>]
     >>> plt.show() # doctest: +SKIP
     """
-    fname = os.path.join(os.path.dirname(__file__), 'ecg.npy')
-    ecg = np.load(fname)
+    with importlib.resources.as_file(_DATADIR.joinpath('ecg.npz')) as f:
+        ecg = np.load(f)['data']
+
     return ecg
 
 
+@functools.cache
 def nino():
     """
     This data contains the averaged monthly sea surface temperature in degrees
@@ -183,8 +197,9 @@ def nino():
     [<matplotlib.lines.Line2D object at ...>]
     >>> plt.show() # doctest: +SKIP
     """
-    fname = os.path.join(os.path.dirname(__file__), 'sst_nino3.npy')
-    sst_csv = np.load(fname)
+    with importlib.resources.as_file(_DATADIR.joinpath('sst_nino3.npz')) as f:
+        sst_csv = np.load(f)['data']
+
     # sst_csv = pd.read_csv("http://www.cpc.ncep.noaa.gov/data/indices/ersst4.nino.mth.81-10.ascii", sep=' ', skipinitialspace=True)
     # take only full years
     n = int(np.floor(sst_csv.shape[0]/12.)*12.)

pywt/tests/test_concurrent.py

@@ -8,9 +8,11 @@
 import warnings
 import numpy as np
 from functools import partial
+
+import pytest
 from numpy.testing import assert_array_equal, assert_allclose
-from pywt._pytest import uses_futures, futures, max_workers
 
+from pywt._pytest import uses_futures, futures, max_workers
 import pywt