Merge pull request #426 from VarunUllanat/dev_graph

Adding windows and MacOS tests to dev_graph

.github/ci-environments/env-py310.yml

@@ -0,0 +1,39 @@
+name: pathml
+
+channels:
+  - conda-forge
+  - pytorch
+
+dependencies:
+  - python<3.11
+  - pip==23.3.2
+  - numpy==1.23.5
+  - scipy<=1.11.4
+  - scikit-image<=0.22.0
+  - matplotlib<=3.8.2
+  - openjdk<=18.0.0
+  - pytorch==1.13.1
+  - h5py==3.10.0
+  - dask<=2023.12.1
+  - pydicom==2.4.4
+  - pytest==7.4.3
+  - pre-commit<=3.6.0
+  - coverage==7.3.4
+  - networkx<=3.2.1
+  # - python-javabridge==4.0.3
+  - pip:
+      - python-bioformats==4.0.7
+      - python-javabridge==4.0.3
+      - protobuf==3.20.3
+      # - deepcell<=0.12.7
+      - onnx==1.14.0
+      - onnxruntime==1.15.1
+      - opencv-contrib-python==4.8.1.78
+      - openslide-python==1.3.1
+      - scanpy==1.9.6
+      - anndata<=0.10.3
+      - tqdm==4.66.1
+      - loguru==0.7.2
+      - pandas<=2.1.4
+      - torch-geometric==2.3.1
+      - jpype1==1.4.1

.github/ci-environments/env-py38.yml

@@ -0,0 +1,39 @@
+name: pathml
+
+channels:
+  - conda-forge
+  - pytorch
+
+dependencies:
+  - python<3.9
+  - pip==23.3.2
+  - numpy==1.23.5
+  - scipy<=1.11.4
+  - scikit-image<=0.22.0
+  - matplotlib<=3.8.2
+  - openjdk<=18.0.0
+  - pytorch==1.13.1
+  - h5py==3.10.0
+  - dask<=2023.12.1
+  - pydicom==2.4.4
+  - pytest==7.4.3
+  - pre-commit<=3.6.0
+  - coverage==7.3.4
+  - networkx<=3.2.1
+  # - python-javabridge==4.0.3
+  - pip:
+      - python-bioformats==4.0.7
+      - python-javabridge==4.0.3
+      - protobuf==3.20.3
+      # - deepcell<=0.12.7
+      - onnx==1.14.0
+      - onnxruntime==1.15.1
+      - opencv-contrib-python==4.8.1.78
+      - openslide-python==1.3.1
+      - scanpy==1.9.6
+      - anndata<=0.10.3
+      - tqdm==4.66.1
+      - loguru==0.7.2
+      - pandas<=2.1.4
+      - torch-geometric==2.3.1
+      - jpype1==1.4.1

.github/ci-environments/env-py39.yml

@@ -0,0 +1,39 @@
+name: pathml
+
+channels:
+  - conda-forge
+  - pytorch
+
+dependencies:
+  - python<3.10
+  - pip==23.3.2
+  - numpy==1.23.5
+  - scipy<=1.11.4
+  - scikit-image<=0.22.0
+  - matplotlib<=3.8.2
+  - openjdk<=18.0.0
+  - pytorch==1.13.1
+  - h5py==3.10.0
+  - dask<=2023.12.1
+  - pydicom==2.4.4
+  - pytest==7.4.3
+  - pre-commit<=3.6.0
+  - coverage==7.3.4
+  - networkx<=3.2.1
+  # - python-javabridge==4.0.3
+  - pip:
+      - python-bioformats==4.0.7
+      - python-javabridge==4.0.3
+      - protobuf==3.20.3
+      # - deepcell<=0.12.7
+      - onnx==1.14.0
+      - onnxruntime==1.15.1
+      - opencv-contrib-python==4.8.1.78
+      - openslide-python==1.3.1
+      - scanpy==1.9.6
+      - anndata<=0.10.3
+      - tqdm==4.66.1
+      - loguru==0.7.2
+      - pandas<=2.1.4
+      - torch-geometric==2.3.1
+      - jpype1==1.4.1

.github/workflows/tests-macos.yml

@@ -0,0 +1,66 @@
+name: Tests MacOS
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - dev_graph
+
+jobs:
+  build:
+    name: Build (${{ matrix.os }}, Python ${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest]
+        python-version: ["3.8"]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install openslide
+        shell: bash -l {0}
+        run: brew install openslide
+
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          miniforge-version: latest
+          use-mamba: true
+          python-version: ${{ matrix.python-version }}
+          activate-environment: pathml
+          environment-file: requirements/environment_mac.yml
+          channels: conda-forge
+
+      - name: Conda info
+        shell: bash -l {0}
+        run: conda info
+
+      - name: Conda Version
+        shell: bash -l {0}
+        run: conda --version
+
+      - name: Set default Temurin JDK 17
+        run: |
+          sudo update-java-alternatives --set temurin-17-jdk-amd64 || true
+          java -version
+
+      - name: Install PathML
+        shell: bash -l {0}      
+        run: pip install -e .
+
+      - name: Install torchvision
+        shell: bash -l {0}
+        run: pip install torchvision==0.14.1
+
+      - name: Test other modules with pytest and generate coverage
+        shell: bash -l {0}
+        run: |
+         COVERAGE_FILE=.coverage_others coverage run -m pytest -m "not slow and not exclude"
+
+      - name: Test tile_stitcher with pytest and generate coverage
+        shell: bash -l {0}
+        run: |
+         COVERAGE_FILE=.coverage_tilestitcher coverage run -m pytest tests/preprocessing_tests/test_tilestitcher.py

.github/workflows/tests-windows.yml

@@ -0,0 +1,115 @@
+name: Tests windows
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - dev_graph
+
+jobs:
+  build:
+    runs-on: windows-latest
+    strategy:
+      max-parallel: 5
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+    timeout-minutes: 120
+
+    steps:
+
+      - uses: actions/checkout@v2
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v2
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      - name: Print JAVA_HOME and PATH
+        run: |
+          echo "JAVA_HOME: $env:JAVA_HOME"
+          echo "PATH: $env:PATH"
+        shell: pwsh
+
+      - name: Set JDK_HOME environment variable
+        run: echo "JDK_HOME=${JAVA_HOME}" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-activate-base: false
+          activate-environment: pathml
+          environment-file: environment.yml
+          # environment-file: '.github/ci-environments/env-py39.yml'
+          # environment-file: .github/ci-environments/env-py${{ matrix.python-version == '3.10' && '310' || matrix.python-version == '3.9' && '39' || '38' }}.yml
+          mamba-version: "*"
+          python-version: ${{ matrix.python-version }}
+
+      # - name: Install JavaBridge and other dependencies
+      #   shell: bash -l {0}
+      #   run: |
+      #     pip install python-javabridge
+
+      - name: Run Python script to find JVM path
+        shell: bash -l {0}
+        run: |
+          python -c "import javabridge; print('javabridge.locate.find_javahome():', javabridge.locate.find_javahome()); print('javabridge.jutil._find_jvm_windows():', javabridge.jutil._find_jvm_windows())"
+    
+      - name: Print JAVA_HOME and PATH Second
+        run: |
+          echo "JAVA_HOME: $env:JAVA_HOME"
+          echo "PATH: $env:PATH"
+        shell: pwsh
+
+
+      - name: Download and Setup OpenSlide
+        run: |
+          # $zipFile = "openslide-win64-20231011.zip"
+          $zipFile = "openslide-win64-20230414.zip"
+          $extractPath = Join-Path $env:GITHUB_WORKSPACE "openslide-binaries"
+          # Invoke-WebRequest -Uri "https://github.com/openslide/openslide-bin/releases/download/v20231011/$zipFile" -OutFile $zipFile
+          Invoke-WebRequest -Uri "https://github.com/openslide/openslide-bin/releases/download/v20230414/$zipFile" -OutFile $zipFile
+          Expand-Archive -Path $zipFile -DestinationPath $extractPath
+          # $binPath = Join-Path $extractPath "openslide-win64-20231011/bin"
+          $binPath = Join-Path $extractPath "openslide-win64-20230414/bin"
+          Write-Host "Adding to PATH: $binPath"
+          echo "OPENSLIDE_PATH=$binPath" | Out-File -Append -FilePath $env:GITHUB_ENV
+          # echo "$binPath" | Out-File -Append -FilePath $env:GITHUB_PATH
+        shell: pwsh
+
+      # Optional: List files in OpenSlide bin directory to verify
+      - name: List files in OpenSlide bin directory
+        run: |
+          Get-ChildItem -Path $env:OPENSLIDE_PATH
+        shell: pwsh
+        env:
+          OPENSLIDE_PATH: ${{ env.OPENSLIDE_PATH }}
+
+      - name: Conda info
+        shell: bash -l {0}
+        run: |
+          conda info
+          conda list
+
+      - name: Install PathML
+        shell: bash -l {0}      
+        run: pip install -e .
+
+      - name: Install torchvision
+        shell: bash -l {0}
+        run: pip install torchvision==0.14.1
+
+      - name: Set OPENSLIDE_PATH
+        run: echo "OPENSLIDE_PATH=D:\a\pathml\pathml\openslide-binaries\openslide-win64-20231011\bin" >> $GITHUB_ENV
+        shell: pwsh
+
+      - name: Test other modules with pytest and generate coverage
+        shell: bash -l {0}
+        run: |
+          coverage run --data-file=.coverage_others -m pytest -m "not slow and not exclude"
+
+      - name: Test tile_stitcher with pytest and generate coverage
+        shell: bash -l {0}
+        run: |
+          coverage run --data-file=.coverage_tilestitcher -m pytest tests/preprocessing_tests/test_tilestitcher.py

pathml/preprocessing/tilestitcher.py

@@ -123,8 +123,8 @@ def _import_qupath_classes(self):
     def format_jvm_options(qupath_jars, memory):
         memory_option = f"-Xmx{memory}"
         formatted_classpath = [
-            path.replace("/", "\\") if platform.system() == "Windows" else path
-            for path in qupath_jars
+            (str(Path(path).as_posix()) if platform.system() != "Windows" else str(Path(path)))
+             for path in qupath_jars
         ]
         class_path_option = "-Djava.class.path=" + os.pathsep.join(formatted_classpath)
         return memory_option, class_path_option
@@ -204,6 +204,7 @@ def setup_bfconvert(self, bfconvert_dir):
             zipfile.BadZipFile,
             PermissionError,
             subprocess.CalledProcessError,
+            OSError,
         ) as e:
             raise BFConvertSetupError(f"Error setting up bfconvert: {e}")
 

setup.py

@@ -1,11 +1,13 @@
 import setuptools
 
-with open("README.md", "r") as fh:
+with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 about = {}
-with open("pathml/_version.py") as f:
+# It's also a good practice to specify encoding here, though it may not be necessary if _version.py contains only ASCII characters
+with open("pathml/_version.py", "r", encoding="utf-8") as f:
     exec(f.read(), about)
+
 version = about["__version__"]
 
 dependency_links = ["https://download.pytorch.org/whl/cu116"]

tests/conftest.py

@@ -3,6 +3,27 @@
 License: GNU GPL 2.0
 """
 
+# Pre-configuration to add the OpenSlide DLL directory to the system's environment variables
+import os
+import sys
+
+def configure_openslide_path():
+    print('Configuring OpenSlide path on', sys.platform)
+
+    # Check if the os.add_dll_directory function is available (Python 3.8+ on Windows)
+    if hasattr(os, 'add_dll_directory'):
+        openslide_path = os.getenv('OPENSLIDE_PATH')
+        if openslide_path:
+            print('Adding OpenSlide path:', openslide_path)
+            os.add_dll_directory(openslide_path)
+        else:
+            raise RuntimeError("OPENSLIDE_PATH environment variable is not set or incorrect.")
+    else:
+        print("os.add_dll_directory not available, ensure your Python version is 3.8 or higher on Windows")
+
+# Call the pre-configuration function before importing OpenSlide
+configure_openslide_path()
+
 import cv2
 import javabridge
 import numpy as np
@@ -12,7 +33,6 @@
 
 from pathml.core import Tile, VectraSlide, types
 
-
 def pytest_sessionfinish(session, exitstatus):
     """
     Pytest will not terminate if javabridge is not killed.
@@ -23,6 +43,16 @@ def pytest_sessionfinish(session, exitstatus):
     javabridge.kill_vm()
 
 
+@pytest.fixture(autouse=True)
+def remove_duplicate_paths():
+    yield  # Wait for the test to finish
+    # Split the PATH by the OS-specific path separator
+    paths = os.environ['PATH'].split(os.pathsep)
+    # Remove duplicates while preserving order
+    unique_paths = list(dict.fromkeys(paths))
+    # Join the unique paths back into a string and set it as the new PATH
+    os.environ['PATH'] = os.pathsep.join(unique_paths)
+
 def create_HE_tile():
     s = openslide.open_slide("tests/testdata/small_HE.svs")
     im_image = s.read_region(level=0, location=(900, 800), size=(500, 500))
@@ -44,7 +74,6 @@ def create_HE_tile():
     tile = Tile(image=im_np_rgb, coords=(1, 3), masks=masks, labels=labs)
     return tile
 
-
 @pytest.fixture
 def tile():
     """