Skip to content

Commit

Permalink
Merge branch 'OSGeo:main' into flake8-1522
Browse files Browse the repository at this point in the history
  • Loading branch information
mshukuno authored Apr 2, 2024
2 parents 3df40c5 + c617f0b commit 6100441
Show file tree
Hide file tree
Showing 17 changed files with 708 additions and 233 deletions.
2 changes: 1 addition & 1 deletion .github/actions/create-upload-suggestions/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ runs:
steps.upload-changes.outputs.artifact-url }})
- name: Fail action if some files were changed
if: >-
${{ (steps.files_changed.outputs.files_changed == 'true') &&
${{ (steps.files_changed.outputs.files_changed == 'true') &&
(steps.inputs.outputs.fail-if-changed == 'true') }}
shell: bash
run: |
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/coverity.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
name: Coverity Scan
on:
workflow_dispatch: # run whenever a contributor calls it
schedule:
schedule:
- cron: '48 5 * * *' # Run at 05:48
# Coverity will let GRASS do a scan a maximum of twice per day, so this schedule will help GRASS fit within that limit with some additional space for manual runs

jobs:
build:
runs-on: [ ubuntu-latest ]
Expand Down
46 changes: 43 additions & 3 deletions .github/workflows/post-pr-reviews.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: Post PR code suggestions

on:
workflow_run:
workflows: ["ClangFormat Check"]
workflows: ["ClangFormat Check", "Python Code Quality"]
types:
- completed
permissions: {}
Expand All @@ -27,7 +27,48 @@ jobs:
github-token: ${{ github.token }}
run-id: ${{github.event.workflow_run.id }}
- uses: reviewdog/action-setup@3f401fe1d58fe77e10d665ab713057375e39b887 # v1.3.0
- run: |
- name: Check what tools have suggestions to post
# Using this pattern to have expected file names explicitly named
id: tools
run: |
for tool_name in $INPUT_TOOL_NAMES
do
INPUT_TOOL_NAME_FILE="diff-${tool_name}.patch"
echo "Checking if tool ${tool_name} left suggestions in ${INPUT_TOOL_NAME_FILE}..."
if [[ -f "${INPUT_TOOL_NAME_FILE}" ]]; then
echo " ${INPUT_TOOL_NAME_FILE} was found for tool ${tool_name}"
echo "$tool_name=true" >> "${GITHUB_OUTPUT}"
else
echo " ${INPUT_TOOL_NAME_FILE} was not found for tool ${tool_name}"
echo "$tool_name=false" >> "${GITHUB_OUTPUT}"
fi
done
env:
INPUT_TOOL_NAMES: >-
black
clang-format
- name: Post Black suggestions
if: ${{ steps.tools.outputs.black == 'true' }}
run: |
TMPFILE="diff-${INPUT_TOOL_NAME}.patch"
GITHUB_ACTIONS="" reviewdog \
-name="${INPUT_TOOL_NAME:-reviewdog-suggester}" \
-f=diff \
-f.diff.strip=1 \
-filter-mode=nofilter \
-guess \
-reporter="github-pr-review" < "${TMPFILE}"
env:
INPUT_TOOL_NAME: black
REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CI_COMMIT: ${{ github.event.workflow_run.head_sha }}
CI_REPO_OWNER: ${{ github.event.workflow_run.repository.owner.login }}
CI_REPO_NAME: ${{ github.event.workflow_run.repository.name }}
# CI_PULL_REQUEST: "" # Populated from reviewdog's "-guess" flag since hard to get
- name: Post ClangFormat suggestions
if: ${{ steps.tools.outputs.clang-format == 'true' }}
run: |
TMPFILE="diff-${INPUT_TOOL_NAME}.patch"
GITHUB_ACTIONS="" reviewdog \
-name="${INPUT_TOOL_NAME:-reviewdog-suggester}" \
-f=diff \
Expand All @@ -36,7 +77,6 @@ jobs:
-guess \
-reporter="github-pr-review" < "${TMPFILE}"
env:
TMPFILE: diff-clang-format.patch
INPUT_TOOL_NAME: clang-format
REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CI_COMMIT: ${{ github.event.workflow_run.head_sha }}
Expand Down
23 changes: 17 additions & 6 deletions .github/workflows/python-code-quality.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,23 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
cache: pip
- name: Upgrade pip
run: python -m pip install --upgrade pip

- name: Install Black only
run: pip install black==${{ matrix.black-version }}

- name: Run Black
run: black .

- name: Create and uploads code suggestions to apply for Black
# Will fail fast here if there are changes required
id: diff-black
uses: ./.github/actions/create-upload-suggestions
with:
tool-name: black
# To keep repo's file structure in formatted changes artifact
extra-upload-changes: .clang-format

- name: Install non-Python dependencies
run: |
Expand All @@ -66,18 +83,12 @@ jobs:
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install -r .github/workflows/python_requirements.txt
pip install -r .github/workflows/optional_requirements.txt
pip install black==${{ matrix.black-version }}
pip install flake8==${{ matrix.flake8-version }}
pip install pylint==${{ matrix.pylint-version }} pytest-github-actions-annotate-failures
pip install bandit[sarif]==${{matrix.bandit-version}}
- name: Run Black
run: |
black --check --diff .
- name: Run Flake8
run: |
flake8 --count --statistics --show-source --jobs=$(nproc) .
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ubuntu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ jobs:

- name: Add extra exclusions to a gunittest config file
run: |
sed 's:exclude =:exclude = ${{
sed 's:exclude =:exclude = ${{
steps.get-exclude.outputs.extra-exclude
}}:g' .gunittest.cfg > .gunittest.extra.cfg
cat .gunittest.extra.cfg
Expand Down
8 changes: 8 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ config.log
config.status*
error.log
grass.pc
autom4te.cache/

# ignore specific file generated during make
gui/wxpython/menustrings.py
Expand Down Expand Up @@ -50,3 +51,10 @@ test_keyvalue_result.txt
# ignore paths generated by helper tools
node_modules
include/VERSION_GIT

# ignore autogenerated documentation
python/grass/docs/_build/
html/
latex/
lib/*/html/
lib/*/latex/
26 changes: 8 additions & 18 deletions docker/alpine/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM python:3-alpine3.19@sha256:c7eb5c92b7933fe52f224a91a1ced27b91840ac9c69c58bef40d602156bcdb41 as common
FROM alpine:3.19@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b as common

# Based on:
# https://github.com/mundialis/docker-grass-gis/blob/master/Dockerfile
Expand Down Expand Up @@ -56,6 +56,7 @@ ENV GRASS_RUN_PACKAGES="\
openblas \
py3-numpy \
py3-pillow \
python3 \
pdal \
pdal-dev \
postgresql15-client \
Expand All @@ -73,18 +74,8 @@ ENV GRASS_RUN_PACKAGES="\

WORKDIR /src

ENV PYTHONBIN=python$PYTHON_VERSION

RUN echo "Install Python";\
apk add --no-cache $PYTHONBIN && \
$PYTHONBIN -m ensurepip && \
rm -r /usr/lib/python*/ensurepip && \
pip$PYTHON_VERSION install --no-cache-dir --upgrade pip setuptools && \
if [ ! -e /usr/bin/pip ]; then ln -s pip$PYTHON_VERSION /usr/bin/pip ; fi && \
if [ ! -e /usr/bin/python ]; then ln -sf /usr/bin/$PYTHONBIN /usr/bin/python; fi && \
rm -r /root/.cache; \
# Add the packages
echo "Install main packages";\
# Add the packages
RUN echo "Install main packages";\
apk update; \
apk add --no-cache $GRASS_RUN_PACKAGES

Expand Down Expand Up @@ -215,15 +206,14 @@ COPY --from=build /usr/local/grass* /usr/local/grass/
COPY --from=build /usr/lib/gdalplugins/*_GRASS.so /usr/lib/gdalplugins/
# run simple LAZ test
COPY docker/testdata/simple.laz /tmp/
COPY docker/testdata/test_grass_session.py docker/alpine/grass_tests.sh /scripts/
COPY docker/testdata/test_grass_session.py /scripts/
COPY docker/testdata/test_grass_python.py docker/alpine/grass_tests.sh /scripts/
COPY docker/testdata/test_grass_python.py /scripts/

# install external Python API
RUN pip3 install --no-cache-dir --upgrade pip six grass-session --ignore-installed six; \
ln -sf /usr/local/grass $(grass --config path); \
RUN ln -sf /usr/local/grass $(grass --config path); \
# run some tests and cleanup
$SHELL /scripts/grass_tests.sh \
&& rm -f /scripts/grass_tests.sh /tmp/simple.laz /scripts/test_grass_session.py; \
&& rm -f /scripts/grass_tests.sh /tmp/simple.laz /scripts/test_grass_python.py; \
# delete unused packages
apk del --no-cache gettext pdal-dev; \
# show installed version
Expand Down
174 changes: 174 additions & 0 deletions imagery/i.pca/testsuite/test_pca.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
"""
Name: i.pca test
Purpose: Tests i.pca.
Author: Hamed Elgizery - [email protected]
Copyright: (C) 2024 by Hamed Elgizery and the GRASS Development Team
Licence: This program is free software under the GNU General Public
License (>=v2). Read the file COPYING that comes with GRASS
for details.
"""
from grass.gunittest.case import TestCase


class TestReport(TestCase):
@classmethod
def setUpClass(cls):
"""Use temporary region settings"""
cls.runModule("g.region", raster="lsat7_2002_10@PERMANENT")
cls.use_temp_region()

@classmethod
def tearDownClass(cls):
cls.runModule(
"g.remove",
flags="f",
type="raster",
name="lsat7_2002_pca.1,lsat7_2002_pca.2,lsat7_2002_pca.3,lsat7_2002_pca.4,lsat7_2002_pca.6",
)
cls.del_temp_region()

def test_pca_sample(self):
"""Testing pca sample"""
self.assertModule(
"i.pca",
input="lsat7_2002_10,lsat7_2002_20,lsat7_2002_30,lsat7_2002_40,lsat7_2002_50,lsat7_2002_70",
output="lsat7_2002_pca",
)

lsat7_2002_pca_info_out = """north=228513
south=214975.5
east=645012
west=629992.5
nsres=28.5
ewres=28.5
rows=475
cols=527
cells=250325
datatype=CELL
ncats=0
comments=\"Eigen values, (vectors), and [percent importance]:PC1 4334.35 ( 0.2824, 0.3342, 0.5092,-0.0087, 0.5264, 0.5217) [83.04%]PC2 588.31 ( 0.2541, 0.1885, 0.2923,-0.7428,-0.5110,-0.0403) [11.27%]PC3 239.22 ( 0.3801, 0.3819, 0.2681, 0.6238,-0.4000,-0.2980) [ 4.58%]PC4 32.85 ( 0.1752,-0.0191,-0.4053, 0.1593,-0.4435, 0.7632) [ 0.63%]PC5 20.73 (-0.6170,-0.2514, 0.6059, 0.1734,-0.3235, 0.2330) [ 0.40%]PC6 4.08 (-0.5475, 0.8021,-0.2282,-0.0607,-0.0208, 0.0252) [ 0.08%]i.pca input=\"lsat7_2002_10,lsat7_2002_20,lsat7_2002_30,lsat7_2002_40\,lsat7_2002_50,lsat7_2002_70\" output=\"lsat7_2002_pca\" rescale=0,255 \percent=99" """

lsat7_2002_pca_univar_out = [
"""n=250325
null_cells=0
cells=250325
min=0
max=255
range=255
mean=60.6958074503146
mean_of_abs=60.6958074503146
stddev=32.8850846003739
variance=1081.42878917375
coeff_var=54.1801583697417
sum=15193678
first_quartile=36
median=51
third_quartile=77
percentile_90=101""",
"""n=250325
null_cells=0
cells=250325
min=0
max=255
range=255
mean=106.099418755618
mean_of_abs=106.099418755618
stddev=26.4487056926998
variance=699.534032819051
coeff_var=24.928228639612
sum=26559337
first_quartile=88
median=104
third_quartile=121
percentile_90=137""",
"""n=250325
null_cells=0
cells=250325
min=0
max=255
range=255
mean=74.1768980325577
mean_of_abs=74.1768980325577
stddev=14.1956266450161
variance=201.515815844691
coeff_var=19.1375307158104
sum=18568332
first_quartile=67
median=74
third_quartile=81
percentile_90=88""",
"""n=250325
null_cells=0
cells=250325
min=0
max=255
range=255
mean=113.285145311096
mean_of_abs=113.285145311096
stddev=10.689092045444
variance=114.256688755974
coeff_var=9.43556369733241
sum=28358104
first_quartile=109
median=114
third_quartile=118
percentile_90=122""",
"""n=250325
null_cells=0
cells=250325
min=0
max=255
range=255
mean=110.346713272745
mean_of_abs=110.346713272745
stddev=8.43087149474902
variance=71.0795941609716
coeff_var=7.64034672596938
sum=27622541
first_quartile=106
median=110
third_quartile=114
percentile_90=118""",
"""n=250325
null_cells=0
cells=250325
min=0
max=255
range=255
mean=115.238465994208
mean_of_abs=115.238465994208
stddev=8.97064489504434
variance=80.4724698329851
coeff_var=7.78441887233665
sum=28847069
first_quartile=110
median=115
third_quartile=121
percentile_90=126""",
]

for i in range(1, 7):
# Asserting the results givien from r.info
self.assertRasterFitsInfo(
raster=f"lsat7_2002_pca.{i}",
reference=lsat7_2002_pca_info_out,
precision=3,
)

# Asserting the results givien from r.univar
univar_out = lsat7_2002_pca_univar_out[i - 1]
self.assertModuleKeyValue(
"r.univar",
flags="eg",
map=f"lsat7_2002_pca.{i}",
reference=univar_out,
precision=3,
sep="=",
)


if __name__ == "__main__":
from grass.gunittest.main import test

test()
Loading

0 comments on commit 6100441

Please sign in to comment.