Switch to poetry, split up make install-deps

Qubes 4.2's dom0 ships with Python 3.11, which is much easier to deal
with than 4.1's 3.8, since it matches with Debian bookworm. As a result,
we can expect most developers to be able to natively use 3.11 and not
need a container wrapper.

It's not straightforward to just swap out pip for poetry here since
poetry sets up a virtualenv per-user instead of globally installing.

At the same time, the `make install-deps` target has grown so that it
encompases both build and test dependencies and is ripe for splitting.
It also had some duplication in that build dependencies were specified
in both the Makefile and specfile.

So we can address everything at once. There are now three effective sets
of dependency groupings:

1) Python lint/test dependencies, specified in pyproject.toml and
installed using poetry.
2) Build dependencies, specified in the `.spec` file and installed via
`make build-deps`.
3) Test dependencies, specified in the Makefile and installed via `make
test-deps`.

The corresponding make targets, container.sh and CI manifests have been
updated to handle this. The `install-deps` target was removed so it
isn't accidentally used anymore.

The launcher dependencies were identical except that virtualenv also
needed PyQt5. We already had a root `make test-launcher`, so fix that up
and get rid of separate launcher dependencies and Makefile.

Fixes #992.
Fixes #947.

.github/dependabot.yml

@@ -1,5 +1,16 @@
 version: 2
 updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    allow:
+      - dependency-type: "all"
+    ignore:
+      - dependency-name: "pyqt*"
+    groups:
+        dependencies:
+          patterns: ["*"]
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/workflows/ci.yml

@@ -13,8 +13,9 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          make install-deps
-          pip install --require-hashes -r requirements/dev-requirements.txt
+          make test-deps
+          pip install poetry==1.8.2
+          poetry install --no-ansi
       - name: Run linters
         run: |
           git config --global --add safe.directory '*'
@@ -28,14 +29,15 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          make install-deps
-          pip install --require-hashes -r requirements/dev-requirements.txt
+          make build-deps
       - name: Build RPM
         run: |
           git config --global --add safe.directory '*'
           make build-rpm
       - name: Check reproducibility
-        run: make reprotest
+        run: |
+          make test-deps
+          make reprotest
   launcher-tests:
     runs-on: ubuntu-latest
     container:
@@ -45,9 +47,9 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          make install-deps
-          pip install --require-hashes -r launcher/dev-requirements.txt
+          make test-deps
+          pip install poetry==1.8.2
+          poetry install --no-ansi
       - name: Run launcher tests
         run: |
-          cd launcher/
-          make check
+          make test-launcher

Makefile

@@ -38,14 +38,20 @@ build-rpm: ## Build RPM package
 reprotest: ## Check RPM package reproducibility
 	TERM=xterm-256color $(CONTAINER) bash -c "sudo ln -s $$PWD/scripts/fake-setarch.py /usr/local/bin/setarch && sudo reprotest 'make build-rpm' 'rpm-build/RPMS/noarch/*.rpm' --variations '+all,+kernel,-time,-fileordering,-domain_host'"
 
-# Installs Fedora 37 package dependencies, to build RPMs and run tests,
-# primarily useful in CI/containers
-.PHONY: install-deps
-install-deps:
-	sudo dnf install -y \
-        git file python3-devel python3-pip python3-qt5 python3-wheel \
-		xorg-x11-server-Xvfb rpmdevtools rpmlint systemd-rpm-macros which libfaketime ShellCheck \
+
+.PHONY: build-deps
+build-deps: ## Install package dependencies to build RPMs
+# Note: build dependencies are specified in the spec file, not here
+	dnf install -y \
+		git file rpmdevtools dnf-plugins-core
+	dnf builddep -y rpm-build/SPECS/securedrop-workstation-dom0-config.spec
+
+.PHONY: test-deps
+test-deps: build-deps ## Install package dependencies for running tests
+	dnf install -y \
+		python3-qt5 xorg-x11-server-Xvfb rpmlint which libfaketime ShellCheck \
 		hostname
+	dnf --setopt=install_weak_deps=False -y install reprotest
 
 clone: assert-dom0 ## Builds rpm && pulls the latest repo from work VM to dom0
 	@./scripts/clone-to-dom0
@@ -173,40 +179,39 @@ venv: ## Provision a Python 3 virtualenv for development (ensure to also install
 check: lint test ## Runs linters and tests
 
 .PHONY: lint
-lint: check-black check-isort flake8 bandit mypy rpmlint shellcheck ## Runs linters (black, isort, flake8, bandit rpmlint, and shellcheck)
 lint: check-black check-isort flake8 mypy bandit rpmlint shellcheck ## Runs linters (black, isort, flake8, mypy, bandit rpmlint, and shellcheck)
 
 .PHONY: bandit
 bandit: ## Runs the bandit security linter
-	bandit -ll --exclude ./.venv,./launcher/.venv -r .
+	poetry run bandit -ll -r .
 
 .PHONY: test-launcher
-test-launcher: ## Runs tests
-	$(CONTAINER) python3 -m pytest -v
+test-launcher: ## Runs launcher tests
+	xvfb-run poetry run python3 -m pytest --cov-report term-missing --cov=sdw_notify --cov=sdw_updater/ --cov=sdw_util -v launcher/tests/
 
 .PHONY: check-black
 check-black: ## Check Python source code formatting with black
-	black --check --diff .
+	poetry run black --check --diff .
 
 .PHONY: black
 black: ## Update Python source code formatting with black
-	black .
+	poetry run black .
 
 .PHONY: check-isort
 check-isort: ## Check Python import organization with isort
-	isort --check-only --diff .
+	poetry run isort --check-only --diff .
 
 .PHONY: isort
 isort: ## Update Python import organization with isort
-	isort .
+	poetry run isort .
 
 .PHONY: flake8
 flake8: ## Validate PEP8 compliance for Python source files
-	flake8
+	poetry run flake8
 
 .PHONY: mypy
 mypy:  ## Type check Python files
-	mypy .
+	poetry run mypy .
 
 .PHONY: rpmlint
 rpmlint: ## Runs rpmlint on the spec file

bootstrap/Dockerfile

@@ -10,10 +10,12 @@ ENV USER_ID ${USER_ID:-0}
 RUN dnf install -y make
 
 COPY Makefile Makefile
+COPY rpm-build/SPECS rpm-build/SPECS
 
-RUN make install-deps
+ARG DEPS=build-deps
+RUN make ${DEPS}
 
-COPY requirements requirements
-RUN pip3 install --no-deps --require-hashes -r requirements/dev-requirements.txt
+# Cleanup
+RUN rm -rf rpm-build
 
 RUN if test $USER_NAME != root ; then useradd --no-create-home --home-dir /tmp --uid $USER_ID $USER_NAME && echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ; fi