Merge pull request #8 from freedomofpress/7-containerize-build-test

Containerize build and test environment, add some developer tools

(cherry picked from commit freedomofpress/securedrop-updater@9b2d169)

.git-blame-ignore-revs

@@ -1 +1,2 @@
 782a04e59ae10d40085b5d8807e255143e5d05c5
+0d4c0144d01c24187c449593ca665e2f0141b507

Makefile

@@ -149,13 +149,44 @@ validate: assert-dom0 ## Checks for local requirements in dev env
 # Not requiring dom0 for linting as that requires extra packages, which we're
 # not installing on dom0, so are only in the developer environment, i.e. Work VM
 
+prep-dom0: prep-dev # Copies dom0 config files
+	sudo qubesctl --show-output --targets dom0 state.highstate
+
+destroy-all: ## Destroys all VMs managed by Workstation salt config
+	./scripts/destroy-vm --all
+
+.PHONY: update-pip-requirements
+update-pip-requirements: ## Updates all Python requirements files via pip-compile.
+	pip-compile --allow-unsafe --generate-hashes --output-file=requirements/dev-requirements.txt requirements/dev-requirements.in
+
+.PHONY: venv
+venv: ## Provision a Python 3 virtualenv for development (ensure to also install OS package for PyQt5)
+	$(PYTHON3) -m venv .venv --system-site-packages
+	.venv/bin/pip install --upgrade pip wheel
+	.venv/bin/pip install --require-hashes -r "requirements/dev-requirements.txt"
+	@echo "#################"
+	@echo "Virtualenv with system-packages is complete."
+	@echo "Make sure to either install the OS package for PyQt5 or install PyQt5==5.14.2 into this virtual environment."
+	@echo "Then run: source .venv/bin/activate"
+
+.PHONY: check
+check: lint test ## Runs linters and tests
+
+.PHONY: lint
+lint: check-black check-isort flake8 bandit mypy rpmlint shellcheck ## Runs linters (black, isort, flake8, bandit rpmlint, and shellcheck)
+
+.PHONY: bandit
+bandit: ## Runs the bandit security linter
+	bandit -ll --exclude ./.venv,./launcher/.venv -r .
+
+.PHONY: test-launcher
+test-launcher: ## Runs tests with the X Virtual framebuffer
+	$(CONTAINER) xvfb-run python3 -m pytest --cov-report term-missing --cov=sdw_notify --cov=sdw_updater --cov=sdw_util -v tests/
+
 .PHONY: check-black
 check-black: ## Check Python source code formatting with black
 	black --check --diff .
 
-.PHONY: lint
-lint: flake8 black mypy ## Runs all linters
-
 .PHONY: black
 black: ## Update Python source code formatting with black
 	black .
@@ -169,9 +200,7 @@ isort: ## Update Python import organization with isort
 	isort --diff .
 
 .PHONY: flake8
-flake8: ## Lints all Python files with flake8
-# Not requiring dom0 since linting requires extra packages,
-# available only in the developer environment, i.e. Work VM.
+flake8: ## Validate PEP8 compliance for Python source files
 	flake8
 
 mypy: ## Type checks Python files
@@ -187,25 +216,6 @@ rpmlint: ## Runs rpmlint on the spec file
 shellcheck: ## Runs shellcheck on all shell scripts
 	./scripts/shellcheck.sh
 
-prep-dom0: prep-dev # Copies dom0 config files
-	sudo qubesctl --show-output --targets dom0 state.highstate
-
-destroy-all: ## Destroys all VMs managed by Workstation salt config
-	./scripts/destroy-vm --all
-
-.PHONY: update-pip-requirements
-update-pip-requirements: ## Updates all Python requirements files via pip-compile.
-	pip-compile --allow-unsafe --generate-hashes --output-file=requirements/dev-requirements.txt requirements/dev-requirements.in
-
-.PHONY: venv
-venv: ## Provision a Python 3 virtualenv for development (ensure to also install OS package for PyQt5)
-	$(PYTHON3) -m venv .venv
-	.venv/bin/pip install --upgrade pip wheel
-	.venv/bin/pip install --require-hashes -r "requirements/dev-requirements.txt"
-	@echo "#################"
-	@echo "Virtualenv is complete."
-	@echo "Run: source .venv/bin/activate"
-
 # Explanation of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" to parse lines for make targets.
 # 2. Check for second field matching, skip otherwise.

bootstrap/Dockerfile

@@ -13,4 +13,7 @@ COPY Makefile Makefile
 
 RUN make install-deps
 
+COPY requirements requirements
+RUN pip3 install --no-deps --require-hashes -r requirements/dev-requirements.txt
+
 RUN if test $USER_NAME != root ; then useradd --no-create-home --home-dir /tmp --uid $USER_ID $USER_NAME && echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ; fi

files/sdw-login

@@ -3,9 +3,9 @@
 Utility script for SecureDrop Workstation. Launches the SecureDrop Workstation
 updater on boot. It will prompt users to apply template and dom0 updates
 """
+import logging
 import os
 import subprocess
-import logging
 import time
 
 SCRIPT_NAME = os.path.basename(__file__)

files/sdw-notify

@@ -5,12 +5,13 @@ for too long without checking for security updates. Writes output to a logfile,
 not stdout. All settings are in Notify utility module.
 """
 import sys
+
+from PyQt5.QtWidgets import QApplication
+
 from sdw_notify import Notify, NotifyApp
 from sdw_updater import Updater, UpdaterApp
 from sdw_util import Util
 
-from PyQt5.QtWidgets import QApplication
-
 Util.configure_logging(Notify.LOG_FILE)
 log = Util.get_logger(Notify.LOG_FILE)
 

files/sdw-updater

@@ -1,14 +1,13 @@
 #!/usr/bin/env python3
-from sdw_updater.UpdaterApp import UpdaterApp
-from sdw_util import Util
-from sdw_updater import Updater
-from sdw_updater.UpdaterApp import launch_securedrop_client
-from sdw_updater.Updater import should_launch_updater
-import sys
 import argparse
+import sys
 
 from PyQt5.QtWidgets import QApplication
 
+from sdw_updater import Updater
+from sdw_updater.Updater import should_launch_updater
+from sdw_updater.UpdaterApp import UpdaterApp, launch_securedrop_client
+from sdw_util import Util
 
 DEFAULT_INTERVAL = 28800  # 8hr default for update interval
 

launcher/Makefile

@@ -9,7 +9,7 @@ venv: ## Provision a Python 3 virtualenv for development (run apt-get install py
 	.venv/bin/pip install --require-hashes -r "dev-requirements.txt"
 	@echo "#################"
 	@echo "Virtualenv with Debian system-packages is complete."
-	@echo "Make sure to install the apt packages for system Qt."
+	@echo "Make sure to either install the OS package for PyQt5 or install PyQt5==5.14.2 into this virtual environment."
 	@echo "Then run: source .venv/bin/activate"
 
 .PHONY: check

launcher/dev-requirements.txt

@@ -70,6 +70,14 @@ coverage[toml]==7.4.4 \
     --hash=sha256:fdfafb32984684eb03c2d83e1e51f64f0906b11e64482df3c5db936ce3839d48 \
     --hash=sha256:ff7687ca3d7028d8a5f0ebae95a6e4827c5616b31a4ee1192bdfde697db110d4
     # via pytest-cov
+exceptiongroup==1.2.0 \
+    --hash=sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14 \
+    --hash=sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68
+    # via pytest
+importlib-metadata==7.1.0 \
+    --hash=sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570 \
+    --hash=sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2
+    # via build
 iniconfig==2.0.0 \
     --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \
     --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374
@@ -181,10 +189,23 @@ stevedore==5.2.0 \
     --hash=sha256:1c15d95766ca0569cad14cb6272d4d31dae66b011a929d7c18219c176ea1b5c9 \
     --hash=sha256:46b93ca40e1114cea93d738a6c1e365396981bb6bb78c27045b7587c9473544d
     # via bandit
+tomli==2.0.1 \
+    --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
+    --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
+    # via
+    #   build
+    #   coverage
+    #   pip-tools
+    #   pyproject-hooks
+    #   pytest
 wheel==0.43.0 \
     --hash=sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85 \
     --hash=sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81
     # via pip-tools
+zipp==3.18.1 \
+    --hash=sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b \
+    --hash=sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 pip==24.0 \
@@ -193,7 +214,7 @@ pip==24.0 \
     # via
     #   -r dev-requirements.in
     #   pip-tools
-setuptools==69.2.0 \
-    --hash=sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e \
-    --hash=sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c
+setuptools==69.5.1 \
+    --hash=sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 \
+    --hash=sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32
     # via pip-tools

launcher/tests/test_notify.py

@@ -1,11 +1,11 @@
 import datetime
 import os
-import pytest
 import re
-
-from unittest import mock
 from importlib.machinery import SourceFileLoader
 from tempfile import TemporaryDirectory
+from unittest import mock
+
+import pytest
 
 relpath_notify = "../sdw_notify/Notify.py"
 path_to_notify = os.path.join(os.path.dirname(os.path.abspath(__file__)), relpath_notify)

launcher/tests/test_updater.py

@@ -1,13 +1,14 @@
 import json
 import os
-import pytest
 import subprocess
-from importlib.machinery import SourceFileLoader
 from datetime import datetime, timedelta
+from importlib.machinery import SourceFileLoader
 from tempfile import TemporaryDirectory
 from unittest import mock
 from unittest.mock import call
 
+import pytest
+
 relpath_updater_script = "../sdw_updater/Updater.py"
 path_to_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), relpath_updater_script)
 updater = SourceFileLoader("Updater", path_to_script).load_module()

launcher/tests/test_updaterapp.py

@@ -1,8 +1,7 @@
 import os
 import unittest
-from unittest import TestCase
-from unittest import mock
 from importlib.machinery import SourceFileLoader
+from unittest import TestCase, mock
 
 from PyQt5.QtWidgets import QApplication
 

launcher/tests/test_util.py

@@ -1,11 +1,11 @@
 import os
-import pytest
 import re
 import subprocess
-
-from unittest import mock
 from importlib.machinery import SourceFileLoader
 from tempfile import TemporaryDirectory
+from unittest import mock
+
+import pytest
 
 # Regex for lock conflicts
 BUSY_LOCK_REGEX = r"Error obtaining lock on '.*'."

requirements/dev-requirements.in

@@ -8,4 +8,5 @@ mypy
 pip-tools
 pip
 pytest
+pytest-cov
 reprotest

requirements/dev-requirements.txt

@@ -46,6 +46,60 @@ click==8.0.3 \
     # via
     #   black
     #   pip-tools
+coverage[toml]==7.4.4 \
+    --hash=sha256:00838a35b882694afda09f85e469c96367daa3f3f2b097d846a7216993d37f4c \
+    --hash=sha256:0513b9508b93da4e1716744ef6ebc507aff016ba115ffe8ecff744d1322a7b63 \
+    --hash=sha256:09c3255458533cb76ef55da8cc49ffab9e33f083739c8bd4f58e79fecfe288f7 \
+    --hash=sha256:09ef9199ed6653989ebbcaacc9b62b514bb63ea2f90256e71fea3ed74bd8ff6f \
+    --hash=sha256:09fa497a8ab37784fbb20ab699c246053ac294d13fc7eb40ec007a5043ec91f8 \
+    --hash=sha256:0f9f50e7ef2a71e2fae92774c99170eb8304e3fdf9c8c3c7ae9bab3e7229c5cf \
+    --hash=sha256:137eb07173141545e07403cca94ab625cc1cc6bc4c1e97b6e3846270e7e1fea0 \
+    --hash=sha256:1f384c3cc76aeedce208643697fb3e8437604b512255de6d18dae3f27655a384 \
+    --hash=sha256:201bef2eea65e0e9c56343115ba3814e896afe6d36ffd37bab783261db430f76 \
+    --hash=sha256:38dd60d7bf242c4ed5b38e094baf6401faa114fc09e9e6632374388a404f98e7 \
+    --hash=sha256:3b799445b9f7ee8bf299cfaed6f5b226c0037b74886a4e11515e569b36fe310d \
+    --hash=sha256:3ea79bb50e805cd6ac058dfa3b5c8f6c040cb87fe83de10845857f5535d1db70 \
+    --hash=sha256:40209e141059b9370a2657c9b15607815359ab3ef9918f0196b6fccce8d3230f \
+    --hash=sha256:41c9c5f3de16b903b610d09650e5e27adbfa7f500302718c9ffd1c12cf9d6818 \
+    --hash=sha256:54eb8d1bf7cacfbf2a3186019bcf01d11c666bd495ed18717162f7eb1e9dd00b \
+    --hash=sha256:598825b51b81c808cb6f078dcb972f96af96b078faa47af7dfcdf282835baa8d \
+    --hash=sha256:5fc1de20b2d4a061b3df27ab9b7c7111e9a710f10dc2b84d33a4ab25065994ec \
+    --hash=sha256:623512f8ba53c422fcfb2ce68362c97945095b864cda94a92edbaf5994201083 \
+    --hash=sha256:690db6517f09336559dc0b5f55342df62370a48f5469fabf502db2c6d1cffcd2 \
+    --hash=sha256:69eb372f7e2ece89f14751fbcbe470295d73ed41ecd37ca36ed2eb47512a6ab9 \
+    --hash=sha256:73bfb9c09951125d06ee473bed216e2c3742f530fc5acc1383883125de76d9cd \
+    --hash=sha256:742a76a12aa45b44d236815d282b03cfb1de3b4323f3e4ec933acfae08e54ade \
+    --hash=sha256:7c95949560050d04d46b919301826525597f07b33beba6187d04fa64d47ac82e \
+    --hash=sha256:8130a2aa2acb8788e0b56938786c33c7c98562697bf9f4c7d6e8e5e3a0501e4a \
+    --hash=sha256:8a2b2b78c78293782fd3767d53e6474582f62443d0504b1554370bde86cc8227 \
+    --hash=sha256:8ce1415194b4a6bd0cdcc3a1dfbf58b63f910dcb7330fe15bdff542c56949f87 \
+    --hash=sha256:9ca28a302acb19b6af89e90f33ee3e1906961f94b54ea37de6737b7ca9d8827c \
+    --hash=sha256:a4cdc86d54b5da0df6d3d3a2f0b710949286094c3a6700c21e9015932b81447e \
+    --hash=sha256:aa5b1c1bfc28384f1f53b69a023d789f72b2e0ab1b3787aae16992a7ca21056c \
+    --hash=sha256:aadacf9a2f407a4688d700e4ebab33a7e2e408f2ca04dbf4aef17585389eff3e \
+    --hash=sha256:ae71e7ddb7a413dd60052e90528f2f65270aad4b509563af6d03d53e979feafd \
+    --hash=sha256:b14706df8b2de49869ae03a5ccbc211f4041750cd4a66f698df89d44f4bd30ec \
+    --hash=sha256:b1a93009cb80730c9bca5d6d4665494b725b6e8e157c1cb7f2db5b4b122ea562 \
+    --hash=sha256:b2991665420a803495e0b90a79233c1433d6ed77ef282e8e152a324bbbc5e0c8 \
+    --hash=sha256:b2c5edc4ac10a7ef6605a966c58929ec6c1bd0917fb8c15cb3363f65aa40e677 \
+    --hash=sha256:b4d33f418f46362995f1e9d4f3a35a1b6322cb959c31d88ae56b0298e1c22357 \
+    --hash=sha256:b91cbc4b195444e7e258ba27ac33769c41b94967919f10037e6355e998af255c \
+    --hash=sha256:c74880fc64d4958159fbd537a091d2a585448a8f8508bf248d72112723974cbd \
+    --hash=sha256:c901df83d097649e257e803be22592aedfd5182f07b3cc87d640bbb9afd50f49 \
+    --hash=sha256:cac99918c7bba15302a2d81f0312c08054a3359eaa1929c7e4b26ebe41e9b286 \
+    --hash=sha256:cc4f1358cb0c78edef3ed237ef2c86056206bb8d9140e73b6b89fbcfcbdd40e1 \
+    --hash=sha256:ccd341521be3d1b3daeb41960ae94a5e87abe2f46f17224ba5d6f2b8398016cf \
+    --hash=sha256:ce4b94265ca988c3f8e479e741693d143026632672e3ff924f25fab50518dd51 \
+    --hash=sha256:cf271892d13e43bc2b51e6908ec9a6a5094a4df1d8af0bfc360088ee6c684409 \
+    --hash=sha256:d5ae728ff3b5401cc320d792866987e7e7e880e6ebd24433b70a33b643bb0384 \
+    --hash=sha256:d71eec7d83298f1af3326ce0ff1d0ea83c7cb98f72b577097f9083b20bdaf05e \
+    --hash=sha256:d898fe162d26929b5960e4e138651f7427048e72c853607f2b200909794ed978 \
+    --hash=sha256:d89d7b2974cae412400e88f35d86af72208e1ede1a541954af5d944a8ba46c57 \
+    --hash=sha256:dfa8fe35a0bb90382837b238fff375de15f0dcdb9ae68ff85f7a63649c98527e \
+    --hash=sha256:e0be5efd5127542ef31f165de269f77560d6cdef525fffa446de6f7e9186cfb2 \
+    --hash=sha256:fdfafb32984684eb03c2d83e1e51f64f0906b11e64482df3c5db936ce3839d48 \
+    --hash=sha256:ff7687ca3d7028d8a5f0ebae95a6e4827c5616b31a4ee1192bdfde697db110d4
+    # via pytest-cov
 diffoscope==256 \
     --hash=sha256:9937e3d70358d5cc00aa5fa8a3db9dd6be532a0d3cd2d064add2cda736a97c2a
     # via -r requirements/dev-requirements.in
@@ -93,12 +147,6 @@ mdurl==0.1.2 \
     --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \
     --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba
     # via markdown-it-py
-mypy-extensions==1.0.0 \
-    --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
-    --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
-    # via
-    #   black
-    #   mypy
 mypy==1.9.0 \
     --hash=sha256:0235391f1c6f6ce487b23b9dbd1327b4ec33bb93934aa986efe8a9563d9349e6 \
     --hash=sha256:190da1ee69b427d7efa8aa0d5e5ccd67a4fb04038c380237a0d96829cb157913 \
@@ -128,6 +176,12 @@ mypy==1.9.0 \
     --hash=sha256:f8a67616990062232ee4c3952f41c779afac41405806042a8126fe96e098419f \
     --hash=sha256:fe28657de3bfec596bbeef01cb219833ad9d38dd5393fc649f4b366840baefe6
     # via -r requirements/dev-requirements.in
+mypy-extensions==1.0.0 \
+    --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
+    --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
+    # via
+    #   black
+    #   mypy
 packaging==23.0 \
     --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \
     --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97
@@ -174,6 +228,12 @@ pyproject-hooks==1.0.0 \
 pytest==7.2.1 \
     --hash=sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5 \
     --hash=sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42
+    # via
+    #   -r requirements/dev-requirements.in
+    #   pytest-cov
+pytest-cov==5.0.0 \
+    --hash=sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652 \
+    --hash=sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857
     # via -r requirements/dev-requirements.in
 python-magic==0.4.27 \
     --hash=sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b \
@@ -257,6 +317,7 @@ tomli==1.2.3 \
     # via
     #   black
     #   build
+    #   coverage
     #   mypy
     #   pyproject-hooks
     #   pytest