Merge pull request #40 from HumbleData/upgrade-python--and-deps

Various upgrades (Python version, dependencies, Poetry, pre-commit, and more)

.pre-commit-config.yaml

@@ -0,0 +1,158 @@
+# Update all versions in this file by running:
+#   $ pre-commit autoupdate
+minimum_pre_commit_version: 2.19.0
+default_language_version:
+  python: python3.9
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.2.0
+    hooks:
+      - id: end-of-file-fixer
+        name: Check for a blank line at the end of scripts (auto-fixes)
+        exclude: ^solutions/
+      - id: trailing-whitespace
+      - id: check-builtin-literals
+        exclude: |
+          (?x)^(
+              solutions/01_04.py|
+              solutions/02_21.py|
+              solutions/04_05.py|
+              solutions/05_01.py|
+              solutions/05_12.py
+          )$
+      - id: check-byte-order-marker
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-toml
+      - id: check-vcs-permalinks
+      - id: check-xml
+      - id: debug-statements
+        exclude: |
+          (?x)^(
+              solutions/01_04.py|
+              solutions/02_21.py|
+              solutions/04_05.py|
+              solutions/05_01.py|
+              solutions/05_12.py
+          )$
+      - id: detect-private-key
+      - id: mixed-line-ending
+      - id: fix-encoding-pragma
+        args: ["--remove"]
+      - id: check-yaml
+      - id: check-added-large-files
+        name: Check for files larger than 5 MB
+        args: ["--maxkb=5120"]
+      - id: check-ast
+        exclude: |
+          (?x)^(
+              solutions/01_04.py|
+              solutions/02_21.py|
+              solutions/04_05.py|
+              solutions/05_01.py|
+              solutions/05_12.py
+          )$
+      - id: check-docstring-first
+  - repo: https://github.com/myint/autoflake
+    rev: v1.4
+    hooks:
+      - id: autoflake
+        args: &autoflake
+          - --in-place
+          - --remove-all-unused-imports
+          - --expand-star-imports
+          - --remove-duplicate-keys
+          - --remove-unused-variables
+        exclude: |
+          (?x)^(
+              solutions/02_01.py
+          )$
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+      - &flake8
+        id: flake8
+        additional_dependencies:
+          - flake8-docstrings==1.6.0
+          - flake8-eradicate==1.2.0
+          - flake8-fixme==1.1.1
+          - flake8-implicit-str-concat==0.2.0
+          - flake8-print==4.0.0
+          - flake8-return==1.1.3
+        args: ["--config=setup.cfg"]
+  - repo: local
+    hooks:
+      - id: linestripper-add
+        name: linestripper-add
+        entry: python linestripper.py add
+        language: system
+        always_run: true
+        pass_filenames: false
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+        exclude: |
+          (?x)^(
+              solutions/01_04.py|
+              solutions/01_05.py|
+              solutions/02_21.py|
+              solutions/04_05.py|
+              solutions/05_01.py|
+              solutions/05_12.py
+          )$
+  - repo: local
+    hooks:
+      - id: linestripper-remove
+        name: linestripper-remove
+        entry: python linestripper.py remove
+        language: system
+        always_run: true
+        pass_filenames: false
+  - repo: https://github.com/asottile/add-trailing-comma
+    rev: v2.2.3
+    hooks:
+      - id: add-trailing-comma
+        args: [--py36-plus]
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        types: [python]
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.32.1
+    hooks:
+      - id: pyupgrade
+        args:
+          - "--py39-plus"
+  - repo: https://github.com/Cielquan/mirrors-poetry
+    rev: 1.1.13
+    hooks:
+      - id: poetry-check
+      - id: poetry-lock
+        args: ["--no-update"]
+      - id: poetry-export
+        args: ["--without-hashes"]
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.6.2
+    hooks:
+      - id: prettier
+        types_or: [yaml]
+        additional_dependencies:
+          - "[email protected]"
+  - repo: local
+    hooks:
+      - id: pylint
+        name: pylint
+        entry: pylint solutions
+        language: system
+        always_run: true
+        pass_filenames: false
+        # stages: [manual]
+      - id: pip-audit
+        name: pip-audit
+        entry: pip-audit
+        language: system
+        always_run: true
+        pass_filenames: false

.python-version

@@ -1 +1 @@
-3.8.5
+3.9.13

.vscode/settings.json

@@ -1,27 +1,49 @@
 {
-    "python.linting.enabled": true,
-    "python.linting.flake8Enabled": true,
-    "python.linting.flake8Args": [],
-    "python.linting.pylintArgs": [],
-    "python.linting.pylintUseMinimalCheckers": false,
-    "python.formatting.provider": "black",
-    "python.formatting.blackArgs": [],
-    "editor.formatOnSave": true,
-    "python.sortImports.args": [
-        "--recursive",
-    ],
-    "[python]": {
-        "editor.codeActionsOnSave": {
-            "source.organizeImports": true
+        // editor & autoformatter settings
+        "editor.formatOnSave": true,
+        "editor.trimAutoWhitespace": true,
+        "files.trimTrailingWhitespace": true,
+        "prettier.enable": true,
+
+        // python - black
+        "python.formatting.provider": "black",
+        "python.formatting.blackArgs": [],
+
+        // python - other
+        "python.languageServer": "Pylance",
+
+        // python - linting & static analysis
+        "python.analysis.extraPaths": ["solutions"],
+        "python.analysis.typeCheckingMode": "basic",
+        "python.linting.enabled": true,
+        "python.linting.flake8Enabled": true,
+        "python.linting.flake8Args": [],
+        "python.linting.pylintEnabled": true,
+        "python.linting.pylintArgs": [],
+
+        // Python files only
+        "[python]": {
+            // isort on save
+            "editor.codeActionsOnSave": {"source.organizeImports": true},
+            // Stop the 'Black does not support "Format Selection"' message every
+            // time you paste something (https://stackoverflow.com/a/63612401/3279076)
+            "editor.formatOnPaste": false
+        },
+
+        // python - pytest (https://code.visualstudio.com/docs/python/testing)
+        "python.testing.unittestEnabled": false,
+        "python.testing.pytestEnabled": true,
+        "python.testing.pytestArgs": ["tests"],
+
+        // git
+        "git.enabled": true,
+
+        // file associations
+        "files.associations": {
+            "**/requirements{/**,*}.{txt,in}": "pip-requirements",
         },
-        // Stop the 'Black does not support "Format Selection"' message every
-        // time you paste something (https://stackoverflow.com/a/63612401/3279076)
-        "editor.formatOnPaste": false
-    },
-    "python.pythonPath": "${env:VIRTUAL_ENV}",
-    // from here: https://github.com/microsoft/vscode-python/wiki/AB-Experiments
-    // being selected made things run differently on different peoples systems.
-    "python.experiments.enabled": false,
+
+    "cSpell.language": "en-GB",
     "cSpell.words": [
         "adelie",
         "boxplot",

CHANGELOG.md

@@ -0,0 +1,31 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+<!-- TOWNCRIER -->
+
+## [1.1.0] - 2022-07-10
+
+- Various upgrades ([#40](https://github.com/HumbleData/beginners-data-workshop/pull/40))
+- **Behind-the-scenes**
+  - Upgrade Python to 3.9.13
+  - Upgrade all dependencies to latest with NumPy/pandas/Matplotlib/scikit-learn matching CoCalc versions.
+  - Switch from `pip-tools` to Poetry (configuration in `pyproject.toml`)
+  - Update Development Setup guidance in `README.md`
+  - Integrate & configure flake8, pylint, black & other linters.
+  - Added pre-commit configuration.
+  - Updated VS Code `settings.json`
+  - Added `linestripper.py` to keep EOF newlines out of solutions code (better attendee UX) yet black-compliant.
+  - Added `CHANGELOG.md` and started versioning releases.
+- **Workshop materials**
+  - Reviewed all workshop materials for deprecations.
+  - Replaced all single quotes in solutions with double quotes for black compliance.
+  - Added EOF newlines to datasets.
+  - Other minor changes: trailing whitespace, trailing commas, isort compliant solution code.
+
+
+## [1.0.0] - 2021-07-23
+Final version used for Humble Data workshops in 2021.

README.md

@@ -25,36 +25,39 @@ pip-compile
 pip-sync
 ```
 
-## Setup
+## Attendee Setup
+
+1. Visit [https://bit.ly/humble-python-setup] to install Python onto your machine.
+2. The notebooks in this project should work great with Anaconda out of the box. If you want to create your own virtual
+   environment for this workshop, your mentor will guide you through the simplest setup.
+
+> Note: The Development Setup outlined below will work for you, but is more complex than what you need to follow this
+  workshop. This setup is designed to make it easy to upgrade Python dependencies (e.g. pandas, Seaborn), and run
+  automated checks to highlight if any code in the notebooks is broken (we call this a "test suite").
+
+## Development Setup
+
+This project uses [pyenv](https://github.com/pyenv/pyenv), [pyenv-virtualenvwrapper](https://github.com/pyenv/pyenv-virtualenvwrapper)
+and [Poetry](https://python-poetry.org/docs/). Please [see here](https://github.com/CoefficientSystems/coefficient-cookiecutter/blob/develop/%7B%7Bcookiecutter.repo_name%7D%7D/docs/getting_started.md) for a step-by-step installation guide.
 
 ```bash
-# Install virtualenv
-pip install virtualenv
-
-
-# Install virtualenvwrapper (http://virtualenvwrapper.readthedocs.org/en/latest/index.html)
-pip install virtualenvwrapper
-# Tell shell to source virtualenvwrapper.sh and where to put the virtualenvs by adding following to .zshrc
-zshconfig
-#    # "Tell shell to source virtualenvwrapper.sh and where to put the virtualenvs"
-#    export WORKON_HOME=$HOME/.virtualenvs
-#    export PROJECT_HOME=$HOME/code
-#    source /usr/local/bin/virtualenvwrapper.sh
-source ~/.zshrc
-source /usr/local/bin/virtualenvwrapper.sh
-# Now let's make a virtualenv
-mkvirtualenv venv
-workon venv
-# Commands `workon venv`, `deactivate`, `lsvirtualenv` and `rmvirtualenv` are useful
-# WARNING: When you brew install formulae that provide Python bindings, you should not be in an active virtual environment.
-# (https://github.com/Homebrew/homebrew/blob/master/share/doc/homebrew/Homebrew-and-Python.md)
-deactivate
-
-
-# Create virtualenv & install packages
-mkvirtualenv beginners-data-workshop
-pip install pip-tools
-pip-sync
+# Install Python
+pyenv install $(cat .python-version)
+pyenv shell $(cat .python-version)
+python -m pip install --upgrade pip
+python -m pip install virtualenvwrapper
+pyenv virtualenvwrapper
+
+# Setup the virtualenv
+mkvirtualenv -p python$(cat .python-version) $(cat .venv)
+python -V
+python -m pip install --upgrade pip
+
+# Install dependencies with Poetry
+poetry self update
+poetry install --no-root --remove-untracked
+
+# Create Jupyter Kernel
 python -m ipykernel install --user --name beginners-data-workshop --display-name "Python (beginners-data-workshop)"
 
 # Install Jupyter Extensions static files

data/food_training/languages.csv

@@ -47,4 +47,4 @@ Switzerland,German/French/Italian/Romansch
 Turkey,Turkish
 Ukraine,Ukrainian
 United Kingdom,English
-Vatican City State,Latin/Italian
+Vatican City State,Latin/Italian

data/food_training/training_2014.csv

@@ -27,4 +27,4 @@ RASFF,Madrid,2015-03-23,2015-03-26,1,,
 Contingency Planning,Padua,2015-03-23,2015-03-27,1,,
 Microbiological Criteria,Lisbon,2015-03-23,2015-03-27,1,,
 Food Composition and Information,Athens,2015-03-23,2015-03-27,1,,
-Control on Contaminants in Feed and Food,Berlin,2015-03-24,2015-03-27,3,,
+Control on Contaminants in Feed and Food,Berlin,2015-03-24,2015-03-27,3,,

data/food_training/training_2015.csv

@@ -121,4 +121,4 @@ Plant Health Controls,Venice/Treviso,2016-03-07,2016-03-11,2,,
 Import Controls on Food and Feed of Non-Animal Origin,Genoa;Italy,2016-03-08,2016-03-11,4,,
 Feed Law,Nantes;France,2016-03-14,2016-03-18,3,,
 Animal by Products (Upgraded),Maribor;Slovenia,2016-03-15,2016-03-18,2,,
-EU Feed Hygiene Rules and HACCP auditing ,Barcelona,2016-03-31,2016-06-03,2,,
+EU Feed Hygiene Rules and HACCP auditing ,Barcelona,2016-03-31,2016-06-03,2,,

data/food_training/training_2016.csv

@@ -115,4 +115,4 @@ Contingency Planning,Venice;IT,2017-03-06,2017-03-10,3,,
 Audit B1 - Standard Level,Valencia,2017-03-06,2017-03-10,2,,
 Audit A,Bratislava,2017-03-20,2017-03-24,1,,
 Food Hygiene and Flexibility,Zagreb/Helsinki,2017-03-20,2017-03-24,2,,
-Animal Health and Disease Prevention for Bees and Zoo Animals,Antwerp,2017-03-28,2017-03-31,2,,
+Animal Health and Disease Prevention for Bees and Zoo Animals,Antwerp,2017-03-28,2017-03-31,2,,