From 84d205997745573185970b44b944c00fd1018182 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Wed, 16 Dec 2020 18:57:54 +0100 Subject: [PATCH 1/6] nix: Clean up PHP definitions I added `packageOverrides` argument to `php` package in nixpkgs, allowing us to simplify the php definitions a lot. --- flake.lock | 6 +- utils/nix/phps.nix | 183 +++++++++++++++++++++++---------------------- 2 files changed, 96 insertions(+), 93 deletions(-) diff --git a/flake.lock b/flake.lock index 33de87462a..68bf3b2a78 100644 --- a/flake.lock +++ b/flake.lock @@ -18,11 +18,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1607690238, - "narHash": "sha256-9QFXxj6pjmHr+950E3/gXo9cz50l0AbFCHZR5eixkXw=", + "lastModified": 1608723372, + "narHash": "sha256-XoCD827n1+bBPVxjKFmAikh7KSIspj9BBj2vnWhtTC4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8006772a054ce57ca18c5955dcd6ec9a62577473", + "rev": "b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4", "type": "github" }, "original": { diff --git a/utils/nix/phps.nix b/utils/nix/phps.nix index 821ea7649c..9af81f136f 100644 --- a/utils/nix/phps.nix +++ b/utils/nix/phps.nix @@ -5,7 +5,86 @@ nixpkgs: final: prev: let - _args = { inherit (prev) callPackage lib stdenv nixosTests; }; + _args = { + inherit (prev) callPackage lib stdenv nixosTests; + + packageOverrides = self: super: { + extensions = super.extensions // { + dom = super.extensions.dom.overrideAttrs (attrs: { + patches = attrs.patches or [] ++ prev.lib.optionals (prev.lib.versionOlder super.php.version "7.2") [ + # Fix tests with libxml2 2.9.10. + (prev.fetchpatch { + url = "https://github.com/php/php-src/commit/e29922f054639a934f3077190729007896ae244c.patch"; + sha256 = "zC2QE6snAhhA7ItXgrc80WlDVczTlZEzgZsD7AS+gtw="; + }) + ]; + }); + + intl = super.extensions.intl.overrideAttrs (attrs: { + doCheck = if prev.lib.versionOlder super.php.version "7.2" then false else attrs.doCheck or true; + patches = attrs.patches or [] ++ prev.lib.optionals (prev.lib.versionOlder super.php.version "7.1") [ + # Fix build with newer ICU. + (prev.fetchpatch { + url = "https://github.com/php/php-src/commit/8d35a423838eb462cd39ee535c5d003073cc5f22.patch"; + sha256 = if prev.lib.versionOlder super.php.version "7.0" then "8v0k6zaE5w4yCopCVa470TMozAXyK4fQelr+KuVnAv4=" else "NO3EY5z1LFWKor9c/9rJo1rpigG5x8W3Uj5+xAOwm+g="; + postFetch = '' + patch "$out" < ${if prev.lib.versionOlder super.php.version "7.0" then ./intl-icu-patch-5.6-compat.patch else ./intl-icu-patch-7.0-compat.patch} + ''; + }) + ]; + }); + + mysqlnd = + if prev.lib.versionOlder super.php.version "7.1" then + super.extensions.mysqlnd.overrideAttrs (attrs: { + # Fix mysqlnd not being able to find headers. + postPatch = attrs.postPatch or "" + "\n" + '' + ln -s $PWD/../../ext/ $PWD + ''; + }) + else + super.extensions.mysqlnd; + + opcache = super.extensions.opcache.overrideAttrs (attrs: { + # The patch do not apply to PHP 5’s opcache. + patches = if prev.lib.versionOlder super.php.version "7.0" then [] else attrs.patches or []; + }); + + openssl = + if prev.lib.versionOlder super.php.version "7.1" then + super.extensions.openssl.overrideAttrs (attrs: { + # PHP ≤ 7.0 requires openssl 1.0. + buildInputs = + let + openssl_1_0_2 = prev.openssl_1_0_2.overrideAttrs (attrs: { + meta = attrs.meta // { + # It is insecure but that should not matter in an isolated test environment. + knownVulnerabilities = []; + }; + }); + in + map (p: if p == prev.openssl then openssl_1_0_2 else p) attrs.buildInputs or []; + }) + else + super.extensions.openssl; + + readline = super.extensions.readline.overrideAttrs (attrs: { + patches = attrs.patches or [] ++ prev.lib.optionals (prev.lib.versionOlder super.php.version "7.2") [ + # Fix readline build + (prev.fetchpatch { + url = "https://github.com/php/php-src/commit/1ea58b6e78355437b79fb7b1f287ba6688fb1c57.patch"; + sha256 = "Lh2h07lKkAXpyBGqgLDNXeiOocksARTYIysLWMon694="; + }) + ]; + }); + + zlib = super.extensions.zlib.overrideAttrs (attrs: { + # The patch does not apply to PHP 7’s zlib. + patches = if prev.lib.versionOlder super.php.version "7.1" then [] else attrs.patches or []; + }); + }; + }; + }; generic = (import "${nixpkgs}/pkgs/development/interpreters/php/generic.nix") _args; @@ -28,104 +107,28 @@ let version = "7.2.34"; sha256 = "DlgW1miiuxSspozvjEMEML2Gw8UjP2xCfRpUqsEnq88="; }); - - fixDom = dom: dom.overrideAttrs (attrs: { - patches = attrs.patches or [] ++ [ - # Fix tests with libxml2 2.9.10 - (prev.fetchpatch { - url = "https://github.com/php/php-src/commit/e29922f054639a934f3077190729007896ae244c.patch"; - sha256 = "zC2QE6snAhhA7ItXgrc80WlDVczTlZEzgZsD7AS+gtw="; - }) - ]; - }); - - fixOpenssl = openssl: openssl.overrideAttrs (attrs: { - # PHP 5.6 requires openssl 1.0. It is insecure but that should not matter in an isolated test environment. - buildInputs = map (p: if p == prev.openssl then prev.openssl_1_0_2.overrideAttrs (attrs: { meta = attrs.meta // { knownVulnerabilities = []; }; }) else p) attrs.buildInputs or []; - }); - - fixZlib = zlib: zlib.overrideAttrs (attrs: { - # The patch do not apply to PHP 7’s zlib. - patches = if prev.lib.versionOlder zlib.version "7.1" then [] else attrs.patches; - }); - - fixOpcache = opcache: opcache.overrideAttrs (attrs: { - # The patch do not apply to PHP 5’s opcache. - patches = if prev.lib.versionOlder opcache.version "7.0" then [] else attrs.patches; - }); - - # Replace mysqlnd dependency by our fixed one. - fixMysqlndDep = ext: ext.overrideAttrs (attrs: - let - origDeps = attrs.internalDeps or []; - mysqlnds = builtins.filter (p: p.extensionName == "mysqlnd") origDeps; - in { - internalDeps = map (p: if p.extensionName == "mysqlnd" then fixMysqlnd p else p) origDeps; - preConfigure = - prev.lib.pipe (attrs.preConfigure or "") [ - # We need to discard string context because replaceStrings does not seem to update it. - builtins.unsafeDiscardStringContext - - # Fix mysqlnd references. - (builtins.replaceStrings (map (p: "${p.dev}") mysqlnds) (map (p: "${(fixMysqlnd p).dev}") mysqlnds)) - - # Re-introduce original context. - (builtins.replaceStrings (map (p: "${p.dev}") origDeps) (map (p: "${p.dev}") origDeps)) - ]; - } - ); - - fixMysqlnd = mysqlnd: mysqlnd.overrideAttrs (attrs: { - postPatch = attrs.postPatch or "" + "\n" + '' - ln -s $PWD/../../ext/ $PWD - ''; - }); - - fixRl = rl: rl.overrideAttrs (attrs: { - patches = attrs.patches or [] ++ [ - # Fix readline build - (prev.fetchpatch { - url = "https://github.com/php/php-src/commit/1ea58b6e78355437b79fb7b1f287ba6688fb1c57.patch"; - sha256 = "Lh2h07lKkAXpyBGqgLDNXeiOocksARTYIysLWMon694="; - }) - ]; - }); - fixIntl = intl: intl.overrideAttrs (attrs: { - doCheck = false; - patches = attrs.patches or [] ++ prev.lib.optionals (prev.lib.versionOlder intl.version "7.1") [ - # Fix build with newer ICU. - (prev.fetchpatch { - url = "https://github.com/php/php-src/commit/8d35a423838eb462cd39ee535c5d003073cc5f22.patch"; - sha256 = if prev.lib.versionOlder intl.version "7.0" then "8v0k6zaE5w4yCopCVa470TMozAXyK4fQelr+KuVnAv4=" else "NO3EY5z1LFWKor9c/9rJo1rpigG5x8W3Uj5+xAOwm+g="; - postFetch = '' - patch "$out" < ${if prev.lib.versionOlder intl.version "7.0" then ./intl-icu-patch-5.6-compat.patch else ./intl-icu-patch-7.0-compat.patch} - ''; - }) - ]; - }); - in { php56 = base56.withExtensions ({ all, ... }: with all; ([ - bcmath calendar curl ctype (fixDom dom) exif fileinfo filter ftp gd - gettext gmp hash iconv (fixIntl intl) json ldap mbstring (fixMysqlndDep mysqli) (fixMysqlnd mysqlnd) (fixOpcache opcache) - (fixOpenssl openssl) pcntl pdo (fixMysqlndDep pdo_mysql) pdo_odbc pdo_pgsql pdo_sqlite pgsql - posix (fixRl readline) session simplexml sockets soap sqlite3 - tokenizer xmlreader xmlwriter zip (fixZlib zlib) + bcmath calendar curl ctype dom exif fileinfo filter ftp gd + gettext gmp hash iconv intl json ldap mbstring mysqli mysqlnd opcache + openssl pcntl pdo pdo_mysql pdo_odbc pdo_pgsql pdo_sqlite pgsql + posix readline session simplexml sockets soap sqlite3 + tokenizer xmlreader xmlwriter zip zlib ] ++ prev.lib.optionals (!prev.stdenv.isDarwin) [ imap ])); php70 = base70.withExtensions ({ all, ... }: with all; ([ - bcmath calendar curl ctype (fixDom dom) exif fileinfo filter ftp gd - gettext gmp hash iconv (fixIntl intl) json ldap mbstring (fixMysqlndDep mysqli) (fixMysqlnd mysqlnd) opcache - (fixOpenssl openssl) pcntl pdo (fixMysqlndDep pdo_mysql) pdo_odbc pdo_pgsql pdo_sqlite pgsql - posix (fixRl readline) session simplexml sockets soap sqlite3 - tokenizer xmlreader xmlwriter zip (fixZlib zlib) + bcmath calendar curl ctype dom exif fileinfo filter ftp gd + gettext gmp hash iconv intl json ldap mbstring mysqli mysqlnd opcache + openssl pcntl pdo pdo_mysql pdo_odbc pdo_pgsql pdo_sqlite pgsql + posix readline session simplexml sockets soap sqlite3 + tokenizer xmlreader xmlwriter zip zlib ] ++ prev.lib.optionals (!prev.stdenv.isDarwin) [ imap ])); php71 = base71.withExtensions ({ all, ... }: with all; ([ - bcmath calendar curl ctype (fixDom dom) exif fileinfo filter ftp gd - gettext gmp hash iconv (fixIntl intl) json ldap mbstring mysqli mysqlnd opcache + bcmath calendar curl ctype dom exif fileinfo filter ftp gd + gettext gmp hash iconv intl json ldap mbstring mysqli mysqlnd opcache openssl pcntl pdo pdo_mysql pdo_odbc pdo_pgsql pdo_sqlite pgsql - posix (fixRl readline) session simplexml sockets soap sqlite3 + posix readline session simplexml sockets soap sqlite3 tokenizer xmlreader xmlwriter zip zlib ] ++ prev.lib.optionals (!prev.stdenv.isDarwin) [ imap ])); From 09274ce13cf3bb34c06d1db4e3486c7bc2bd2899 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Wed, 16 Dec 2020 20:37:53 +0100 Subject: [PATCH 2/6] nix: Add python This will allow us to run integration test and create-zipball script reproducibly. --- flake.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flake.nix b/flake.nix index 7883c55734..d3f9364b49 100644 --- a/flake.nix +++ b/flake.nix @@ -19,12 +19,18 @@ php = pkgs.${phpPackage}.withExtensions ({ enabled, all }: with all; enabled ++ [ imagick ]); + + python = pkgs.python3.withPackages (pp: with pp; [ + requests + bcrypt + ]); in pkgs.mkShell { nativeBuildInputs = [ php pkgs.zola pkgs.nodejs_latest + python ] ++ (with php.packages; [ composer psalm From e98a43108d128a8ea9678c6c272fa8fa36f6b019 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Wed, 16 Dec 2020 20:47:20 +0100 Subject: [PATCH 3/6] ci: Switch to Nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously we were installing - PHP using Travis’s built-in mechanism that downloads distributions-specific tarballs (PHP 7.1 is not compatible with focal), - Composer using itself, - Python using pyenv (quite slow), - Python libraries using pip, - Node using nvm, - jq using apt, - PHP dependencies using Composer, - client-side dependencies using npm. Switching to Nix will allow us to get rid of all except the last two methods and achieve fully reproducible environment across both CI and developers’ machines. The travis config is based on https://github.com/cachix/travis-ci-example. Also allows us to switch distribution to focal. Additionally, I had to add shebang to bump-version.js and make it executable since npm run does not seem to set $NODE any more. --- .travis.yml | 79 ++++++++++++++++++++++------------------- flake.nix | 7 +++- utils/bintray.sh | 5 +-- utils/bump-version.js | 1 + utils/create-zipball.py | 1 + 5 files changed, 54 insertions(+), 39 deletions(-) mode change 100644 => 100755 utils/bump-version.js mode change 100644 => 100755 utils/create-zipball.py diff --git a/.travis.yml b/.travis.yml index 6c4f9eb035..0eefeb0a3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,29 +1,28 @@ -# This file lints the code, builds a package and then deploys it. -# For the deployment, $BINTRAY_USER, $BINTRAY_KEY and $GH_TOKEN environment -# variables need to be set. -language: php +# This file lints the code, runs tests, builds a package and then deploys it. +# The following environment variables need to be set: +# - “$BINTRAY_USER” and “$BINTRAY_KEY” for pushing built package to Bintray +# - “$GH_TOKEN” to avoid Composer being throttled by GitHub +# - “$CACHIX_AUTH_TOKEN” for uploading built Nix packages to Cachix +language: nix +dist: focal +nix: 2.3.9 +sudo: false -dist: xenial +env: + global: + - CACHIX_CACHE=fossar + - COMPOSER_NO_INTERACTION=1 matrix: include: - - php: 8.0 - - php: 7.4 - - php: 7.3 - - php: 7.2 - - php: 7.1 - - php: 7.0 - env: CS_FIXER=true LINT_JS=true - - php: 5.6 - env: DEPLOY=true + - env: PHP=80 + - env: PHP=74 + - env: PHP=73 + - env: PHP=72 + - env: PHP=71 + - env: PHP=70 CS_FIXER=true LINT_JS=true + - env: PHP=56 DEPLOY=true fast_finish: true -env: - - COMPOSER_NO_INTERACTION=1 - -addons: - apt: - packages: - - jq cache: directories: @@ -31,29 +30,37 @@ cache: - $HOME/.composer/cache before_install: - - nvm install 12 - - pyenv install 3.6.3 - - pyenv global 3.6.3 - - pip3 install requests bcrypt - - if [ -n "$GH_TOKEN" ]; then composer config github-oauth.github.com ${GH_TOKEN}; fi - - composer self-update + # Nix & Cachix set-up. + - echo "trusted-users = $USER" | sudo tee -a /etc/nix/nix.conf + - sudo systemctl restart nix-daemon + - nix-env -iA nixpkgs.cachix + - cachix use $CACHIX_CACHE + - nix path-info --all > /tmp/store-path-pre-build + + # Update flake.nix to match the current CI job from matrix. + - sed -i "s/matrix.php = \"php\";/matrix.php = \"php${PHP}\";/" flake.nix + + - if [ -n "$GH_TOKEN" ]; then nix-shell --run "composer config github-oauth.github.com ${GH_TOKEN}"; fi install: - - phpenv config-rm xdebug.ini - - npm run install-dependencies + - nix-shell --run "npm run install-dependencies" script: - - if [ "$LINT_JS" = true ]; then npm run lint:client; fi - - npm run lint:server - - if [ "$CS_FIXER" = true ]; then npm run cs:server; fi - - npm run test:server - - npm run test:integration + - if [ "$LINT_JS" = true ]; then nix-shell --run "npm run lint:client"; fi + - nix-shell --run "npm run lint:server" + - if [ "$CS_FIXER" = true ]; then nix-shell --run "npm run cs:server"; fi + - nix-shell --run "npm run test:server" + - nix-shell --run "npm run test:integration" + +after_success: + # Upload built Nix packages to Cachix. + - comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_CACHE before_deploy: - git config --global user.email 'Travis CI' - git config --global user.name 'builds@travis-ci.org' - - source utils/package.sh - - source utils/bintray.sh + - nix-shell --run "utils/package.sh" + - SELFOSS_ZIPBALL=$(nix-shell --run "utils/bintray.sh") deploy: - provider: bintray diff --git a/flake.nix b/flake.nix index d3f9364b49..6cb3f2cb58 100644 --- a/flake.nix +++ b/flake.nix @@ -14,6 +14,8 @@ outputs = { self, flake-compat, nixpkgs, utils }: let + matrix.php = "php"; + mkDevShell = pkgs: phpPackage: let php = pkgs.${phpPackage}.withExtensions ({ enabled, all }: with all; enabled ++ [ @@ -31,11 +33,14 @@ pkgs.zola pkgs.nodejs_latest python + pkgs.jq ] ++ (with php.packages; [ composer psalm phpstan ]); + + LOCALE_ARCHIVE = "${pkgs.glibcLocales}/lib/locale/locale-archive"; }; in utils.lib.eachDefaultSystem (system: @@ -50,7 +55,7 @@ packages = { inherit (pkgs) php56 php70 php71 php72; }; - devShell = mkDevShell pkgs "php"; + devShell = mkDevShell pkgs matrix.php; } ); } diff --git a/utils/bintray.sh b/utils/bintray.sh index d87c39482d..df703f1fd4 100755 --- a/utils/bintray.sh +++ b/utils/bintray.sh @@ -1,4 +1,5 @@ #!/bin/sh -export SELFOSS_ZIPBALL=$(echo selfoss-*.zip) -export SELFOSS_VERSION=$(jq -r '.ver' package.json) +SELFOSS_ZIPBALL=$(echo selfoss-*.zip) +SELFOSS_VERSION=$(jq -r '.ver' package.json) sed -i "s/SELFOSS_VERSION/$SELFOSS_VERSION/g;s/SELFOSS_ZIPBALL/$SELFOSS_ZIPBALL/g" utils/bintray.json +echo $SELFOSS_ZIPBALL diff --git a/utils/bump-version.js b/utils/bump-version.js old mode 100644 new mode 100755 index 86a4c6ff36..7f699e8ea7 --- a/utils/bump-version.js +++ b/utils/bump-version.js @@ -1,3 +1,4 @@ +#!/usr/bin/env node const fs = require('fs'); if (process.argv.length <= 2) { diff --git a/utils/create-zipball.py b/utils/create-zipball.py old mode 100644 new mode 100755 index 02cdf29490..69210c8be4 --- a/utils/create-zipball.py +++ b/utils/create-zipball.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import json import logging import os From dcc4b07e44edee3ddd71718e63bc2200f4614136 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Thu, 17 Dec 2020 12:25:52 +0100 Subject: [PATCH 4/6] docs: Mention integration tests requirements --- docs/content/docs/development/setting-up.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/docs/development/setting-up.md b/docs/content/docs/development/setting-up.md index 5118c8856e..318e5238d5 100644 --- a/docs/content/docs/development/setting-up.md +++ b/docs/content/docs/development/setting-up.md @@ -9,6 +9,8 @@ Then you will be able to run `npm run install-dependencies` to install the libra To run the server side you will need at least [PHP](https://www.php.net/downloads) to be able to run the development server using `php -S 127.0.0.1:8000 run.php`. It would be also nice to have an array of database servers (MySQL and PostgreSQL) and web servers (Apache httpd and nginx) but the server built into PHP and SQLite will suffice for small changes. +Integration tests require [Python](https://python.org/) ≥ 3.6 with `requests` and `bcrypt` Python libraries. + For changing the selfoss web page in `docs/` directory, you will also want [Zola](https://www.getzola.org/documentation/getting-started/installation/). You can install all of the above using your package manager of choice or by downloading the programs from the linked pages. From 7e063777190cad438713d6e37e26f42f81526c56 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Fri, 18 Dec 2020 14:38:33 +0100 Subject: [PATCH 5/6] nix: Simplify and comment flake.nix --- .travis.yml | 2 +- flake.nix | 68 ++++++++++++++++++++++++++++++++++------------------- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0eefeb0a3f..d4c78b166d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ before_install: - nix path-info --all > /tmp/store-path-pre-build # Update flake.nix to match the current CI job from matrix. - - sed -i "s/matrix.php = \"php\";/matrix.php = \"php${PHP}\";/" flake.nix + - sed -i "s/matrix.phpPackage = \"php\";/matrix.phpPackage = \"php${PHP}\";/" flake.nix - if [ -n "$GH_TOKEN" ]; then nix-shell --run "composer config github-oauth.github.com ${GH_TOKEN}"; fi diff --git a/flake.nix b/flake.nix index 6cb3f2cb58..f588ad22bc 100644 --- a/flake.nix +++ b/flake.nix @@ -2,11 +2,13 @@ description = "selfoss feed reader and aggregator"; inputs = { + # Shim to make flake.nix work with stable Nix. flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; + # Repository with software packages. nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; utils.url = "github:numtide/flake-utils"; @@ -14,48 +16,66 @@ outputs = { self, flake-compat, nixpkgs, utils }: let - matrix.php = "php"; + # Configure the development shell here (e.g. for CI). - mkDevShell = pkgs: phpPackage: + # By default, we use the default PHP version from Nixpkgs. + matrix.phpPackage = "php"; + in + # For each supported platform, + utils.lib.eachDefaultSystem (system: let - php = pkgs.${phpPackage}.withExtensions ({ enabled, all }: with all; enabled ++ [ + # Let’s merge the package set from Nixpkgs with our custom PHP versions. + pkgs = import nixpkgs.outPath { + inherit system; + overlays = [ + (import ./utils/nix/phps.nix nixpkgs.outPath) + ]; + }; + + # Create a PHP package from the selected PHP package, with some extra extensions enabled. + php = pkgs.${matrix.phpPackage}.withExtensions ({ enabled, all }: with all; enabled ++ [ imagick ]); + # Create a Python package with some extra packages installed. python = pkgs.python3.withPackages (pp: with pp; [ - requests + # For integration tests. bcrypt + requests ]); - in - pkgs.mkShell { + in { + # Expose shell environment for development. + devShell = pkgs.mkShell { nativeBuildInputs = [ + # Composer and PHP for back-end. php - pkgs.zola + php.packages.composer + + # Back-end code validation. + php.packages.psalm + php.packages.phpstan + + # npm for front-end. pkgs.nodejs_latest - python + + # For building zip archive. pkgs.jq - ] ++ (with php.packages; [ - composer - psalm - phpstan - ]); - LOCALE_ARCHIVE = "${pkgs.glibcLocales}/lib/locale/locale-archive"; - }; - in - utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs.outPath { - inherit system; - overlays = [ - (import ./utils/nix/phps.nix nixpkgs.outPath) + # For building zip archive and integration tests. + python + + # Website generator. + pkgs.zola ]; + + # node-gyp wants some locales, let’s make them available through an environment variable. + LOCALE_ARCHIVE = "${pkgs.glibcLocales}/lib/locale/locale-archive"; }; - in { + + # Expose our custom PHP packages for testing. packages = { inherit (pkgs) php56 php70 php71 php72; }; - devShell = mkDevShell pkgs matrix.php; } ); } From e6e1ba73e2d602bebfc97e70c27925b446c6858b Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Fri, 18 Dec 2020 16:41:31 +0100 Subject: [PATCH 6/6] docs: Document Nix workflow --- docs/content/docs/development/setting-up.md | 4 +- docs/content/docs/development/using-nix.md | 44 +++++++++++++++++++++ docs/sass/style.scss | 14 +++++++ 3 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 docs/content/docs/development/using-nix.md diff --git a/docs/content/docs/development/setting-up.md b/docs/content/docs/development/setting-up.md index 318e5238d5..abb6daa967 100644 --- a/docs/content/docs/development/setting-up.md +++ b/docs/content/docs/development/setting-up.md @@ -15,6 +15,4 @@ For changing the selfoss web page in `docs/` directory, you will also want [Zola You can install all of the above using your package manager of choice or by downloading the programs from the linked pages. -Alternately, on Linux and MacOS, you can run [Nix package manager](https://nixos.org/download.html)’s `nix-shell` command, and you will find yourself in a development environment with all the necessary dependencies on `PATH`. - -Or even nicer, you can install [direnv](https://direnv.net/) and your terminal will load the Nix-based development environment automatically when you `cd` into the `selfoss` directory. +Alternately, on Linux and MacOS, you can run [Nix package manager](https://nixos.org/download.html)’s `nix-shell` command, and you will find yourself in a development environment with all the necessary dependencies on `PATH`. See [Using Nix](@/docs/development/using-nix.md) for more information. diff --git a/docs/content/docs/development/using-nix.md b/docs/content/docs/development/using-nix.md new file mode 100644 index 0000000000..1a26bf3eba --- /dev/null +++ b/docs/content/docs/development/using-nix.md @@ -0,0 +1,44 @@ ++++ +title = "Using Nix" +weight = 20 ++++ + +While by no means necessary and you can install all dependencies [manually](@/docs/development/setting-up.md), [Nix package manager](https://nixos.org/download.html) can download all dependencies for development and make them available automatically. It can be used on Linux systems in parallel with your system package manager, and even on MacOS (though list of supported packages is narrower there). + +After you install Nix, you just need to run `nix-shell` command in the selfoss repository, and you will find yourself in a development environment with all the necessary dependencies on `PATH`. + +Or, even nicer, you can install [direnv](https://direnv.net/) and your terminal will load the Nix-based development environment automatically when you `cd` into the selfoss repository directory. + +## How does Nix work {#nix} + +Nix package manager evaluates package descriptions written in the *Nix* expression language and then turns them into concrete instructions for building packages (*\*.drv* files). Those can then be built or, if the package has already been built by Nixpkgs’ infrastructure, a prebuilt package can be downloaded from binary cache. + +Nix is a full-fledged functional programming language (think JSON with functions) and the expressions describing packages are just arbitrary files. Nix community also has a central repository called [*Nixpkgs*](https://github.com/NixOS/nixpkgs) that contains thousands of software packages plus library of functions to make creating package descriptions more convenient. + +Nix provides, among other things, `nix-shell` command. Running it will evaluate a dummy package described in `shell.nix` file and place you into a new shell environment with dependencies of that package added to `PATH` environment variable. This will allow you to run them as if you installed them any other way. + +Nix language allows loading files containing Nix expressions using `import "path"` command and downloading git repositories and returning paths they were cloned into using `builtins.fetchGit { url = "foo"; rev = "bar"; sha256 = "xxx"; }` function. Using these primitives,[^flakes] we load a specific snapshot (git commit) from Nixpkgs repository and use PHP, composer, npm and other dependencies from there in our shell environment. + +[^flakes]: Recently, *Flakes*, an experimental, more high-level method for expressing dependencies between git repositories has been introduced. It allows us to specify dependency on other repositories like Nixpkgs declaratively in `flake.nix` file, and separates the information about pinned versions into a `flake.lock` file instead of the low-level functions. Our `shell.nix` is actually a [compatibility shim](https://github.com/edolstra/flake-compat/) that calls `builtins.fetchGit` function with the data from the lock file. + +## Our set-up {#selfoss-nix} + +As we have already mentioned, we describe the development environment in `flake.nix` and we pull some packages from Nixpkgs repository. For maintenance reasons, Nixpkgs usually only contains a single version of each package or, in case of platforms like PHP, single version of each supported branch. Since selfoss aims to support even shared hosts with older PHP versions, we have to build those versions ourselves. Fortunately, it is quite easy using existing Nixpkgs infrastructure – we just pass version, hash and some patches to the Nixpkgs’s PHP builder function (see [`utils/nix/phps.nix`](https://github.com/fossar/selfoss/blob/master/utils/nix/phps.nix)). + +## Bumping pinned dependencies {#bumping} + +The pinned Nixpkgs version can be updated with `nixUnstable` using `nix flake update --recreate-lock-file`, or with stable Nix using `nix-shell -I nixpkgs=channel:nixos-unstable -p nixUnstable --run 'nix --experimental-features "nix-command flakes" flake update --recreate-lock-file'`. + +## Optimizing the workflow {#optimizing} + +Nix uses binary cache to avoid building packages from Nixpkgs over and over again but the packages we created for old PHP versions are obviously not cached by the official Nixpkgs cache. You can install [Cachix](https://docs.cachix.org/installation.html) and run `cachix use fossar` to enjoy the same benefits for those packages as well. It is not necessary for the default PHP version, though, only if you want to test selfoss on one of the [unmaintained versions](#switching-php). + +As mentioned above, we are not actually using Flakes but emulate them using stable Nix features. With `nixUnstable`, we can use Flakes directly and benefit from dramatically increased performance. After you install `nixUnstable` and enable the experimental features `echo 'experimental-features = nix-command flakes' >> ~/.config/nix/nix.conf`, you can use `nix develop` instead of `nix-shell` or include [nix-direnv](https://github.com/nix-community/nix-direnv) into your direnv configuration to make direnv use Flakes as well. + +## Switching PHP versions {#switching-php} + +By default, the environment will contain the default PHP version from Nixpkgs. You can change the version by replacing `matrix.php = "php";` with `matrix.php = "phpXY";` in `flake.nix`, where `X` and `Y` are major and minor version respectively. + +Supported versions depend on which versions are in the pinned version of Nixpkgs in [`pkgs/development/interpreters/php`](https://github.com/NixOS/nixpkgs/tree/nixpkgs-unstable/pkgs/development/interpreters/php) and whichever versions are we keeping locally in [`utils/nix/phps.nix`](https://github.com/fossar/selfoss/blob/master/utils/nix/phps.nix). + +After you change the value, exit the shell and start a new one, or if you are using `direnv`, execute `touch shell.nix` to trigger the reload of the environment. diff --git a/docs/sass/style.scss b/docs/sass/style.scss index 53fda46a51..51a58c9f91 100644 --- a/docs/sass/style.scss +++ b/docs/sass/style.scss @@ -401,3 +401,17 @@ kbd { margin-top: 0; } } + +// Footnotes from pulldown-cmark +.footnote-definition { + display: flex; + font-size: 0.95em; +} + +.footnote-definition p { + margin: 0; +} + +.footnote-definition-label { + margin-right: 0.95em; +}