ci: Use Nix for obtaining dependencies

.travis.yml

@@ -1,59 +1,66 @@
-# This file lints the code, builds a package and then deploys it.
-# For the deployment, $BINTRAY_USER, $BINTRAY_KEY and $GH_TOKEN environment
-# variables need to be set.
-language: php
+# This file lints the code, runs tests, builds a package and then deploys it.
+# The following environment variables need to be set:
+# - “$BINTRAY_USER” and “$BINTRAY_KEY” for pushing built package to Bintray
+# - “$GH_TOKEN” to avoid Composer being throttled by GitHub
+# - “$CACHIX_AUTH_TOKEN” for uploading built Nix packages to Cachix
+language: nix
+dist: focal
+nix: 2.3.9
+sudo: false
 
-dist: xenial
+env:
+    global:
+        - CACHIX_CACHE=fossar
+        - COMPOSER_NO_INTERACTION=1
 
 matrix:
     include:
-        - php: 8.0
-        - php: 7.4
-        - php: 7.3
-        - php: 7.2
-        - php: 7.1
-        - php: 7.0
-          env: CS_FIXER=true LINT_JS=true
-        - php: 5.6
-          env: DEPLOY=true
+        - env: PHP=80
+        - env: PHP=74
+        - env: PHP=73
+        - env: PHP=72
+        - env: PHP=71
+        - env: PHP=70 CS_FIXER=true LINT_JS=true
+        - env: PHP=56 DEPLOY=true
     fast_finish: true
-env:
-    - COMPOSER_NO_INTERACTION=1
-
-addons:
-    apt:
-        packages:
-            - jq
 
 cache:
     directories:
         - vendor
         - $HOME/.composer/cache
 
 before_install:
-    - nvm install 12
-    - pyenv install 3.6.3
-    - pyenv global 3.6.3
-    - pip3 install requests bcrypt
-    - if [ -n "$GH_TOKEN" ]; then composer config github-oauth.github.com ${GH_TOKEN}; fi
-    - composer self-update
+    # Nix & Cachix set-up.
+    - echo "trusted-users = $USER" | sudo tee -a /etc/nix/nix.conf
+    - sudo systemctl restart nix-daemon
+    - nix-env -iA nixpkgs.cachix
+    - cachix use $CACHIX_CACHE
+    - nix path-info --all > /tmp/store-path-pre-build
+
+    # Update flake.nix to match the current CI job from matrix.
+    - sed -i "s/matrix.phpPackage = \"php\";/matrix.phpPackage = \"php${PHP}\";/" flake.nix
+
+    - if [ -n "$GH_TOKEN" ]; then nix-shell --run "composer config github-oauth.github.com ${GH_TOKEN}"; fi
 
 install:
-    - phpenv config-rm xdebug.ini
-    - npm run install-dependencies
+    - nix-shell --run "npm run install-dependencies"
 
 script:
-    - if [ "$LINT_JS" = true ]; then npm run lint:client; fi
-    - npm run lint:server
-    - if [ "$CS_FIXER" = true ]; then npm run cs:server; fi
-    - npm run test:server
-    - npm run test:integration
+    - if [ "$LINT_JS" = true ]; then nix-shell --run "npm run lint:client"; fi
+    - nix-shell --run "npm run lint:server"
+    - if [ "$CS_FIXER" = true ]; then nix-shell --run "npm run cs:server"; fi
+    - nix-shell --run "npm run test:server"
+    - nix-shell --run "npm run test:integration"
+
+after_success:
+    # Upload built Nix packages to Cachix.
+    - comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_CACHE
 
 before_deploy:
     - git config --global user.email 'Travis CI'
     - git config --global user.name '[email protected]'
-    - source utils/package.sh
-    - source utils/bintray.sh
+    - nix-shell --run "utils/package.sh"
+    - SELFOSS_ZIPBALL=$(nix-shell --run "utils/bintray.sh")
 
 deploy:
     - provider: bintray

docs/content/docs/development/setting-up.md

@@ -9,10 +9,10 @@ Then you will be able to run `npm run install-dependencies` to install the libra
 
 To run the server side you will need at least [PHP](https://www.php.net/downloads) to be able to run the development server using `php -S 127.0.0.1:8000 run.php`. It would be also nice to have an array of database servers (MySQL and PostgreSQL) and web servers (Apache httpd and nginx) but the server built into PHP and SQLite will suffice for small changes.
 
+Integration tests require [Python](https://python.org/) ≥ 3.6 with `requests` and `bcrypt` Python libraries.
+
 For changing the selfoss web page in `docs/` directory, you will also want [Zola](https://www.getzola.org/documentation/getting-started/installation/).
 
 You can install all of the above using your package manager of choice or by downloading the programs from the linked pages.
 
-Alternately, on Linux and MacOS, you can run [Nix package manager](https://nixos.org/download.html)’s `nix-shell` command, and you will find yourself in a development environment with all the necessary dependencies on `PATH`.
-
-Or even nicer, you can install [direnv](https://direnv.net/) and your terminal will load the Nix-based development environment automatically when you `cd` into the `selfoss` directory.
+Alternately, on Linux and MacOS, you can run [Nix package manager](https://nixos.org/download.html)’s `nix-shell` command, and you will find yourself in a development environment with all the necessary dependencies on `PATH`. See [Using Nix](@/docs/development/using-nix.md) for more information.

docs/content/docs/development/using-nix.md

@@ -0,0 +1,44 @@
++++
+title = "Using Nix"
+weight = 20
++++
+
+While by no means necessary and you can install all dependencies [manually](@/docs/development/setting-up.md), [Nix package manager](https://nixos.org/download.html) can download all dependencies for development and make them available automatically. It can be used on Linux systems in parallel with your system package manager, and even on MacOS (though list of supported packages is narrower there).
+
+After you install Nix, you just need to run `nix-shell` command in the selfoss repository, and you will find yourself in a development environment with all the necessary dependencies on `PATH`.
+
+Or, even nicer, you can install [direnv](https://direnv.net/) and your terminal will load the Nix-based development environment automatically when you `cd` into the selfoss repository directory.
+
+## How does Nix work {#nix}
+
+Nix package manager evaluates package descriptions written in the *Nix* expression language and then turns them into concrete instructions for building packages (*\*.drv* files). Those can then be built or, if the package has already been built by Nixpkgs’ infrastructure, a prebuilt package can be downloaded from binary cache.
+
+Nix is a full-fledged functional programming language (think JSON with functions) and the expressions describing packages are just arbitrary files. Nix community also has a central repository called [*Nixpkgs*](https://github.com/NixOS/nixpkgs) that contains thousands of software packages plus library of functions to make creating package descriptions more convenient.
+
+Nix provides, among other things, `nix-shell` command. Running it will evaluate a dummy package described in `shell.nix` file and place you into a new shell environment with dependencies of that package added to `PATH` environment variable. This will allow you to run them as if you installed them any other way.
+
+Nix language allows loading files containing Nix expressions using `import "path"` command and downloading git repositories and returning paths they were cloned into using `builtins.fetchGit { url = "foo"; rev = "bar"; sha256 = "xxx"; }` function. Using these primitives,[^flakes] we load a specific snapshot (git commit) from Nixpkgs repository and use PHP, composer, npm and other dependencies from there in our shell environment.
+
+[^flakes]: Recently, *Flakes*, an experimental, more high-level method for expressing dependencies between git repositories has been introduced. It allows us to specify dependency on other repositories like Nixpkgs declaratively in `flake.nix` file, and separates the information about pinned versions into a `flake.lock` file instead of the low-level functions. Our `shell.nix` is actually a [compatibility shim](https://github.com/edolstra/flake-compat/) that calls `builtins.fetchGit` function with the data from the lock file.
+
+## Our set-up  {#selfoss-nix}
+
+As we have already mentioned, we describe the development environment in `flake.nix` and we pull some packages from Nixpkgs repository. For maintenance reasons, Nixpkgs usually only contains a single version of each package or, in case of platforms like PHP, single version of each supported branch. Since selfoss aims to support even shared hosts with older PHP versions, we have to build those versions ourselves. Fortunately, it is quite easy using existing Nixpkgs infrastructure – we just pass version, hash and some patches to the Nixpkgs’s PHP builder function (see [`utils/nix/phps.nix`](https://github.com/fossar/selfoss/blob/master/utils/nix/phps.nix)).
+
+## Bumping pinned dependencies {#bumping}
+
+The pinned Nixpkgs version can be updated with `nixUnstable` using `nix flake update --recreate-lock-file`, or with stable Nix using `nix-shell -I nixpkgs=channel:nixos-unstable -p nixUnstable --run 'nix --experimental-features "nix-command flakes" flake update --recreate-lock-file'`.
+
+## Optimizing the workflow {#optimizing}
+
+Nix uses binary cache to avoid building packages from Nixpkgs over and over again but the packages we created for old PHP versions are obviously not cached by the official Nixpkgs cache. You can install [Cachix](https://docs.cachix.org/installation.html) and run `cachix use fossar` to enjoy the same benefits for those packages as well. It is not necessary for the default PHP version, though, only if you want to test selfoss on one of the [unmaintained versions](#switching-php).
+
+As mentioned above, we are not actually using Flakes but emulate them using stable Nix features. With `nixUnstable`, we can use Flakes directly and benefit from dramatically increased performance. After you install `nixUnstable` and enable the experimental features `echo 'experimental-features = nix-command flakes' >> ~/.config/nix/nix.conf`, you can use `nix develop` instead of `nix-shell` or include [nix-direnv](https://github.com/nix-community/nix-direnv) into your direnv configuration to make direnv use Flakes as well.
+
+## Switching PHP versions {#switching-php}
+
+By default, the environment will contain the default PHP version from Nixpkgs. You can change the version by replacing `matrix.php = "php";` with `matrix.php = "phpXY";` in `flake.nix`, where `X` and `Y` are major and minor version respectively.
+
+Supported versions depend on which versions are in the pinned version of Nixpkgs in [`pkgs/development/interpreters/php`](https://github.com/NixOS/nixpkgs/tree/nixpkgs-unstable/pkgs/development/interpreters/php) and whichever versions are we keeping locally in [`utils/nix/phps.nix`](https://github.com/fossar/selfoss/blob/master/utils/nix/phps.nix).
+
+After you change the value, exit the shell and start a new one, or if you are using `direnv`, execute `touch shell.nix` to trigger the reload of the environment.

docs/sass/style.scss

@@ -401,3 +401,17 @@ kbd {
         margin-top: 0;
     }
 }
+
+// Footnotes from pulldown-cmark
+.footnote-definition {
+    display: flex;
+    font-size: 0.95em;
+}
+
+.footnote-definition p {
+    margin: 0;
+}
+
+.footnote-definition-label {
+    margin-right: 0.95em;
+}

flake.nix

@@ -2,49 +2,80 @@
   description = "selfoss feed reader and aggregator";
 
   inputs = {
+    # Shim to make flake.nix work with stable Nix.
     flake-compat = {
       url = "github:edolstra/flake-compat";
       flake = false;
     };
 
+    # Repository with software packages.
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
 
     utils.url = "github:numtide/flake-utils";
   };
 
   outputs = { self, flake-compat, nixpkgs, utils }:
     let
-      mkDevShell = pkgs: phpPackage:
-        let
-          php = pkgs.${phpPackage}.withExtensions ({ enabled, all }: with all; enabled ++ [
-            imagick
-          ]);
-        in
-          pkgs.mkShell {
-            nativeBuildInputs = [
-              php
-              pkgs.zola
-              pkgs.nodejs_latest
-            ] ++ (with php.packages; [
-              composer
-              psalm
-              phpstan
-            ]);
-          };
+      # Configure the development shell here (e.g. for CI).
+
+      # By default, we use the default PHP version from Nixpkgs.
+      matrix.phpPackage = "php";
     in
+      # For each supported platform,
       utils.lib.eachDefaultSystem (system:
         let
+          # Let’s merge the package set from Nixpkgs with our custom PHP versions.
           pkgs = import nixpkgs.outPath {
             inherit system;
             overlays = [
               (import ./utils/nix/phps.nix nixpkgs.outPath)
             ];
           };
+
+          # Create a PHP package from the selected PHP package, with some extra extensions enabled.
+          php = pkgs.${matrix.phpPackage}.withExtensions ({ enabled, all }: with all; enabled ++ [
+            imagick
+          ]);
+
+          # Create a Python package with some extra packages installed.
+          python = pkgs.python3.withPackages (pp: with pp; [
+            # For integration tests.
+            bcrypt
+            requests
+          ]);
         in {
+          # Expose shell environment for development.
+          devShell = pkgs.mkShell {
+            nativeBuildInputs = [
+              # Composer and PHP for back-end.
+              php
+              php.packages.composer
+
+              # Back-end code validation.
+              php.packages.psalm
+              php.packages.phpstan
+
+              # npm for front-end.
+              pkgs.nodejs_latest
+
+              # For building zip archive.
+              pkgs.jq
+
+              # For building zip archive and integration tests.
+              python
+
+              # Website generator.
+              pkgs.zola
+            ];
+
+            # node-gyp wants some locales, let’s make them available through an environment variable.
+            LOCALE_ARCHIVE = "${pkgs.glibcLocales}/lib/locale/locale-archive";
+          };
+
+          # Expose our custom PHP packages for testing.
           packages = {
             inherit (pkgs) php56 php70 php71 php72;
           };
-          devShell = mkDevShell pkgs "php";
         }
       );
 }

utils/bintray.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
-export SELFOSS_ZIPBALL=$(echo selfoss-*.zip)
-export SELFOSS_VERSION=$(jq -r '.ver' package.json)
+SELFOSS_ZIPBALL=$(echo selfoss-*.zip)
+SELFOSS_VERSION=$(jq -r '.ver' package.json)
 sed -i "s/SELFOSS_VERSION/$SELFOSS_VERSION/g;s/SELFOSS_ZIPBALL/$SELFOSS_ZIPBALL/g" utils/bintray.json
+echo $SELFOSS_ZIPBALL

utils/bump-version.js

@@ -1,3 +1,4 @@
+#!/usr/bin/env node
 const fs = require('fs');
 
 if (process.argv.length <= 2) {

utils/create-zipball.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import json
 import logging
 import os