Add github action to build the phpList package for release (#763)

* create composer.json with the required dependencies

* put things in the right place with davidbarratt/custom-installer

* add updater

* add script to install plugin code

* add some plugins

* use bramleys version

* update dependency

* allow plugins from other vendors

* add some more plugins

* add some more plugins

* ignore updater

* rsync instead of mv

* add bash script to run tests, meant to be run inside a docker container

* flush privileges on login creation

* add selenium drivers

* run the tests

* add Dockerfile for release verification

* add behat file

* test upload to S3 action

* on tag of release

* typo

* add token to avoid rate limiting

* on second step as well

* upload to folder

* use ssh key for github auth

* try the http-auth way

* fix php version

* the theme is now installed with composer

* selenium drivers are set up with composer

* comment "debug step" that's failing

* do not ignore composer.lock

* try a new action

* fix bucket name

* update region

* add workflow to build a release

* copy test to test folder

* lock php version

* remove debug output

* use var for the bucket

* parse the version from the tag

* switch to latest ubuntu

* check s3cmd on 18.04

* go back to 18.04

* run as sudo

* put folder back to avoid confusion

* use RELEASE_VERSION

* fix path

* tidy up the zip files, to reduce size

* put the zips in a different bucket

* install plugins as a seperate step, as Composer skips it

* show github ref

* install required tools

* avoid annoying incompatibility

* go back to 18.04, there's a weird issue on mysql commandline in 20.04

* discover rsync location

* remove trigger on test action

* trigger the release build on tagging a new version

* parse off the v of the version, to follow previous convention

* remove failing PHP versions

* update some files with the new version

* typo

.github/workflows/build-release.yml

@@ -0,0 +1,192 @@
+
+name: Build Release
+
+on:
+  push:
+    tags: v[1-9]+.[0-9]+.*
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+    continue-on-error: ${{ matrix.experimental }}
+
+    strategy:
+      fail-fast: false     
+      matrix:   
+        php-version: ['7.4']
+        experimental: [false]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Find the version
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/v}" >> $GITHUB_ENV
+
+      - name: Show the discovered version
+        run: |
+          echo $RELEASE_VERSION
+          echo ${{ env.RELEASE_VERSION }}
+          echo $GITHUB_REF
+          echo ${{ env.GITHUB_REF }}
+
+      - name: Set up PHP ${{ matrix.php-version }}
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php-version }}
+
+      - name: Setup Packages
+        run: |
+          cd $GITHUB_WORKSPACE
+          sudo apt update && sudo apt install -y rsync
+
+      - name: Cache Composer dependencies
+        uses: actions/cache@v2
+        with:
+          path: /tmp/composer-cache
+          key: ${{ matrix.php-version }}-${{ hashFiles('**/composer.lock') }}
+
+      - name: Install dependencies
+        uses: php-actions/composer@v5
+        with:
+          dev: yes
+          args: --prefer-dist --no-interaction
+          php_version: 7.4
+          php_extensions: xml 
+          version: 2
+
+      - name: Install Plugins
+        run: |
+          cd $GITHUB_WORKSPACE
+          ./scripts/install-plugins.sh vendor public_html/lists/admin/plugins/
+
+      - name: Create Database
+        run: |
+          sudo systemctl start mysql.service
+          sudo mysql -proot  -e 'drop database if exists phplistdb'
+          sudo mysqladmin -proot create phplistdb
+          sudo mysql -proot -e 'grant all on phplistdb.* to phplist@localhost identified by "phplist"'
+
+      - name: Start Test Server
+        run: |
+          cd $GITHUB_WORKSPACE
+          cp -fv tests/default.behat.yml tests/behat.yml
+          cp -fv tests/ci/config.php public_html/lists/config/config.php
+          mkdir -p output/screenshots
+          mkdir -p build/mails
+          ./bin/start-selenium > output/selenium.log 2>&1 &
+          sleep 5
+          sudo php -S 0.0.0.0:80 -t public_html > /dev/null 2>&1 &
+
+      - name: Check PHP syntax errors
+        uses: overtrue/[email protected]
+        with:
+          path: ./public_html
+
+      - name: Report Versions
+        run: |
+          google-chrome --version
+          php -v
+          chromedriver -v
+          geckodriver -V
+          which geckodriver
+          which chromedriver
+          which rsync
+          rsync --version
+          firefox -v
+          vendor/bin/behat -V
+
+      - name: Run BDD Tests
+        run: |
+          cd $GITHUB_WORKSPACE/tests
+          ../vendor/bin/behat -n -fprogress -p chrome --strict --stop-on-failure --tags=@initialise
+          ../vendor/bin/behat -n -fprogress -p chrome --strict --stop-on-failure --tags="~@initialise && ~@wip"
+
+      - name: Clean Up
+        run: |
+          cd $GITHUB_WORKSPACE
+          cd ..
+          mv phplist3 phplist-$RELEASE_VERSION
+          sudo find . -type d -exec chmod 755 {} \;
+          sudo find . -type f -exec chmod 644 {} \;
+          cd phplist-$RELEASE_VERSION
+          chmod 777 public_html/lists/admin/plugins
+          cat public_html/lists/admin/init.php | tr '\n' '\r' | sed 's~//## remove on rollout.*//## end remove on rollout ###~define("VERSION","'${RELEASE_VERSION}'");~' | tr '\r' '\n' > /tmp/$$.init
+          mv -f /tmp/$$.init public_html/lists/admin/init.php
+          sed -i s~define\(\'STRUCTUREVERSION\'.*~define\(\'STRUCTUREVERSION\',\"${RELEASE_VERSION}\"\)\;~ public_html/lists/admin/structure.php
+          sed -i s/^VERSION=/VERSION=${RELEASE_VERSION}/ VERSION
+
+          cd ..
+          zip -rq9 phplist-${RELEASE_VERSION}.zip phplist-${RELEASE_VERSION}
+          tar cf phplist-${RELEASE_VERSION}.tar phplist-${RELEASE_VERSION}
+
+          remove=(
+            default.behat.yml
+            TESTING.md
+            .dotgitlab-ci.yml
+            .travis.yml
+            composer.json
+            composer.lock
+            .git
+            .github
+            .php_cs
+            .gitmodules
+            Dockerfile
+            Vagrantfile
+            .styleci.yml 
+            .gitignore 
+            .gitsvnextmodules 
+            tests
+            output.log
+            output
+            build
+            .phplist-cache
+            docker_tag
+            vendor
+            public_html/lists/admin/tests
+            scripts
+            bin/start-selenium
+            bin/fake-sendmail.sh
+            bin/imgur-uploader.sh
+            public_html/lists/base/tests
+            public_html/lists/base/.htaccess
+            public_html/lists/base/phpunit.xml.dist 
+            public_html/lists/base/composer.lock
+            public_html/lists/base/composer.json
+            public_html/lists/base/CODE_OF_CONDUCT.md 
+            public_html/lists/admin/ui/default
+          )
+
+          for item in ${remove[@]}; do
+            [[ -f phplist-$RELEASE_VERSION/$item ]] && {
+              tar -vf phplist-${RELEASE_VERSION}.tar --delete phplist-$RELEASE_VERSION/$item
+              zip -d phplist-${RELEASE_VERSION}.zip phplist-$RELEASE_VERSION/$item
+            }
+            [[ -d phplist-$RELEASE_VERSION/$item ]] && {
+              tar -vf phplist-${RELEASE_VERSION}.tar --delete phplist-$RELEASE_VERSION/$item
+              zip -d phplist-${RELEASE_VERSION}.zip phplist-$RELEASE_VERSION/$item/\*
+            }
+          done
+
+          gzip phplist-${RELEASE_VERSION}.tar
+          mv phplist-${RELEASE_VERSION}.tar.gz phplist-${RELEASE_VERSION}.tgz
+
+          md5sum phplist-${RELEASE_VERSION}.* > phplist-${RELEASE_VERSION}.md5
+          sha256sum phplist-${RELEASE_VERSION}.* > phplist-${RELEASE_VERSION}.sha256
+          sha1sum phplist-${RELEASE_VERSION}.* > phplist-${RELEASE_VERSION}.sha1
+          ls -l 
+          mv phplist-$RELEASE_VERSION phplist3 
+          cd $GITHUB_WORKSPACE
+
+      - name: Set up S3cmd cli tool
+        uses: s3-actions/[email protected]
+        with:
+          provider: aws
+          region: 'us-east-1'
+          access_key: ${{ secrets.AWS_KEY_ID }}
+          secret_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Upload the files
+        run: |
+          cd ..
+          s3cmd put phplist-${RELEASE_VERSION}.* s3://${{ secrets.AWS_S3_VERSIONS_BUCKET }}/
+          s3cmd put phplist3/public_html/lists/admin/images/power-phplist.png s3://${{ secrets.AWS_S3_POWERED_BUCKET }}/images/${RELEASE_VERSION}/

.github/workflows/main.yml

@@ -11,13 +11,11 @@ jobs:
     strategy:
       fail-fast: false     
       matrix:   
-        php-version: ['7.1', '7.2', '7.3', '7.4']
+        php-version: ['7.2', '7.3', '7.4']
         experimental: [false]
         include:
         - php-version: '8.0'
           experimental: true
-        - php-version: '7.0'
-          experimental: true
 
     steps:
       - uses: actions/checkout@v2
@@ -26,6 +24,8 @@ jobs:
         uses: shivammathur/setup-php@v2
         with:
           php-version: ${{ matrix.php-version }}
+        env:
+          COMPOSER_TOKEN: ${{ secrets.COMPOSER_TOKEN }}
 
       - name: Setup Packages
         run: |
@@ -38,11 +38,17 @@ jobs:
           path: /tmp/composer-cache
           key: ${{ matrix.php-version }}-${{ hashFiles('**/composer.lock') }}
 
+      - name: Add HTTP basic auth credentials
+        run: echo '${{ secrets.COMPOSER_AUTH_JSON }}' > $GITHUB_WORKSPACE/auth.json
+
       - name: Install dependencies
         uses: php-actions/composer@v5
         with:
           dev: yes
           args: --prefer-dist --no-interaction
+          php_version: 7.4
+          php_extensions: xml 
+          version: 2
 
       - name: Create Database
         run: |
@@ -51,13 +57,13 @@ jobs:
           sudo mysqladmin -proot create phplistdb
           sudo mysql -proot -e 'grant all on phplistdb.* to phplist@"%" identified by "phplist"'
 
-      - name: Set bootlist theme
-        run: |
-          cd $GITHUB_WORKSPACE/public_html/lists/admin/ui/
-          wget https://github.com/phpList/phplist-ui-bootlist/archive/master.tar.gz
-          tar -xzf master.tar.gz
-          mv phplist-ui-bootlist-master phplist-ui-bootlist
-          rm master.tar.gz
+      # - name: Set bootlist theme
+      #   run: |
+      #     cd $GITHUB_WORKSPACE/public_html/lists/admin/ui/
+      #     wget https://github.com/phpList/phplist-ui-bootlist/archive/master.tar.gz
+      #     tar -xzf master.tar.gz
+      #     mv phplist-ui-bootlist-master phplist-ui-bootlist
+      #     rm master.tar.gz
 
       - name: Start Test Server
         run: |
@@ -75,17 +81,16 @@ jobs:
         with:
           path: ./public_html
 
-      - name: Report Versions
-        run: |
-          google-chrome --version
-          php -v
-          chromedriver -v
-          geckodriver -V
-          which geckodriver
-          which chromedriver
-          firefox -v
-          vendor/bin/behat -V
-          cd vendor/bin/ && sudo ln -s /usr/bin/chromedriver && sudo ln -s /usr/bin/geckodriver
+      # - name: Report Versions
+      #   run: |
+      #     google-chrome --version
+      #     php -v
+      #     chromedriver -v
+      #     geckodriver -V
+      #     which geckodriver
+      #     which chromedriver
+      #     firefox -v
+      #     vendor/bin/behat -V
 
       - name: Run BDD Tests
         run: |

.github/workflows/s3upload-test.yml

@@ -0,0 +1,31 @@
+
+
+name: Upload to S3
+
+on:
+  push:
+    tags: testing-[1-9]+.[0-9]+.[0-9]+
+
+jobs:
+  upload:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@master
+      # - uses: shallwefootball/s3-upload-action@master
+      #   with:
+      #     aws_key_id: ${{ secrets.AWS_KEY_ID }}
+      #     aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+      #     aws_bucket: ${{ secrets.AWS_BUCKET }}
+      #     source_dir: 'doc'
+      #     destination_dir: '/test'
+      - name: Set up S3cmd cli tool
+        uses: s3-actions/[email protected]
+        with:
+          provider: aws # default is linode
+          region: 'us-east-1'
+          access_key: ${{ secrets.AWS_KEY_ID }}
+          secret_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Interact with object storage
+        run: |
+          s3cmd sync --recursive --acl-public doc s3://powered.phplist/test/
+          s3cmd info s3://powered.phplist

.gitignore

@@ -1,7 +1,6 @@
 /.svn/
 /plugins
 vendor/
-composer.lock
 /behat.yml
 .php_cs.cache
 /.idea/
@@ -11,3 +10,4 @@ public_html/lists/admin/info/
 /build
 .DS_Store
 .vagrant
+/updater

Dockerfile

@@ -0,0 +1,31 @@
+
+FROM debian:buster-slim
+
+LABEL maintainer="[email protected]" 
+
+RUN apt -y update && apt -y upgrade
+
+RUN apt install -y -qq postfix
+
+RUN apt install -y php-cli mariadb-server bash sudo composer git php-curl php-mysqli php-dom make firefox-esr wget
+## otherwise jdk fails, https://github.com/geerlingguy/ansible-role-java/issues/64
+RUN mkdir -p /usr/share/man/man1
+RUN apt install -y default-jdk
+
+RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && apt -y install ./google-chrome-stable_current_amd64.deb
+
+## debugging utils, that can be removed once it works
+RUN apt install -y vim curl telnet psutils
+
+RUN useradd -m -s /bin/bash -d /home/phplist phplist
+
+COPY . /var/www/phplist3
+RUN chown -R phplist: /var/www
+USER phplist
+WORKDIR /var/www/phplist3/
+RUN rm -rf vendor
+
+
+ENTRYPOINT [ "./scripts/run-tests.sh" ]
+
+#ENTRYPOINT [ "/bin/bash" ]