Merge pull request #70 from diegoferigo/feature/bdist

Continuous Delivery of PyPI manylinux1 Wheel
robotology-legacy · Oct 10, 2019 · b4e4a2c · b4e4a2c
2 parents 6f3441f + fa46551
commit b4e4a2c
Show file tree

Hide file tree

Showing 8 changed files with 241 additions and 42 deletions.
diff --git a/.ci/Dockerfile → .docker/Dockerfile.ci b/.ci/Dockerfile → .docker/Dockerfile.ci
@@ -23,40 +23,19 @@ RUN apt-get update &&\
         clang-8 \
         make \
         cmake \
+        cmake-curses-gui \
         ninja-build \
         valgrind \
-        ccache \
         libgflags-dev \
         python3-pip \
         python3.6 \
         python3.7 \
         libpython3.6-dev \
         libpython3.7-dev \
+        swig \
         &&\
     rm -rf /var/lib/apt/lists/*
 
-# Enable CCACHE by default
-ENV PATH=/usr/lib/ccache:${PATH}
-RUN mkdir "$HOME/.ccache" &&\
-    echo "max_size = 2.0G" > "$HOME/.ccache/ccache.conf"
-
-# Install swig
-RUN apt-get update &&\
-    apt-get install -y --no-install-recommends \
-        autotools-dev \
-        automake \
-        bison \
-        libpcre3-dev &&\
-    rm -rf /var/lib/apt/lists/* &&\
-    cd /tmp/ &&\
-    git clone --depth 1 -b rel-4.0.0 https://github.com/swig/swig.git &&\
-    cd swig &&\
-    sh autogen.sh &&\
-    ./configure &&\
-    make -j8 &&\
-    make install &&\
-    rm -r /tmp/swig
-
 # Install ignition robotics
 RUN echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable `lsb_release -cs` main" \
         > /etc/apt/sources.list.d/gazebo-stable.list &&\
@@ -114,7 +93,7 @@ RUN apt-get update &&\
 ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
 ENV PYTHONPATH=${PYTHONPATH}:/usr/local/lib/python3/dist-packages
 
-# Setup gym-ignition
+# Setup gym-ignition if installed from source
 ARG install_prefix="/usr/local"
 ENV IGN_GAZEBO_SYSTEM_PLUGIN_PATH="${install_prefix}/lib/gympp/plugins"
 ENV IGN_GAZEBO_RESOURCE_PATH="${install_prefix}/share/gympp/gazebo/worlds:${install_prefix}/share/gympp/gazebo/models"
@@ -125,6 +104,11 @@ ENV PYTHONPATH="${PYTHONPATH}:${install_prefix}/lib/python3.6/site-packages:${in
 ENV VIRTUAL_ENV=/venv
 ENV PATH=${VIRTUAL_ENV}/bin:${PATH}
 
+# Setup gym-ignition if installed from pip
+ARG site_pkgs="${VIRTUAL_ENV}/lib/python3.6/site-packages"
+ENV IGN_GAZEBO_SYSTEM_PLUGIN_PATH="${IGN_GAZEBO_SYSTEM_PLUGIN_PATH}:${site_pkgs}/gym_ignition/plugins"
+ENV IGN_GAZEBO_RESOURCE_PATH="${IGN_GAZEBO_RESOURCE_PATH}:${site_pkgs}/gym_ignition_data:${site_pkgs}/gym_ignition_data/worlds"
+
 COPY entrypoint.sh /entrypoint.sh
 COPY setup_virtualenv.sh /setup_virtualenv.sh
 ENTRYPOINT ["/entrypoint.sh"]

diff --git a/.docker/Dockerfile.pypi b/.docker/Dockerfile.pypi
@@ -0,0 +1,100 @@
+ARG from=ubuntu:bionic
+FROM ${from}
+
+# Install tools and toolchain
+RUN apt-get update &&\
+    apt-get install -y --no-install-recommends \
+        wget \
+        software-properties-common \
+        apt-transport-https \
+        apt-utils \
+        gnupg2 \
+        nano \
+        &&\
+    wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \
+    apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' &&\
+    apt-get update &&\
+    apt-get install -y --no-install-recommends \
+        git \
+        g++ \
+        g++-8 \
+        make \
+        cmake \
+        cmake-curses-gui \
+        ninja-build \
+        libgflags-dev \
+        python3-pip \
+        python3.6 \
+        libpython3.6-dev \
+        swig \
+        rename \
+        &&\
+    rm -rf /var/lib/apt/lists/*
+
+# Install ignition robotics
+RUN echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable `lsb_release -cs` main" \
+        > /etc/apt/sources.list.d/gazebo-stable.list &&\
+    echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-prerelease `lsb_release -cs` main" \
+        > /etc/apt/sources.list.d/gazebo-prerelease.list &&\
+    echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-nightly `lsb_release -cs` main" \
+        > /etc/apt/sources.list.d/gazebo-prerelease.list &&\
+    wget http://packages.osrfoundation.org/gazebo.key -O - | apt-key add - &&\
+    apt-get update &&\
+    apt-get install -y --no-install-recommends \
+        libignition-cmake2-dev \
+        libignition-plugin-dev \
+        libignition-math6-dev \
+        libignition-common3-dev \
+        libignition-transport7-dev \
+        libignition-msgs4-dev \
+        libignition-tools-dev \
+        libignition-fuel-tools3-dev \
+        libsdformat8-dev \
+        libignition-physics-dev \
+        libignition-rendering-dev \
+        libignition-sensors2-dev \
+        libignition-gui2-dev \
+        &&\
+    rm -rf /var/lib/apt/lists/*
+
+# Install ign-gazebo from my fork (statically compiled)
+RUN cd /tmp &&\
+    git clone --depth 1 https://github.com/diegoferigo/ign-gazebo  &&\
+    cd ign-gazebo && mkdir -p build && cd build &&\
+    cmake \
+        -DCMAKE_BUILD_TYPE="Release" \
+        -DBUILD_TESTING:BOOL=OFF \
+        -DCMAKE_INSTALL_PREFIX:PATH="/usr" \
+        -DBUILD_SHARED_LIBS:BOOL=OFF \
+        .. &&\
+    cmake --build . --target install -- -j8 &&\
+    rm -r /tmp/ign-gazebo
+
+# Install idyntree (statically compiled)
+RUN apt-get update &&\
+    apt-get install -y --no-install-recommends python3-numpy libxml2-dev coinor-libipopt-dev &&\
+    rm -rf /var/lib/apt/lists/* &&\
+    git clone --depth 1 -b feature/numpy-and-ik https://github.com/diegoferigo/idyntree /tmp/idyntree &&\
+    mkdir -p /tmp/idyntree/build && cd /tmp/idyntree/build &&\
+    cmake .. \
+        -GNinja \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DIDYNTREE_USES_PYTHON=True \
+        -DPython_ADDITIONAL_VERSIONS=3.6 \
+        -DIDYNTREE_USES_PYTHON_VERSION=3.6 \
+        -DPYTHON_NUMPY_INCLUDE_DIR:PATH=/usr/lib/python3/dist-packages/numpy/core/include \
+        -DBUILD_SHARED_LIBS:BOOL=OFF \
+        &&\
+    cmake --build . --target install &&\
+    rm -r /tmp/idyntree
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
+ENV PYTHONPATH=${PYTHONPATH}:/usr/local/lib/python3/dist-packages
+
+# Prepare virtualenv variables
+ENV VIRTUAL_ENV=/venv
+ENV PATH=${VIRTUAL_ENV}/bin:${PATH}
+
+COPY entrypoint.sh /entrypoint.sh
+COPY setup_virtualenv.sh /setup_virtualenv.sh
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["bash"]
diff --git a/.ci/entrypoint.sh → .docker/entrypoint.sh b/.ci/entrypoint.sh → .docker/entrypoint.sh
diff --git a/.ci/setup_virtualenv.sh → .docker/setup_virtualenv.sh b/.ci/setup_virtualenv.sh → .docker/setup_virtualenv.sh
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -3,34 +3,37 @@ name: 'CI Docker Image'
 on:
   push:
     paths:
-      - ".ci/*"
+      - ".docker/*"
       - ".github/workflows/docker.yml"
   pull_request:
     paths:
-      - ".ci/*"
+      - ".docker/*"
       - ".github/workflows/docker.yml"
   schedule:
     # Execute a weekly build on Monday at 2AM UTC
     - cron:  '0 2 * * 1'
 
 jobs:
-
   docker:
-    name: 'Image diegoferigo/gym-ignition:ci'
+    name: 'diegoferigo/gym-ignition:${{ matrix.tag }}'
     runs-on: ubuntu-latest
     strategy:
       matrix:
         baseimage: ['ubuntu:bionic']
+        tag:
+          - ci
+          - pypi
 
     steps:
       - uses: actions/checkout@master
 
       - name: 'Build'
         run: |
-          cd .ci
-          docker build \
+          cd .docker/
+          docker build . \
             --pull --build-arg from=${{ matrix.baseimage }} \
-            --rm -t diegoferigo/gym-ignition:ci .
+            --rm -t diegoferigo/gym-ignition:${{ matrix.tag }} \
+            -f Dockerfile.${{ matrix.tag }}
 
       - name: Login
         if: |
@@ -47,4 +50,4 @@ jobs:
           github.repository == 'robotology/gym-ignition' &&
           github.event_name != 'pull_request' &&
           github.ref == 'refs/heads/master'
-        run: docker push diegoferigo/gym-ignition:ci
+        run: docker push diegoferigo/gym-ignition:${{ matrix.tag }}
diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml
@@ -22,6 +22,8 @@ jobs:
         python_version:
           - 3.6
     container:
+      # The sdist is tested using the :ci image tag that should mostly match the developer
+      # setup with ignition robotics installed from the official ppa
       image: diegoferigo/gym-ignition:ci
       env:
         CC: gcc
@@ -76,3 +78,107 @@ jobs:
         env:
           TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+
+  # ===================
+  # BDIST_WHEEL PACKAGE
+  # ===================
+
+  # The bdist is created using the :pypi image and tested in the ubuntu:bionic
+  # host that should mostly match the user setup
+  bdist:
+    name: bdist_wheel
+    needs: sdist
+    strategy:
+      matrix:
+        os:
+          - ubuntu-18.04
+        python_version:
+          - 3.6
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@master
+
+      # Configure the host ubuntu system. It will be used as clean system for the
+      - name: Setup python
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python_version }}
+          architecture: x64
+
+      # Validate the last tag accordingly to PEP440
+      # From https://stackoverflow.com/a/37972030/12150968
+      - name: Validate Tag for PEP440 compliance
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y source-highlight
+          last_tag="$(git describe --abbrev=0 --tags)"
+          rel_regexp='^(\d+!)?(\d+)(\.\d+)+([\.\-\_])?((a(lpha)?|b(eta)?|c|r(c|ev)?|pre(view)?)\d*)?(\.?(post|dev)\d*)?$'
+          echo ""
+          echo $last_tag
+          echo ""
+          check-regexp ${rel_regexp} ${last_tag}
+          match=$(check-regexp ${rel_regexp} ${last_tag} | grep matches | cut -d ' ' -f 5)
+          test $match -eq 1 && true
+
+      # Workaround to export environment variables that persist in next steps
+      # https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+      - name: Setup Environment
+        run: |
+          echo "::set-env name=CC::gcc"
+          echo "::set-env name=CXX::g++"
+          echo "::set-env name=PYTHON_VERSION::${{ matrix.python }}"
+          env
+
+      - name: Download diegoferigo/gym-ignition:pypi
+        run: 'docker pull diegoferigo/gym-ignition:pypi'
+
+      - name: Create container
+        run: |
+          docker run \
+            -d -i --rm --name pypi -v $(pwd):/github -w /github \
+            -e PYTHON_VERSION=$PYTHON_VERSION -e CC=$CC -e CXX=$CXX \
+            diegoferigo/gym-ignition:pypi bash
+          sleep 15
+
+      - name: Create package
+        run: docker exec -i pypi sh -c 'python setup.py bdist_wheel'
+
+      - name: Rename wheel
+        run: |
+          docker exec -i -w /github/dist pypi \
+            sh -c 'find -type f -name "*.whl" -exec rename.ul linux manylinux1 {} +'
+
+      - name: Install twine
+        run: pip install twine
+
+      - name: Upload it to Test PiPY
+        run: twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*
+        env:
+          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME_TESTPYPI }}
+          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TESTPYPI }}
+
+      - name: Download and start clean system
+        run : |
+          docker pull diegoferigo/gym-ignition:ci
+          docker run \
+            -d -i --name test -e PYTHON_VERSION=$PYTHON_VERSION \
+            diegoferigo/gym-ignition:ci bash
+          sleep 15
+
+      - name: Install and test
+        run: |
+          docker exec -i -e RELEASE_TAG=$(git describe --abbrev=0 --tags) test sh -c \
+            'pip install \
+              --index-url https://test.pypi.org/simple/ \
+              --extra-index-url https://pypi.org/simple \
+              gym-ignition==$RELEASE_TAG'
+          docker exec -i test sh -c "git clone https://github.com/robotology/gym-ignition"
+          docker exec -i test sh -c "cd gym-ignition/tests/python && pytest"
+
+      - name: Push to PyPI
+        if: github.event.action == 'published'
+        run: twine upload --verbose dist/*
+        env:
+          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -106,6 +106,10 @@ add_subdirectory(deps)
 # Add the C++ sources subdirectory
 add_subdirectory(gympp)
 
+# This target is not found automatically when searching for 'all' components and
+# ign-gazebo is statically compiled. Adding this find_package here solves the problem.
+find_package(ignition-gazebo2-gui)
+
 find_package(ignition-gazebo2 COMPONENTS all)
 if(ignition-gazebo2_FOUND)
     add_subdirectory(ignition)

diff --git a/README.md b/README.md
@@ -47,6 +47,12 @@
     </a>
 </p>
 
+<p align="center">
+    <a href="https://colab.research.google.com/github/robotology/gym-ignition/blob/master/examples/colab/RandomPolicy.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
+    </a>
+</p>
+
 <p align="center">⚠️ Warning, <a href="https://en.wikipedia.org/wiki/Here_be_dragons">Here be Dragons</a> ⚠️</p>
 <p align="center">You just reached a very unstable corner.<br/>Welcome, but mind your step. </p>
 
@@ -110,19 +116,15 @@ At the time of writing, Gym-Ignition offers the following features:
 
 ## Setup
 
-You can start exploring the project opening the [RandomPolicy.ipynb](examples/colab/RandomPolicy.ipynb).
-
-⚠️ The local installation of Gym-Ignition, given the early status of development, is still not very user-friendly. We are actively working on it for improving the user experience.
+The setup instructions expect a **Ubuntu** distribution with at least **Python 3.6**.
 
-Follow these initial common steps, and then select the type of installation that you prefer.
+The process is different whether you're an _user_ that wants to create environments using Gym-Ignition or you are a _developer_ that wants to edit the Python and C++ code. Common steps:
 
-1. Install all the Ignition Robotics suit excluding `ign-gazebo` following the [official documentation](https://ignitionrobotics.org/docs/latest/install).
-1. Install `ign-gazebo` from our [temporary fork](https://github.com/diegoferigo/ign-gazebo).
-1. Install SWIG with `apt install swig`.
-1. Update CMake if you have a version < 3.12 from the [official ppa](https://apt.kitware.com/).
+1. Install [SWIG](https://github.com/swig/swig) with `apt install swig`
 
 ### Are you a user?
 
+1. Install the Ignition Robotics suite following the [official documentation](https://ignitionrobotics.org/docs/latest/install)
 1. Install Gym-Ignition with `pip install gym-ignition` (preferably in a [virtual environment](https://docs.python.org/3.6/tutorial/venv.html))
 1. Execute the following to export the required environment variables:
    ```sh
@@ -139,8 +141,8 @@ After these steps, you should be able to execute the example [`launch_cartpole.p
 
 ### Are you a developer?
 
-Developers should install separately the C++ and Python components.
-
+1. Install all the Ignition Robotics suite except `ignition-gazebo2` following the [official documentation](https://ignitionrobotics.org/docs/latest/install)
+1. Install `ign-gazebo` from our [temporary fork](https://github.com/diegoferigo/ign-gazebo)
 1. Clone this repository
 1. Build and install the CMake project
    ```sh