diff --git a/.github/scripts/setup_instance.sh b/.github/scripts/setup_instance.sh new file mode 100644 index 000000000..923902715 --- /dev/null +++ b/.github/scripts/setup_instance.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +set -euxo pipefail + +DISKS="gcloud compute disks" +IMAGES="gcloud compute images" +INSTANCES="gcloud compute instances" +VMX="https://compute.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" +DEFAULT_ZONE=us-central1-a + +function createDisk() { + # Create disk + # + # $1: disk name + # $2: image project + # $3: image-family + # $4: zone + zone=${4:-$DEFAULT_ZONE} + + diskName=$1 + ${DISKS} create $diskName \ + --image-project $2 \ + --image-family $3 \ + --size 20GB \ + --zone $zone +} + +function createImage() { + # Create image with nested virtualization + # + # $1: image name + # $2: disk name + # $3: zone + zone=${3:-$DEFAULT_ZONE} + + imageName=$1 + ${IMAGES} create $imageName \ + --source-disk $2 \ + --source-disk-zone $zone \ + --licenses "${VMX}" +} + +function createVM() { + # Create VM with nested virtualization + # + # $1: VM name + # $2: image name + # $3: zone + zone=${3:-$DEFAULT_ZONE} + + instanceName=$1 + ${INSTANCES} create $1 \ + --zone $zone \ + --min-cpu-platform "Intel Haswell" \ + --image $2 +} + +# Entrypoint +# +# $1: vm name +# $2: image project +# $3: image family +# $4: zone +vmName=$1 +imageProject=$2 +imageFamily=$3 +zone=${4:-$DEFAULT_ZONE} + +# create disk +diskName=$vmName-disk +createDisk $vmName-disk $imageProject $imageFamily $zone + +# create image +imageName=$vmName-image +createImage $imageName $diskName $zone + +# create vm +createVM $1 $imageName $zone diff --git a/.github/scripts/setup_kimchi_ubuntu.sh b/.github/scripts/setup_kimchi_ubuntu.sh new file mode 100644 index 000000000..8998a87c4 --- /dev/null +++ b/.github/scripts/setup_kimchi_ubuntu.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euxo pipefail + +function get_deps() { + echo $(python3 -c "import yaml;print(' '.join(yaml.load(open('dependencies.yaml'), Loader=yaml.FullLoader)[\"$1\"][\"$2\"]))") +} + + +# install deps +sudo apt update +sudo apt install -y python3-pip +sudo apt install -y $(get_deps development-deps common) +sudo apt install -y $(get_deps development-deps ubuntu) + +sudo apt install -y $(get_deps runtime-deps common) +sudo apt install -y $(get_deps runtime-deps ubuntu | sed 's/python3-cheetah//') + +pip3 install -r requirements-UBUNTU.txt +pip3 install -r requirements-dev.txt + +# autogen and make +./autogen.sh --system +make diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 000000000..29ce916bc --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,55 @@ +name: kimchi-CI +on: ["push"] + +env: + VM_NAME: ubuntu-${{ github.sha }} + ZONE: us-central1-a + PROJECT_ID: ${{ secrets.GCP_PROJECT }} + WOK_DIR: wok + KIMCHI_DIR: wok/src/wok/plugins/kimchi/ + +jobs: + run-kimchi-tests: + runs-on: ubuntu-latest + steps: + + - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master + with: + version: '270.0.0' + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + - run: gcloud config set project $PROJECT_ID + + - uses: actions/checkout@v2-beta + + - name: create instance + run: bash .github/scripts/setup_instance.sh $VM_NAME ubuntu-os-cloud ubuntu-1804-lts + shell: bash + + - name: checkout repos + run: | + gcloud compute ssh $VM_NAME --zone=$ZONE --command "git clone https://github.com/kimchi-project/wok" + gcloud compute ssh $VM_NAME --zone=$ZONE --command "git clone https://github.com/$GITHUB_REPOSITORY $KIMCHI_DIR" + gcloud compute ssh $VM_NAME --zone=$ZONE --command "cd $KIMCHI_DIR; git checkout $GITHUB_SHA" + shell: bash + + - name: setup wok deps + run: | + gcloud compute ssh $VM_NAME --zone=$ZONE --command "cd $WOK_DIR; bash .github/scripts/setup_wok_ubuntu.sh" + shell: bash + + - name: setup kimchi deps + run: gcloud compute ssh $VM_NAME --zone=$ZONE --command "cd $KIMCHI_DIR; bash .github/scripts/setup_kimchi_ubuntu.sh" + shell: bash + + - name: run tests + run: gcloud compute ssh $VM_NAME --zone=$ZONE --command "cd $KIMCHI_DIR; sudo make check-local; sudo make check" + shell: bash + + - name: Cleanup instance + if: always() + run: | + gcloud compute instances delete ${VM_NAME} --delete-disks=all --zone=$ZONE -q || true + gcloud compute images delete ${VM_NAME}-image -q || true + gcloud compute disks delete ${VM_NAME}-disk --zone=$ZONE -q || true + shell: bash diff --git a/docs/CI.md b/docs/CI.md new file mode 100644 index 000000000..49dc375a2 --- /dev/null +++ b/docs/CI.md @@ -0,0 +1,38 @@ +# How to setup Kimchi CI + +Due to nested virtualization requirements, most of CI tools does not offer the minimum requirement to test kimchi. To solve this, we have an integration with [Google Cloud](https://cloud.google.com/), which create an instance and trigger the tests. + +# Create a service account in GCP + +To run the tests, its necessary to create a service account with the following iam roles: + +* `roles/iam.serviceAccountUser` +* `roles/compute.admin` + +Here is a script to help +``` +# create account +$ gcloud iam service-accounts create [SA-NAME] \ + --description "[SA-DESCRIPTION]" \ + --display-name "[SA-DISPLAY-NAME]" + +# create the key to be store as a secret +$ gcloud iam service-accounts keys create ~/key.json \ + --iam-account [SA-NAME]@[PROJECT-ID].iam.gserviceaccount.com + +# add iam roles +$ gcloud projects add-iam-policy-binding [PROJECT-ID] --member=serviceAccount:[SA-EMAIL] --role=roles/iam.serviceAccountUser + +$ gcloud projects add-iam-policy-binding [PROJECT-ID] --member=serviceAccount:[SA-EMAIL] --role=roles/compute.admin +``` + +# Setting up secrets + +The code for the CI is already at our repo, you just need to set some Github Actions Secrets, the procedure is described here: https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets#creating-encrypted-secrets + +1) Create a secret named `GCP_PROJECT` with the project ID +2) Create a secret named `GCP_SA_EMAIL` with the service account email +3) Create a secret named `GCP_SA_KEY` with the service account json with base64: `cat my-key.json | base64` + +# Testing the CI +Create a PR to see if the PR works \ No newline at end of file