release_cli_and_assets.yml

name: Steampipe CLI Release

on:
  workflow_dispatch:
    inputs:
      version:
        description: "The version to release (must be prefixed with 'v')"
        required: true

env:
  PROJECT_ID: steampipe
  CORE_REPO: us-docker.pkg.dev/steampipe/steampipe
  ORG: turbot
  ASSET_IMAGE_NAME: assets
  CONFIG_SCHEMA_VERSION: "2020-11-18"
  VERSION: ${{ github.event.inputs.version }}
  STEAMPIPE_UPDATE_CHECK: false
  GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
  SPIPETOOLS_PG_CONN_STRING: ${{ secrets.SPIPETOOLS_PG_CONN_STRING }}
  SPIPETOOLS_TOKEN: ${{ secrets.SPIPETOOLS_TOKEN }}

jobs:

  verify_input:
    name: Verify Inputs
    runs-on: ubuntu-latest
    steps:
    - name: Trim asset version prefix and Validate
      run: |-
        echo $VERSION
        trim=${VERSION#"v"}
        echo $trim
        if [[ $trim =~  ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
          echo "Version OK: $trim"
        else
          echo "Invalid version: $trim"
          exit 1
        fi
        echo "VERSION=${trim}" >> $GITHUB_ENV
        
    - name: Validate Branch
      uses: actions/checkout@v4
      with:
        ref: ${{ github.event.ref }}

  create_test_build:
    name: Create Test Build
    runs-on: ubuntu-latest
    needs: [verify_input]
    steps:
    - name: Trim asset version prefix and Validate
      run: |-
        echo $VERSION
        trim=${VERSION#"v"}
        echo $trim
        if [[ $trim =~  ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
          echo "Version OK: $trim"
        else
          echo "Invalid version: $trim"
          exit 1
        fi
        echo "VERSION=${trim}" >> $GITHUB_ENV
        
    - name: Checkout
      uses: actions/checkout@v4
      with:
        path: steampipe
        ref: ${{ github.event.ref }}

    - name: Checkout Pipe Fittings Components repository
      uses: actions/checkout@v4
      with:
        repository: turbot/pipe-fittings
        path: pipe-fittings
        ref: v1.6.x

    - name: Set up Go
      uses: actions/setup-go@v5
      with:
        go-version: 1.22

    - name: Change directory to steampipe
      run: cd steampipe

    - name: Run CLI Unit Tests
      run: |
        cd steampipe
        go clean -testcache
        go test -timeout 30s ./... -test.v

    - name: Install GoReleaser
      uses: goreleaser/goreleaser-action@v6
      with:
        install-only: true

    - name: Run GoReleaser
      run: |
        cd steampipe
        goreleaser release --clean --snapshot --skip=publish
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

    - name: Move build artifacts
      run: |
        mkdir ~/artifacts
        mv $GITHUB_WORKSPACE/steampipe/dist/steampipe_linux_amd64.tar.gz ~/artifacts/linux.tar.gz
        mv $GITHUB_WORKSPACE/steampipe/dist/steampipe_linux_arm64.tar.gz ~/artifacts/linux-arm.tar.gz
        mv $GITHUB_WORKSPACE/steampipe/dist/steampipe_darwin_amd64.zip ~/artifacts/darwin.zip
        mv $GITHUB_WORKSPACE/steampipe/dist/steampipe_darwin_arm64.zip ~/artifacts/darwin-arm.zip

    - name: List Build Artifacts
      run: ls -l ~/artifacts

    - name: Save Linux Build Artifact
      uses: actions/upload-artifact@v4
      with:
        name: build-artifact-linux
        path: ~/artifacts/linux.tar.gz
        if-no-files-found: error
        overwrite: true

    - name: Save Linux ARM Build Artifact
      uses: actions/upload-artifact@v4
      with:
        name: build-artifact-linux-arm
        path: ~/artifacts/linux-arm.tar.gz
        if-no-files-found: error
        overwrite: true

    - name: Save MacOS Build Artifact
      uses: actions/upload-artifact@v4
      with:
        name: build-artifact-darwin
        path: ~/artifacts/darwin.zip
        if-no-files-found: error
        overwrite: true

    - name: Save MacOS ARM Build Artifact
      uses: actions/upload-artifact@v4
      with:
        name: build-artifact-darwin-arm
        path: ~/artifacts/darwin-arm.zip
        if-no-files-found: error
        overwrite: true

  run_acceptance_tests:
    name: Acceptance tests
    needs: [create_test_build]
    strategy:
      fail-fast: false
      matrix:
        platform: [ubuntu-latest]
        test_block:
          - "migration"
          - "brew"
          - "installation"
          - "plugin"
          - "service"
          - "settings"
          - "ssl"
          - "blank_aggregators"
          - "search_path"
          - "chaos_and_query"
          - "dynamic_schema"
          - "dynamic_aggregators"
          - "cache"
          - "performance"
          - "config_precedence"
          - "cloud"
          - "snapshot"
          - "schema_cloning"
          - "exit_codes"
          - "force_stop"
        exclude:
          - platform: ubuntu-latest
            test_block: chaos_and_query
    runs-on: ${{ matrix.platform }}
    steps:
    - name: Trim asset version prefix and Validate
      run: |-
        echo $VERSION
        trim=${VERSION#"v"}
        echo $trim
        if [[ $trim =~  ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
          echo "Version OK: $trim"
        else
          echo "Invalid version: $trim"
          exit 1
        fi
        echo "VERSION=${trim}" >> $GITHUB_ENV

    - name: Checkout
      uses: actions/checkout@v4
      with:
        submodules: true
        ref: ${{ github.event.ref }}

    - name: Set up Go
      uses: actions/setup-go@v5
      with:
        go-version: 1.22

    - name: Setup BATS
      uses: mig4/setup-bats@v1
      with:
        bats-version: 1.2.1

    - name: Prepare for downloads
      id: prepare-for-downloads
      run: |
        mkdir ~/artifacts

    - name: Download Linux Build Artifacts
      uses: actions/download-artifact@v4
      if: ${{ matrix.platform == 'ubuntu-latest' }}
      with:
        name: build-artifact-linux
        path: ~/artifacts

    - name: Download Darwin Build Artifacts
      uses: actions/download-artifact@v4
      if: ${{ matrix.platform == 'macos-13' }}
      with:
        name: build-artifact-darwin
        path: ~/artifacts

    - name: Extract Ubuntu Artifacts and Install Binary
      if: ${{ matrix.platform == 'ubuntu-latest' }}
      run: |
        mkdir ~/build
        tar -xf ~/artifacts/linux.tar.gz -C ~/build

    - name: Extract Darwin Artifacts and Install Binary
      if: ${{ matrix.platform == 'macos-13' }}
      run: |
        mkdir ~/build
        unzip ~/artifacts/darwin.zip -d ~/build

    - name: Set PATH
      run: |
        echo "PATH=$PATH:$HOME/build:$GTIHUB_WORKSPACE/tests/acceptance/lib/bats-core/libexec" >> $GITHUB_ENV

    - name: Go install jd
      run: |
        go install github.com/josephburnett/jd@latest

    - name: Install DB
      id: install-db
      continue-on-error: false
      run: |
        steampipe service start
        steampipe plugin install chaos chaosdynamic
        steampipe service stop

    - name: Run Test Suite
      id: run-test-suite
      timeout-minutes: 15
      run: |
        chmod +x $GITHUB_WORKSPACE/tests/acceptance/run.sh
        $GITHUB_WORKSPACE/tests/acceptance/run.sh ${{ matrix.test_block }}.bats
        steampipe service stop --force

  smoke_test_ubuntu_24:
    name: Smoke test (Ubuntu 24, x86_64)
    runs-on: ubuntu-latest
    needs: [create_test_build]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download Linux Build Artifact
        uses: actions/download-artifact@v4
        with:
          name: build-artifact-linux
          path: ./artifacts
  
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
  
      - name: Pull Ubuntu latest Image
        run: docker pull ubuntu:latest
  
      - name: Create and Start Ubuntu latest Container
        run: |
          docker run -d --name ubuntu-24-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts ubuntu:latest tail -f /dev/null

      - name: Get runner/container info
        run: |
          docker exec ubuntu-24-test /scripts/linux_container_info.sh
  
      - name: Install dependencies, create user, and assign necessary permissions
        run: |
          docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh

      - name: Run smoke tests
        run: |
          docker exec -u steampipe ubuntu-24-test /scripts/smoke_test.sh
  
      - name: Stop and Remove Container
        run: |
          docker stop ubuntu-24-test
          docker rm ubuntu-24-test

  smoke_test_centos_9:
    name: Smoke test (Centos stream 9, x86_64)
    runs-on: ubuntu-latest
    needs: [create_test_build]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
  
      - name: Download Linux Build Artifact
        uses: actions/download-artifact@v4
        with:
          name: build-artifact-linux
          path: ./artifacts
  
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
  
      - name: Pull CentOS Stream 9 image
        run: docker pull quay.io/centos/centos:stream9
  
      - name: Create and Start CentOS stream9 Container
        run: |
          docker run -d --name centos-stream9-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts quay.io/centos/centos:stream9 tail -f /dev/null

      - name: Get runner/container info
        run: |
          docker exec centos-stream9-test /scripts/linux_container_info.sh
  
      - name: Install dependencies, create user, and assign necessary permissions
        run: |
          docker exec centos-stream9-test /scripts/prepare_centos_container.sh
        
      - name: Run smoke tests
        run: |
          docker exec -u steampipe centos-stream9-test /scripts/smoke_test.sh
  
      - name: Stop and Remove Container
        run: |
          docker stop centos-stream9-test
          docker rm centos-stream9-test

  smoke_test_amazonlinux:
    name: Smoke test (Amazonlinux 2023, x86_64)
    runs-on: ubuntu-latest
    needs: [create_test_build]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download Linux Build Artifact
        uses: actions/download-artifact@v4
        with:
          name: build-artifact-linux
          path: ./artifacts
  
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
  
      - name: Pull Amazon Linux 2023 Image
        run: docker pull amazonlinux:2023
  
      - name: Create and Start Amazon Linux 2023 Container
        run: |
          docker run -d --name amazonlinux-2023-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts amazonlinux:2023 tail -f /dev/null

      - name: Get runner/container info
        run: |
          docker exec amazonlinux-2023-test /scripts/linux_container_info.sh
  
      - name: Install dependencies, create user, and assign necessary permissions
        run: |
          docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh

      - name: Run smoke tests
        run: |
          docker exec -u steampipe amazonlinux-2023-test /scripts/smoke_test.sh
  
      - name: Stop and Remove Container
        run: |
          docker stop amazonlinux-2023-test
          docker rm amazonlinux-2023-test

  # TODO (pskr) fix and enable this test
  # smoke_test_ubuntu_24_arm64:
  #   name: Smoke test (Ubuntu 24, ARM64)
  #   runs-on: ubuntu-latest
  #   needs: [create_test_build]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v4

  #     - name: Download Linux Build Artifact
  #       uses: actions/download-artifact@v4
  #       with:
  #         name: build-artifact-linux-arm
  #         path: ./artifacts
  
  #     - name: Set up Docker Buildx
  #       uses: docker/setup-buildx-action@v3
  
  #     - name: Set up QEMU
  #       uses: docker/setup-qemu-action@v3
  #       with:
  #         platforms: arm64
  
  #     - name: Create Buildx Builder
  #       run: |
  #         docker buildx create --use --name mybuilder
  #         docker buildx inspect --bootstrap
  
  #     - name: Pull Ubuntu 24 ARM64 Image
  #       run: docker pull arm64v8/ubuntu:latest
  
  #     - name: Create and Start Ubuntu 24 ARM64 Container
  #       run: |
  #         docker run -d --name ubuntu-24-arm64-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts arm64v8/ubuntu:latest tail -f /dev/null

  #     - name: Get runner/container info
  #       run: |
  #         docker exec ubuntu-24-arm64-test /scripts/linux_container_info.sh

  #     - name: Install dependencies, create user, and assign necessary permissions
  #       run: |
  #         docker exec ubuntu-24-arm64-test /scripts/prepare_ubuntu_arm_container.sh

  #     - name: Run smoke tests
  #       run: |
  #         docker exec -u steampipe ubuntu-24-arm64-test /scripts/smoke_test.sh
  
  #     - name: Stop and Remove Container
  #       run: |
  #         docker stop ubuntu-24-arm64-test
  #         docker rm ubuntu-24-arm64-test
   
  smoke_test_darwin_arm:
    name: Smoke test (MacOS 14, ARM64)
    runs-on: macos-latest
    needs: [create_test_build]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download Darwin Build Artifact
        uses: actions/download-artifact@v4
        with:
          name: build-artifact-darwin-arm
          path: ~/artifacts

      - name: Extract Darwin Artifacts and Install Binary
        run: |
          unzip ~/artifacts/darwin-arm.zip -d /usr/local/bin

      - name: Install jq
        run: |
          brew install jq

      - name: Get runner/container info
        run: |
          uname -a
          sw_vers

      - name: Run Smoke tests
        run: |
          ls -al $GITHUB_WORKSPACE/scripts
          chmod +x $GITHUB_WORKSPACE/scripts/smoke_test.sh
          $GITHUB_WORKSPACE/scripts/smoke_test.sh

  create_release_tag:
    name: Tag Release
    needs: [run_acceptance_tests, smoke_test_ubuntu_24, smoke_test_amazonlinux, smoke_test_centos_9, smoke_test_darwin_arm]
    runs-on: ubuntu-latest
    steps:
    - name: Trim asset version prefix and Validate
      run: |-
        echo $VERSION
        trim=${VERSION#"v"}
        echo $trim
        if [[ $trim =~  ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
          echo "Version OK: $trim"
        else
          echo "Invalid version: $trim"
          exit 1
        fi
        echo "VERSION=${trim}" >> $GITHUB_ENV
        
    - name: Checkout
      uses: actions/checkout@v4
      with:
        ref: ${{ github.event.ref }}

    - name: Unshallow
      run: git fetch --prune --unshallow

    - name: Tag Release
      run: |
        git config user.name "Steampipe GitHub Actions Bot"
        git config user.email noreply@github.com
        git tag ${{ github.event.inputs.version }}
        git push origin ${{ github.event.inputs.version }}

  ensure_branch_in_homebrew:
    name: Ensure branch exists in homebrew-tap
    needs: [create_release_tag]
    runs-on: ubuntu-latest
    steps:
      - name: Parse semver string
        id: semver_parser
        uses: booxmedialtd/ws-action-parse-semver@v1
        with:
          input_string: ${{ github.event.inputs.version }}

      - name: Checkout
        if: steps.semver_parser.outputs.prerelease == ''
        uses: actions/checkout@v4
        with:
          repository: turbot/homebrew-tap
          token: ${{ secrets.GH_ACCESS_TOKEN }}
          ref: main

      - name: Delete base branch if exists
        if: steps.semver_parser.outputs.prerelease == ''
        run: |
          git fetch --all
          git push origin --delete bump-brew
          git push origin --delete $VERSION
        continue-on-error: true

      - name: Create base branch
        if: steps.semver_parser.outputs.prerelease == ''
        run: |
          git checkout -b bump-brew
          git push --set-upstream origin bump-brew

  build_and_release_cli:
    name: Release CLI
    needs: [create_release_tag, ensure_branch_in_homebrew]
    runs-on: ubuntu-latest
    steps:
    - name: Trim asset version prefix and Validate
      run: |-
        echo $VERSION
        trim=${VERSION#"v"}
        echo $trim
        if [[ $trim =~  ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
          echo "Version OK: $trim"
        else
          echo "Invalid version: $trim"
          exit 1
        fi
        echo "VERSION=${trim}" >> $GITHUB_ENV
        
    - name: Checkout
      uses: actions/checkout@v4
      with:
        path: steampipe
        ref: ${{ github.event.inputs.version }}

    - name: Checkout Pipe Fittings Components repository
      uses: actions/checkout@v4
      with:
        repository: turbot/pipe-fittings
        path: pipe-fittings
        ref: v1.6.x

    - name: Set up Go
      uses: actions/setup-go@v5
      with:
        go-version: 1.22

    - name: Install GoReleaser
      uses: goreleaser/goreleaser-action@v6
      with:
        install-only: true

    - name: Run GoReleaser
      run: |
        cd steampipe
        goreleaser release --clean
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  create_pr_in_homebrew:
    name: Create PR in homebrew-tap
    needs: [ensure_branch_in_homebrew, build_and_release_cli]
    runs-on: ubuntu-latest
    env:
      Version: ${{ github.event.inputs.version }}
    steps:
      - name: Parse semver string
        id: semver_parser
        uses: booxmedialtd/ws-action-parse-semver@v1
        with:
          input_string: ${{ github.event.inputs.version }}

      - name: Checkout
        if: steps.semver_parser.outputs.prerelease == ''
        uses: actions/checkout@v4
        with:
          repository: turbot/homebrew-tap
          token: ${{ secrets.GH_ACCESS_TOKEN }}
          ref: main

      - name: Create a new branch off the base branch
        if: steps.semver_parser.outputs.prerelease == ''
        run: |
          git fetch --all
          git checkout bump-brew
          git checkout -b $VERSION
          git push --set-upstream origin $VERSION

      - name: Close pull request if already exists
        if: steps.semver_parser.outputs.prerelease == ''
        run: |
          gh pr close $VERSION
        continue-on-error: true

      - name: Create pull request
        if: steps.semver_parser.outputs.prerelease == ''
        run: |
          gh pr create --base main --head $VERSION --title "$Version" --body "Update formula"

  update_pr_for_versioning:
    name: Update PR
    needs: [create_pr_in_homebrew]
    runs-on: ubuntu-latest
    env:
      Version: ${{ github.event.inputs.version }}
    steps:
      - name: Parse semver string
        id: semver_parser
        uses: booxmedialtd/ws-action-parse-semver@v1
        with:
          input_string: ${{ github.event.inputs.version }}

      - name: Checkout
        if: steps.semver_parser.outputs.prerelease == ''
        uses: actions/checkout@v4
        with:
          repository: turbot/homebrew-tap
          token: ${{ secrets.GH_ACCESS_TOKEN }}
          ref: ${{ github.event.inputs.version }}

      - name: Update live version
        if: steps.semver_parser.outputs.prerelease == ''
        run: |
          scripts/formula_versioning.sh
          git config --global user.email "puskar@turbot.com"
          git config --global user.name "Puskar Basu"
          git add .
          git commit -m "Versioning brew formulas"
          git push origin $VERSION

  clean_up:
    # let's clean up the artifacts.
    # in case this step isn't reached,
    # artifacts automatically expire after 90 days anyway
    # refer:
    #   https://docs.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts#downloading-and-deleting-artifacts-after-a-workflow-run-is-complete
    name: Clean Up Artifacts
    needs: update_pr_for_versioning
    runs-on: ubuntu-latest
    steps:
    - name: Clean up Linux Build
      uses: geekyeggo/delete-artifact@v5
      with:
        name: build-artifact-linux

    - name: Clean up Darwin Build
      uses: geekyeggo/delete-artifact@v5
      with:
        name: build-artifact-darwin