benchmarks.yml

name: Collector Benchmarks

on:
  workflow_call:
    inputs:
      collector-tag:
        description: |
          Tag used for running the benchmarks
        type: string
        required: true
      collector-qa-tag:
        description: |
          Tag used for QA containers
        type: string
        required: true
      collector-tests-tag:
        description: |
          Tag used for the test container
        type: string
        required: true

jobs:
  benchmarks:
    uses: ./.github/workflows/integration-tests-vm-type.yml
    with:
      vm_type: benchmark
      run-benchmarks: true
      collector-tag: ${{ inputs.collector-tag }}
      collector-qa-tag: ${{ inputs.collector-qa-tag }}
      collector-tests-tag: ${{ inputs.collector-tests-tag }}
      collector-repo: quay.io/rhacs-eng/collector
    secrets: inherit

  calculate-baseline:
    runs-on: ubuntu-24.04
    needs:
      - benchmarks
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Authenticate with GCP
        uses: 'google-github-actions/auth@v2'
        with:
          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_CI_VM_SVC_ACCT }}'

      - uses: 'google-github-actions/setup-gcloud@v2'

      - uses: actions/download-artifact@v4
        with:
          name: benchmark-logs
          path: ${{ github.workspace }}

      - name: Install python deps
        run: python3 -m pip install -r ./integration-tests/scripts/baseline/requirements.txt

      - name: Compare with the Baseline
        run: |
          jq -s 'flatten' ./container-logs/*/*/perf.json > integration-tests/perf-all.json

          if [[ "${RUNNER_DEBUG}" == "1" ]]; then
              cat integration-tests/perf-all.json
          fi

          ./integration-tests/scripts/baseline/main.py --test integration-tests/perf-all.json \
            | sort \
            | awk -F "," -f ./integration-tests/scripts/baseline/format-cpu.awk > benchmark-cpu.md

          ./integration-tests/scripts/baseline/main.py --test integration-tests/perf-all.json \
            | sort \
            | awk -F "," -f ./integration-tests/scripts/baseline/format-mem.awk > benchmark-mem.md

          delimiter=$(openssl rand -hex 8)
          {
            echo "PERF_TABLE<<${delimiter}"
            cat benchmark-cpu.md
            echo "---"
            cat benchmark-mem.md
            echo "${delimiter}"
          } >> "$GITHUB_ENV"
        env:
          GOOGLE_CREDENTIALS_COLLECTOR_CI_VM_SVC_ACCT: "${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_CI_VM_SVC_ACCT }}"

      - name: Update Baseline
        if: |
          ((github.event_name != 'pull_request' && github.ref == 'master') ||
          contains(github.event.pull_request.labels.*.name, 'update-baseline'))
        run: |
          ./integration-tests/scripts/baseline/main.py --update integration-tests/perf-all.json
        env:
          GOOGLE_CREDENTIALS_COLLECTOR_CI_VM_SVC_ACCT: "${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_CI_VM_SVC_ACCT }}"

      - name: Comment on PR
        if: github.event_name == 'pull_request'
        uses: mshick/add-pr-comment@v2
        with:
          message: |
            ${{ env.PERF_TABLE }}
          message-id: performance-comment