Add benchmark for ci #70
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests | |
| on: | |
| push: | |
| branches: [ main ] | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| pull_request: | |
| types: [ opened, synchronize, reopened, edited, labeled ] | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| jobs: | |
| debug_tests: | |
| name: debug_tests | |
| # https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution | |
| # https://github.com/orgs/community/discussions/26261 | |
| if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| # https://github.com/hmarr/debug-action | |
| #- uses: hmarr/debug-action@v2 | |
| - name: Show PR labels | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "Workflow triggered by ${{ github.event_name }}" | |
| if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
| echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
| fi | |
| - name: Check out code | |
| if: ${{ !cancelled() && !failure() }} | |
| uses: actions/checkout@v3 | |
| with: | |
| ssh-key: ${{ secrets.MY_DEPLOY_KEY }} | |
| - name: Start builder container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| TZ=$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}') | |
| sudo docker rm -f infinity_build && sudo docker run -d --privileged --name infinity_build --network=host -e TZ=$TZ -v $PWD:/infinity infiniflow/infinity_builder:ubuntu2310 | |
| - name: Build debug version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec infinity_build bash -c "git config --global --add safe.directory /infinity && cd /infinity && rm -fr cmake-build-debug && mkdir -p cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_JOB_POOL_LINK:STRING=link_pool -DCMAKE_JOB_POOLS:STRING=link_pool=1 -S /infinity -B /infinity/cmake-build-debug && cmake --build /infinity/cmake-build-debug" | |
| - name: Unit test debug version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec infinity_build bash -c "cd /infinity/ && cmake-build-debug/src/test_main" | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec infinity_build bash -c "cd /infinity/python && python3 setup.py install" | |
| - name: Start infinity debug version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| # Run a command in the background | |
| sudo docker exec infinity_build bash -c "cd /infinity/ && rm -fr /tmp/infinity && cmake-build-debug/src/infinity > debug.log 2>&1" & | |
| - name: pysdk & sqllogictest debug version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec infinity_build bash -c "cd /infinity/ && python3 tools/sqllogictest.py" | |
| - name: Stop infinity debug | |
| if: ${{ !cancelled() }} # always run this step even if previous steps failed | |
| run: sudo kill $(pidof cmake-build-debug/src/infinity) | |
| - name: Collect infinity debug output | |
| # GitHub Actions interprets output lines starting with "Error" as error messages, and it automatically sets the step status to failed when such lines are detected. | |
| if: ${{ !cancelled() }} # always run this step even if previous steps failed | |
| run: cat debug.log 2>/dev/null || true | |
| release_tests: | |
| name: release_tests | |
| # https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution | |
| # https://github.com/orgs/community/discussions/26261 | |
| if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| # https://github.com/hmarr/debug-action | |
| #- uses: hmarr/debug-action@v2 | |
| - name: Show PR labels | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "Workflow triggered by ${{ github.event_name }}" | |
| if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
| echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
| fi | |
| - name: Check out code | |
| if: ${{ !cancelled() && !failure() }} | |
| uses: actions/checkout@v3 | |
| with: | |
| ssh-key: ${{ secrets.MY_DEPLOY_KEY }} | |
| - name: Start builder container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| TZ=$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}') | |
| sudo docker rm -f infinity_build && sudo docker run -d --privileged --name infinity_build --network=host -e TZ=$TZ -v $PWD:/infinity infiniflow/infinity_builder:ubuntu2310 | |
| - name: Build release version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec infinity_build bash -c "git config --global --add safe.directory /infinity && cd /infinity && rm -fr cmake-build-release && mkdir -p cmake-build-release && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_JOB_POOL_LINK:STRING=link_pool -DCMAKE_JOB_POOLS:STRING=link_pool=1 -S /infinity -B /infinity/cmake-build-release && cmake --build /infinity/cmake-build-release" | |
| - name: Unit test release version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec infinity_build bash -c "cd /infinity/ && cmake-build-release/src/test_main" | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec infinity_build bash -c "cd /infinity/python && python3 setup.py install" | |
| - name: Start infinity debug version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| # Run a command in the background | |
| sudo docker exec infinity_build bash -c "cd /infinity/ && rm -fr /tmp/infinity && cmake-build-debug/src/infinity > debug.log 2>&1" & | |
| - name: pysdk & sqllogictest debug version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec infinity_build bash -c "cd /infinity/ && python3 tools/sqllogictest.py" | |
| - name: Stop infinity release | |
| if: ${{ !cancelled() }} # always run this step even if previous steps failed | |
| run: sudo kill $(pidof cmake-build-release/src/infinity) | |
| - name: Collect infinity release output | |
| # GitHub Actions interprets output lines starting with "Error" as error messages, and it automatically sets the step status to failed when such lines are detected. | |
| if: ${{ !cancelled() }} # always run this step even if previous steps failed | |
| run: cat release.log 2>/dev/null || true | |
| benchmark: | |
| name: benchmark | |
| if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'benchmark') }} | |
| runs-on: [ "self-hosted", "benchmark" ] | |
| steps: | |
| - name: Show PR labels | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "Workflow triggered by ${{ github.event_name }}" | |
| if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
| echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
| fi | |
| - name: Ensure workspace ownership | |
| if: ${{ !cancelled() && !failure() }} | |
| run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
| - name: Check out code | |
| if: ${{ !cancelled() && !failure() }} | |
| uses: actions/checkout@v3 | |
| with: | |
| ssh-key: ${{ secrets.MY_DEPLOY_KEY }} | |
| - name: Start builder container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| TZ=$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}') | |
| sudo docker rm -f infinity_build && sudo docker run -d --privileged --name infinity_build --network=host -e TZ=$TZ -v $PWD:/infinity infiniflow/infinity_builder:ubuntu2310 | |
| - name: Build release version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec infinity_build bash -c "cd /infinity && rm -fr cmake-build-release && mkdir -p cmake-build-release && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_JOB_POOL_LINK:STRING=link_pool -DCMAKE_JOB_POOLS:STRING=link_pool=1 -S /infinity -B /infinity/cmake-build-release && cmake --build /infinity/cmake-build-release --target knn_import_benchmark knn_query_benchmark" | |
| - name: Prepare sift dataset | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo chmod +x ./tools/ci_tools/check_benchmark_result.py && sudo mkdir -p test/data/benchmark && sudo ln -s $HOME/benchmark/sift1M test/data/benchmark/sift_1m | |
| - name: Generate config file | |
| if: ${{ !cancelled() && !failure() }} | |
| run: mkdir -p $PWD/db_tmp && cat conf/infinity_conf.toml | sed -e "s|/var/infinity|$PWD/db_tmp|g" > $PWD/db_tmp/infinity_conf.toml | |
| - name: Import sift dataset | |
| if: ${{ !cancelled() && !failure() }} | |
| run: ./cmake-build-release/benchmark/local_infinity/knn_import_benchmark sift $PWD/test/data $PWD/db_tmp | |
| - name: Benchmark sift search 1 thread repeat 10000 times | |
| if: ${{ !cancelled() && !failure() }} | |
| run: echo $(date) "Benchmark sift search 1 thread average time:" $(echo "1 10000" | ./cmake-build-release/benchmark/local_infinity/knn_query_benchmark sift 200 $PWD/test/data $PWD/db_tmp | awk '/Total cost/ {total+=$4; count+=1} END {print total/count}') >> $HOME/benchmark/benchmark_sift_1_thread.log | |
| - name: Benchmark sift search 4 threads repeat 10000 times | |
| if: ${{ !cancelled() && !failure() }} | |
| run: echo $(date) "Benchmark sift search 4 threads average time:" $(echo "4 10000" | ./cmake-build-release/benchmark/local_infinity/knn_query_benchmark sift 200 $PWD/test/data $PWD/db_tmp | awk '/Total cost/ {total+=$4; count+=1} END {print total/count}') >> $HOME/benchmark/benchmark_sift_4_threads.log | |
| - name: Benchmark sift check result | |
| if: ${{ !cancelled() && !failure() }} | |
| run: ./tools/ci_tools/check_benchmark_result.py sift_1 $HOME/benchmark/benchmark_sift_1_thread.log | |
| - name: Benchmark sift check result | |
| if: ${{ !cancelled() }} | |
| run: ./tools/ci_tools/check_benchmark_result.py sift_4 $HOME/benchmark/benchmark_sift_4_threads.log |