From 28ac4e98708d01de46868ed41bbd39dfb86db00d Mon Sep 17 00:00:00 2001 From: ourownstory Date: Wed, 18 Oct 2023 15:05:02 -0700 Subject: [PATCH 01/12] update metrics ci with new tests and move to poetry --- .github/workflows/metrics.yml | 22 +++++++++++++++------- pyproject.toml | 4 ++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 5510a13c7..7d1f65230 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -16,16 +16,18 @@ jobs: runs-on: ubuntu-latest container: docker://ghcr.io/iterative/cml:0-dvc2-base1 steps: - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Install Python "3.10" + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install Poetry + uses: snok/install-poetry@v1 - name: Install dependencies - run: | - python3 -m pip install --upgrade pip - pip3 install -e . - pip3 install pytest - pip3 install tabulate # required to md export - pip3 install kaleido # required for plotly export + run: poetry install - name: Train model run: | pytest tests/test_model_performance.py @@ -49,10 +51,16 @@ jobs: echo "## Model Training" >> report.md echo "### PeytonManning" >> report.md cml asset publish tests/metrics/PeytonManning.svg --md >> report.md + echo "### PeytonManning_test30" >> report.md + cml asset publish tests/metrics/PeytonManning_test30.svg --md >> report.md echo "### YosemiteTemps" >> report.md cml asset publish tests/metrics/YosemiteTemps.svg --md >> report.md + echo "### YosemiteTemps_test20" >> report.md + cml asset publish tests/metrics/YosemiteTemps_test20.svg --md >> report.md echo "### AirPassengers" >> report.md cml asset publish tests/metrics/AirPassengers.svg --md >> report.md + echo "### AirPassengers_test30" >> report.md + cml asset publish tests/metrics/AirPassengers_test30.svg --md >> report.md echo "\n" >> report.md # Post reports as comments in GitHub PRs cml comment update --pr report.md # post to PR diff --git a/pyproject.toml b/pyproject.toml index ba05b7a22..87c05c1d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,9 @@ black = { extras = ["jupyter"], version = "^23.1.0" } flake8 = "^5.0.4" isort = "^5.11.5" pytest = "^7.2.2" -kaleido = "0.2.1" +kaleido = "0.2.1" # required for plotly static image export pytest-cov = "^4.0.0" -tabulate = "^0.9" # Used in CI model metrics tests only, for PR bot +tabulate = "^0.9" # Used in model metrics CI only; md export for github-actions bot [tool.poetry.group.docs] optional = true From 65a83a0f2eba5faf8224f3f8db86e921bf835c5d Mon Sep 17 00:00:00 2001 From: ourownstory Date: Wed, 18 Oct 2023 15:14:16 -0700 Subject: [PATCH 02/12] addess deprecated use of --pr --- .github/workflows/metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 7d1f65230..9fefd6e5d 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -63,7 +63,7 @@ jobs: cml asset publish tests/metrics/AirPassengers_test30.svg --md >> report.md echo "\n" >> report.md # Post reports as comments in GitHub PRs - cml comment update --pr report.md # post to PR + cml comment update --target=pr report.md # post to PR cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3 From c9c15e71f9d31ab5bcfef34cba7d9f4b4cf11e6f Mon Sep 17 00:00:00 2001 From: ourownstory Date: Wed, 18 Oct 2023 15:16:16 -0700 Subject: [PATCH 03/12] no venv --- .github/workflows/metrics.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 9fefd6e5d..ba2b6967c 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -15,6 +15,8 @@ jobs: metrics: runs-on: ubuntu-latest container: docker://ghcr.io/iterative/cml:0-dvc2-base1 + env: + POETRY_VIRTUALENVS_CREATE: false steps: - name: Checkout uses: actions/checkout@v3 From 4ca665eed47cbe4a69d0251cd5bc230c93e30cde Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Wed, 18 Oct 2023 15:24:32 -0700 Subject: [PATCH 04/12] remove python install --- .github/workflows/metrics.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index ba2b6967c..ff9e18c7c 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -22,10 +22,6 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Install Python "3.10" - uses: actions/setup-python@v4 - with: - python-version: "3.10" - name: Install Poetry uses: snok/install-poetry@v1 - name: Install dependencies From 15fe61c01efb8a22318e0849e76d88489be79696 Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Wed, 18 Oct 2023 15:43:00 -0700 Subject: [PATCH 05/12] Update pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 87c05c1d8..f1ea5dbef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,9 +41,9 @@ live = ["livelossplot"] black = { extras = ["jupyter"], version = "^23.1.0" } flake8 = "^5.0.4" isort = "^5.11.5" -pytest = "^7.2.2" +pytest = "^7.2" kaleido = "0.2.1" # required for plotly static image export -pytest-cov = "^4.0.0" +pytest-cov = "^4.0" tabulate = "^0.9" # Used in model metrics CI only; md export for github-actions bot [tool.poetry.group.docs] From 512988a5cbcd61d6bdd2d662843863aa5472cbf4 Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Wed, 18 Oct 2023 15:43:42 -0700 Subject: [PATCH 06/12] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f1ea5dbef..5fa023d22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,8 +42,8 @@ black = { extras = ["jupyter"], version = "^23.1.0" } flake8 = "^5.0.4" isort = "^5.11.5" pytest = "^7.2" -kaleido = "0.2.1" # required for plotly static image export pytest-cov = "^4.0" +kaleido = "0.2.1" # required for plotly static image export tabulate = "^0.9" # Used in model metrics CI only; md export for github-actions bot [tool.poetry.group.docs] From 7666b172b8f8551f417e83c784c276c0387f3404 Mon Sep 17 00:00:00 2001 From: ourownstory Date: Wed, 18 Oct 2023 15:47:53 -0700 Subject: [PATCH 07/12] update lock --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 53d195e19..fee1ecb84 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4314,4 +4314,4 @@ live = ["livelossplot"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "cf9b835b7aa3ca7b8f63cd6830a39930caec05b12688652b715faf55d35b235f" +content-hash = "e9101b1a0eb4ed0cc7cd2d81f3dc43740bf74c6faab62a55e8d6e34734084a4c" From e7d0345a1689bd51509ab8ff8cc0dcd0f731b260 Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Wed, 18 Oct 2023 15:57:51 -0700 Subject: [PATCH 08/12] pip install poetry --- .github/workflows/metrics.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index ff9e18c7c..bcdef03fb 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -25,7 +25,9 @@ jobs: - name: Install Poetry uses: snok/install-poetry@v1 - name: Install dependencies - run: poetry install + run: | + poetry install + pip3 install pytest # should have been installed by poetry but is not - name: Train model run: | pytest tests/test_model_performance.py From 52bcd906e93353c9d598b62722c899db35927885 Mon Sep 17 00:00:00 2001 From: ourownstory Date: Wed, 18 Oct 2023 16:06:40 -0700 Subject: [PATCH 09/12] fix poetry run --- .github/workflows/metrics.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index bcdef03fb..7a1542445 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -22,15 +22,17 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Install Python "3.10" + uses: actions/setup-python@v4 + with: + python-version: "3.10" - name: Install Poetry uses: snok/install-poetry@v1 - name: Install dependencies run: | poetry install - pip3 install pytest # should have been installed by poetry but is not - name: Train model - run: | - pytest tests/test_model_performance.py + run: poetry run pytest tests/test_model_performance.py - name: Download metrics from main uses: dawidd6/action-download-artifact@v2 with: From 9f0f137bee4e35021e526e476c56e8c42239e74d Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Wed, 18 Oct 2023 16:09:31 -0700 Subject: [PATCH 10/12] Update metrics.yml remove python install --- .github/workflows/metrics.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 7a1542445..5b435aa4a 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -22,15 +22,10 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Install Python "3.10" - uses: actions/setup-python@v4 - with: - python-version: "3.10" - name: Install Poetry uses: snok/install-poetry@v1 - name: Install dependencies - run: | - poetry install + run: poetry install - name: Train model run: poetry run pytest tests/test_model_performance.py - name: Download metrics from main From b7d0e3a4a7cabcda57c59deab8771c4765b832c3 Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Wed, 18 Oct 2023 16:17:15 -0700 Subject: [PATCH 11/12] remove poetry --- .github/workflows/metrics.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 5b435aa4a..885b35e38 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -22,12 +22,15 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Install Poetry - uses: snok/install-poetry@v1 - name: Install dependencies - run: poetry install + run: | + python3 -m pip install --upgrade pip + pip3 install . + pip3 install pytest + pip3 install tabulate # required to md export + pip3 install kaleido # required for plotly export - name: Train model - run: poetry run pytest tests/test_model_performance.py + run: pytest tests/test_model_performance.py - name: Download metrics from main uses: dawidd6/action-download-artifact@v2 with: From 7a1bd007f7675207f8f53e320020bdf032076804 Mon Sep 17 00:00:00 2001 From: ourownstory Date: Wed, 18 Oct 2023 16:33:03 -0700 Subject: [PATCH 12/12] fix typo --- tests/test_model_performance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_model_performance.py b/tests/test_model_performance.py index 22fa9697d..f3ba7d1bc 100644 --- a/tests/test_model_performance.py +++ b/tests/test_model_performance.py @@ -208,10 +208,10 @@ def test_YosemiteTemps_test20(): accuracy_metrics["time"] = round(end - start, 2) accuracy_metrics["system_performance"] = round(system_speed, 5) accuracy_metrics["system_std"] = round(std, 5) - with open(os.path.join(DIR, "tests", "metrics", "YosemiteTemps_test30.json"), "w") as outfile: + with open(os.path.join(DIR, "tests", "metrics", "YosemiteTemps_test20.json"), "w") as outfile: json.dump(accuracy_metrics, outfile) - create_metrics_plot(metrics).write_image(os.path.join(DIR, "tests", "metrics", "YosemiteTemps_test30.svg")) + create_metrics_plot(metrics).write_image(os.path.join(DIR, "tests", "metrics", "YosemiteTemps_test20.svg")) def test_AirPassengers():