From 599c0661262773a8c6236baad64261e6a5544072 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 10:37:34 -0800
Subject: [PATCH 01/68] feat: update pr.yml to open veda-config pr and publish
 collections

---
 .github/workflows/pr.yml | 83 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 81 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index cb3a0e0d..2bbe5153 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -230,9 +230,88 @@ jobs:
     runs-on: ubuntu-latest
     needs: create-mdx-files
     steps:
-      - name: Open veda-config PR
+      - name: Set up Git
         run: |
-          echo "NO-OP. Placeholder for future job that will open a Pull Request in veda-config for a dashboard preview for the new/changed datasets."
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Clone `veda-config`
+        env:
+          VEDA_CONFIG_REPO_ACCESS_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
+        run: git clone https://${{ env.VEDA_CONFIG_REPO_ACCESS_TOKEN }}@github.com/${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}.git
+
+      # Creates a PR in veda-config with the following changes:
+      # 1. the mdx files for all published collections
+      # 2. updates the stac/raster urls in .env file
+      # This step needs a GH_TOKEN that has permissions to create a PR in veda-config
+      - name: Create PR with changes
+        id: create-pr
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          VEDA_CONFIG_REPO_ACCESS_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
+          COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
+          PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
+        run: |
+          files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")
+          hash=$(echo -n "$files_string" | md5sum | cut -d ' ' -f 1)
+          NEW_BRANCH="add-dataset-$hash"
+          cd ${{ vars.VEDA_CONFIG_REPO_NAME }}
+          git fetch origin
+          if git ls-remote --exit-code --heads origin $NEW_BRANCH; then
+            git push origin --delete $NEW_BRANCH
+          fi
+          git checkout -b $NEW_BRANCH
+
+          # Update the env vars to staging based on env vars
+          sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
+          cp -r ../datasets/* datasets/
+          git add .
+          git commit -m "Add dataset(s)"
+          git push origin $NEW_BRANCH
+          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
+
+          echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
+          echo "PR creation succeeded"
+
+      # Updates the comment with a link to the above PR
+      - name: Update PR comment with PR creation result
+        if: success()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+        run: |
+          PR_URL=${{ steps.create-pr.outputs.PR_URL }}
+          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+          UPDATED_BODY="$CURRENT_BODY
+
+          **A PR has been created with the dataset configuration: 🗺️ [PR link]($PR_URL)**"
+          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+      - name: Update PR comment on PR creation failure
+        if: failure() && steps.create-pr.outcome == 'failure'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+        run: |
+          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+          UPDATED_BODY="$CURRENT_BODY
+
+          **Failed ❌ to create a PR with the dataset configuration. 😔 **"
+          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+      # If the workflow fails at any point, the PR comment will be updated
+      - name: Update PR comment on overall workflow failure
+        if: failure() && steps.create-pr.outcome != 'failure'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+        run: |
+          WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+          UPDATED_BODY="$CURRENT_BODY
+
+          ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
+          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
 
   publish-to-prod-on-pr-merge:
     if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true }}

From 3d4b2d0c0ad96e31803429857a6227d861a68fee Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 10:42:45 -0800
Subject: [PATCH 02/68] feat: add test dataset-config

---
 .../staging/dataset-config/test.json          | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 ingestion-data/staging/dataset-config/test.json

diff --git a/ingestion-data/staging/dataset-config/test.json b/ingestion-data/staging/dataset-config/test.json
new file mode 100644
index 00000000..eafd5b72
--- /dev/null
+++ b/ingestion-data/staging/dataset-config/test.json
@@ -0,0 +1,31 @@
+{
+    "collection": "hls-swir-falsecolor-composite-THIRD-TEST",
+    "title": "HLS SWIR FalseColor Composite",
+    "spatial_extent": {
+        "xmin": -156.75,
+        "ymin": 20.80,
+        "xmax": -156.55,
+        "ymax": 20.94
+    },
+    "temporal_extent": {
+        "startdate": "2023-08-08T00:00:00Z",
+        "enddate": "2023-08-08T23:59:59Z"
+    },
+    "data_type": "cog",
+    "license": "CC0-1.0",
+    "description": "HLS falsecolor composite imagery using Bands 12, 8A, and 4.",
+    "is_periodic": false,
+    "time_density": "day",
+    "sample_files": [
+        "s3://veda-data-store-staging/maui-fire/Lahaina_HLS_2023-08-08_SWIR_falsecolor_cog.tif",
+        "s3://veda-data-store-staging/maui-fire/Lahaina_HLS_2023-08-13_SWIR_falsecolor_cog.tif"
+    ],
+    "discovery_items": [
+        {
+            "discovery": "s3",
+            "prefix": "maui-fire/",
+            "bucket": "veda-data-store-staging",
+            "filename_regex": "(.*)SWIR_falsecolor(.*).tif$"
+        }
+    ]
+}
\ No newline at end of file

From af17bf44db2ae829b5d256feae82e6586b2f9cd0 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 10:57:47 -0800
Subject: [PATCH 03/68] fix: add debugging

---
 .github/workflows/pr.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 2bbe5153..e201747e 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -310,9 +310,14 @@ jobs:
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
           UPDATED_BODY="$CURRENT_BODY
 
+          # Output WORKFLOW_URL to logs for verification
+          echo "Workflow URL: $WORKFLOW_URL"
+
           ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
           gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
 
+          echo "Updated Comment Body: $UPDATED_BODY"
+
   publish-to-prod-on-pr-merge:
     if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true }}
     runs-on: ubuntu-latest

From 608b6909587829df76ef2085ce47c393d5bdc86b Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 11:49:14 -0800
Subject: [PATCH 04/68] fix: add more debugging statements

---
 .github/workflows/pr.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index e201747e..092b1846 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -235,6 +235,11 @@ jobs:
           git config --global user.name "github-actions[bot]"
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
 
+      - name: Debug Variables
+        run: |
+          echo "CONFIG_REPO_ORG: ${{ vars.VEDA_CONFIG_REPO_ORG }}"
+          echo "CONFIG_REPO_NAME: ${{ vars.VEDA_CONFIG_REPO_NAME }}"
+
       - name: Clone `veda-config`
         env:
           VEDA_CONFIG_REPO_ACCESS_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}

From 865928c0925c90275c3565b960905d8d85745ede Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 13:05:46 -0800
Subject: [PATCH 05/68] fix: add environment to job

---
 .github/workflows/pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 092b1846..6ef17e61 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -229,6 +229,7 @@ jobs:
   open-veda-config-pr:
     runs-on: ubuntu-latest
     needs: create-mdx-files
+    environment: staging
     steps:
       - name: Set up Git
         run: |

From 8cb96313d1a990e4a71dc23944937efa6b9316ae Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 13:55:12 -0800
Subject: [PATCH 06/68] fix: testing

---
 .github/workflows/pr.yml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 6ef17e61..2131b3da 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -269,15 +269,16 @@ jobs:
           git checkout -b $NEW_BRANCH
 
           # Update the env vars to staging based on env vars
-          sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
+          #sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
+          cat .env
           cp -r ../datasets/* datasets/
-          git add .
-          git commit -m "Add dataset(s)"
-          git push origin $NEW_BRANCH
-          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
+          # git add .
+          # git commit -m "Add dataset(s)"
+          # git push origin $NEW_BRANCH
+          # PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
 
-          echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
-          echo "PR creation succeeded"
+          # echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
+          # echo "PR creation succeeded"
 
       # Updates the comment with a link to the above PR
       - name: Update PR comment with PR creation result

From a86093e2bee777ef01a8f6f3f9b08b828f313704 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 14:33:09 -0800
Subject: [PATCH 07/68] fix: add debugging for directory

---
 .github/workflows/pr.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 2131b3da..4ef9bdd5 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -271,6 +271,9 @@ jobs:
           # Update the env vars to staging based on env vars
           #sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
           cat .env
+          ls
+          echo "Checking directory below"
+          ls ..
           cp -r ../datasets/* datasets/
           # git add .
           # git commit -m "Add dataset(s)"

From f351a1ae2c44664632497e9a0a8be3afb2214a7e Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 14:45:45 -0800
Subject: [PATCH 08/68] fix: add github debugging

---
 .github/workflows/pr.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 4ef9bdd5..ddb804c8 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -270,11 +270,8 @@ jobs:
 
           # Update the env vars to staging based on env vars
           #sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
-          cat .env
-          ls
-          echo "Checking directory below"
-          ls ..
-          cp -r ../datasets/* datasets/
+          # cp -r ../datasets/* datasets/
+          git status
           # git add .
           # git commit -m "Add dataset(s)"
           # git push origin $NEW_BRANCH

From 43766374d4a79f11a6755d9afb79b4aa3c354b9c Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 15:00:22 -0800
Subject: [PATCH 09/68] fix: add logs to create-mdx-files step

---
 .github/workflows/pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index ddb804c8..9cca6191 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -225,6 +225,7 @@ jobs:
           do
             python3 ./scripts/generate-mdx.py "$file"
           done
+          ls
 
   open-veda-config-pr:
     runs-on: ubuntu-latest

From 5e5a1a63d3e5d8cf23a8ea9ddcd5689f27737765 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 15:13:49 -0800
Subject: [PATCH 10/68] fix: add step to output files

---
 .github/workflows/pr.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 9cca6191..cbf4323d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -225,7 +225,12 @@ jobs:
           do
             python3 ./scripts/generate-mdx.py "$file"
           done
-          ls
+
+      - name: List files in workspace
+        run: |
+          echo "Listing all files to verify .mdx file generation:"
+          ls -la
+          ls ./scripts/
 
   open-veda-config-pr:
     runs-on: ubuntu-latest

From b256e481f7bf6b26866d33aabab29f7e927b0295 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 15:19:47 -0800
Subject: [PATCH 11/68] fix: update ls path

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index cbf4323d..b82f104f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -230,7 +230,7 @@ jobs:
         run: |
           echo "Listing all files to verify .mdx file generation:"
           ls -la
-          ls ./scripts/
+          ls ./ingestion-data/dataset-mdx/
 
   open-veda-config-pr:
     runs-on: ubuntu-latest

From 1013d4d66183c3e2878ebb7e997ae3b75abafa75 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 15:41:26 -0800
Subject: [PATCH 12/68] fix: replace git clone with checkout

---
 .github/workflows/pr.yml | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index b82f104f..670ff76e 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -247,10 +247,20 @@ jobs:
           echo "CONFIG_REPO_ORG: ${{ vars.VEDA_CONFIG_REPO_ORG }}"
           echo "CONFIG_REPO_NAME: ${{ vars.VEDA_CONFIG_REPO_NAME }}"
 
-      - name: Clone `veda-config`
-        env:
-          VEDA_CONFIG_REPO_ACCESS_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
-        run: git clone https://${{ env.VEDA_CONFIG_REPO_ACCESS_TOKEN }}@github.com/${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}.git
+      - name: Checkout veda-config repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ vars.VEDA_CONFIG_REPO_ORG }}/${{vars.VEDA_CONFIG_REPO_NAME }}
+          token: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
+          path: datasets
+
+      - name: Copy MDX file to veda-config
+        run: |
+          echo "Copying .mdx file to veda-config repository"
+          ls ./ingestion-data/dataset-mdx/*
+          git status
+          # cp ./ingestion-data/dataset-mdx/* datasets/
+          ls -la datasets
 
       # Creates a PR in veda-config with the following changes:
       # 1. the mdx files for all published collections

From 3c7b34be1a26b16633f97116f95cf65f14f9dcce Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 15:43:54 -0800
Subject: [PATCH 13/68] fix: typo

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 670ff76e..8a344af5 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -257,7 +257,7 @@ jobs:
       - name: Copy MDX file to veda-config
         run: |
           echo "Copying .mdx file to veda-config repository"
-          ls ./ingestion-data/dataset-mdx/*
+          ls ./ingestion-data/dataset-mdx/
           git status
           # cp ./ingestion-data/dataset-mdx/* datasets/
           ls -la datasets

From 94f6e4a0647da5d6394b92f921067d2751868392 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 15:50:38 -0800
Subject: [PATCH 14/68] fix: merge steps

---
 .github/workflows/pr.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 8a344af5..4c666d9d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -232,11 +232,6 @@ jobs:
           ls -la
           ls ./ingestion-data/dataset-mdx/
 
-  open-veda-config-pr:
-    runs-on: ubuntu-latest
-    needs: create-mdx-files
-    environment: staging
-    steps:
       - name: Set up Git
         run: |
           git config --global user.name "github-actions[bot]"

From 3eb2059e74bb18a292c6db0cb123c8ad9b5a0180 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 15:54:28 -0800
Subject: [PATCH 15/68] fix: add environment

---
 .github/workflows/pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 4c666d9d..4734526b 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -205,6 +205,7 @@ jobs:
 
   create-mdx-files:
     runs-on: ubuntu-latest
+    environment: staging
     needs: publish-new-datasets
     steps:
       - name: Checkout code

From 63886d165d04072a5c4ecded040c56c6d89a3fe4 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 16:07:11 -0800
Subject: [PATCH 16/68] fix: try to copy generated file to new directory in new
 repo

---
 .github/workflows/pr.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 4734526b..946d130d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -255,8 +255,8 @@ jobs:
           echo "Copying .mdx file to veda-config repository"
           ls ./ingestion-data/dataset-mdx/
           git status
-          # cp ./ingestion-data/dataset-mdx/* datasets/
-          ls -la datasets
+          cp ingestion-data/dataset-mdx/ datasets/
+          git status
 
       # Creates a PR in veda-config with the following changes:
       # 1. the mdx files for all published collections
@@ -273,7 +273,6 @@ jobs:
           files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")
           hash=$(echo -n "$files_string" | md5sum | cut -d ' ' -f 1)
           NEW_BRANCH="add-dataset-$hash"
-          cd ${{ vars.VEDA_CONFIG_REPO_NAME }}
           git fetch origin
           if git ls-remote --exit-code --heads origin $NEW_BRANCH; then
             git push origin --delete $NEW_BRANCH

From 31c3aeda4f6acf9aeb8371b1b08a7855bf31ddd8 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 16:19:08 -0800
Subject: [PATCH 17/68] fix: add modifications for copying untracked file

---
 .github/workflows/pr.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 946d130d..fe9e2492 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -250,12 +250,15 @@ jobs:
           token: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
           path: datasets
 
-      - name: Copy MDX file to veda-config
+      - name: Copy untracked mdx file to veda-config
         run: |
           echo "Copying .mdx file to veda-config repository"
           ls ./ingestion-data/dataset-mdx/
           git status
-          cp ingestion-data/dataset-mdx/ datasets/
+          git ls-files --others --exclude-standard | while read file; do
+            echo "Copying $file to veda-config/datasets"
+            cp --parents "$file" /datasets/
+          done
           git status
 
       # Creates a PR in veda-config with the following changes:
@@ -281,7 +284,8 @@ jobs:
 
           # Update the env vars to staging based on env vars
           #sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
-          # cp -r ../datasets/* datasets/
+          cp -r datasets/* veda-config/datasets/
+          ls veda-config/datasets
           git status
           # git add .
           # git commit -m "Add dataset(s)"

From dbf55f41bffbfe927a62d8c033428d1892726d25 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 16:23:17 -0800
Subject: [PATCH 18/68] fix: create source directory

---
 .github/workflows/pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index fe9e2492..4ee58638 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -253,6 +253,7 @@ jobs:
       - name: Copy untracked mdx file to veda-config
         run: |
           echo "Copying .mdx file to veda-config repository"
+          mkdir -p datasets
           ls ./ingestion-data/dataset-mdx/
           git status
           git ls-files --others --exclude-standard | while read file; do

From 4d18d7c45b0610d93e152488b76e298ea0e6ec0c Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 16:29:57 -0800
Subject: [PATCH 19/68] fix: update copying commands

---
 .github/workflows/pr.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 4ee58638..8eb07d24 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -257,8 +257,9 @@ jobs:
           ls ./ingestion-data/dataset-mdx/
           git status
           git ls-files --others --exclude-standard | while read file; do
-            echo "Copying $file to veda-config/datasets"
-            cp --parents "$file" /datasets/
+            echo "Copying $file to datasets directory"
+            mkdir -p "datasets/$(dirname "$file")"
+            cp "$file" "/datasets/$file"
           done
           git status
 

From 91dc819487cb462fe9476ea048745fb3bb8e4cda Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 17 Dec 2024 16:33:02 -0800
Subject: [PATCH 20/68] fix: update command

---
 .github/workflows/pr.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 8eb07d24..0f5e28a2 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -258,8 +258,8 @@ jobs:
           git status
           git ls-files --others --exclude-standard | while read file; do
             echo "Copying $file to datasets directory"
-            mkdir -p "datasets/$(dirname "$file")"
-            cp "$file" "/datasets/$file"
+            mkdir -p "datasets"
+            cp "$file" "/datasets/"
           done
           git status
 

From 788ca5faa5f8b56edaafcaacbb02cdb7978e860d Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 09:46:05 -0800
Subject: [PATCH 21/68] fix: add debugging for copying file

---
 .github/workflows/pr.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 0f5e28a2..996603ec 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -259,7 +259,8 @@ jobs:
           git ls-files --others --exclude-standard | while read file; do
             echo "Copying $file to datasets directory"
             mkdir -p "datasets"
-            cp "$file" "/datasets/"
+            ls
+            cp "$file" "datasets/"
           done
           git status
 

From 5ef917f65954a9c87b4518faf53bc358f0cb8a7b Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 09:50:52 -0800
Subject: [PATCH 22/68] fix: add more logging, remove extraneous copy

---
 .github/workflows/pr.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 996603ec..2a20e351 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -287,10 +287,10 @@ jobs:
 
           # Update the env vars to staging based on env vars
           #sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
-          cp -r datasets/* veda-config/datasets/
           ls veda-config/datasets
           git status
-          # git add .
+          git add .
+          echo "Added file for tracking"
           # git commit -m "Add dataset(s)"
           # git push origin $NEW_BRANCH
           # PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))

From f3fa2b71689c9e97b3823cc32e4060ead91a9049 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 10:02:07 -0800
Subject: [PATCH 23/68] fix: remove ls path

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 2a20e351..816d7c4f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -287,7 +287,7 @@ jobs:
 
           # Update the env vars to staging based on env vars
           #sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
-          ls veda-config/datasets
+          ls
           git status
           git add .
           echo "Added file for tracking"

From 91842609e5dd666cb3a5b27e9d722bfa816dd58b Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 10:09:27 -0800
Subject: [PATCH 24/68] fix: attempt to create pr

---
 .github/workflows/pr.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 816d7c4f..029ac978 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -291,12 +291,12 @@ jobs:
           git status
           git add .
           echo "Added file for tracking"
-          # git commit -m "Add dataset(s)"
-          # git push origin $NEW_BRANCH
-          # PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
+          git commit -m "Add dataset(s)"
+          git push origin $NEW_BRANCH
+          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
 
-          # echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
-          # echo "PR creation succeeded"
+          echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
+          echo "PR creation succeeded"
 
       # Updates the comment with a link to the above PR
       - name: Update PR comment with PR creation result

From ff68de35e2347c79375d30df1de12a94d933316e Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 10:39:23 -0800
Subject: [PATCH 25/68] fix: update pr creation command to specify org/repo
 directly

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 029ac978..06ca31f0 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -293,7 +293,7 @@ jobs:
           echo "Added file for tracking"
           git commit -m "Add dataset(s)"
           git push origin $NEW_BRANCH
-          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
+          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -R ${{ vars.VEDA_CONFIG_REPO_ORG }}/${{vars.VEDA_CONFIG_REPO_NAME }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR creation succeeded"

From 41c9dd1d3ad3b48fd5c767b0d283938764d5c141 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 10:44:13 -0800
Subject: [PATCH 26/68] fix: add debugging

---
 .github/workflows/pr.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 06ca31f0..12188194 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -290,10 +290,11 @@ jobs:
           ls
           git status
           git add .
-          echo "Added file for tracking"
+          echo "**DEBUG**-Added file for tracking"
           git commit -m "Add dataset(s)"
           git push origin $NEW_BRANCH
-          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -R ${{ vars.VEDA_CONFIG_REPO_ORG }}/${{vars.VEDA_CONFIG_REPO_NAME }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
+          echo "**DEBUG**-Committed and pushed file"
+          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -R ${{ vars.VEDA_CONFIG_REPO_ORG }}/${{vars.VEDA_CONFIG_REPO_NAME }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)") --verbose)
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR creation succeeded"

From 189221f085a9287200e0600bf4fae99e96e56812 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 11:07:18 -0800
Subject: [PATCH 27/68] fix: check branch

---
 .github/workflows/pr.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 12188194..d82bb65a 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -248,6 +248,7 @@ jobs:
         with:
           repository: ${{ vars.VEDA_CONFIG_REPO_ORG }}/${{vars.VEDA_CONFIG_REPO_NAME }}
           token: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
+          ref: develop
           path: datasets
 
       - name: Copy untracked mdx file to veda-config
@@ -288,6 +289,8 @@ jobs:
           # Update the env vars to staging based on env vars
           #sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
           ls
+          git remote -v
+          git branch
           git status
           git add .
           echo "**DEBUG**-Added file for tracking"

From a5ba6771920bff22c3a57bf87520f41a9cc017de Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:10:04 -0800
Subject: [PATCH 28/68] fix: change token reference

---
 .github/workflows/pr.yml | 89 ++++++++++++++++++++--------------------
 1 file changed, 44 insertions(+), 45 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index d82bb65a..c03db44d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -272,8 +272,7 @@ jobs:
       - name: Create PR with changes
         id: create-pr
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          VEDA_CONFIG_REPO_ACCESS_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
           COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
         run: |
@@ -303,49 +302,49 @@ jobs:
           echo "PR creation succeeded"
 
       # Updates the comment with a link to the above PR
-      - name: Update PR comment with PR creation result
-        if: success()
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-        run: |
-          PR_URL=${{ steps.create-pr.outputs.PR_URL }}
-          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-          UPDATED_BODY="$CURRENT_BODY
-
-          **A PR has been created with the dataset configuration: 🗺️ [PR link]($PR_URL)**"
-          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
-      - name: Update PR comment on PR creation failure
-        if: failure() && steps.create-pr.outcome == 'failure'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-        run: |
-          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-          UPDATED_BODY="$CURRENT_BODY
-
-          **Failed ❌ to create a PR with the dataset configuration. 😔 **"
-          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
-      # If the workflow fails at any point, the PR comment will be updated
-      - name: Update PR comment on overall workflow failure
-        if: failure() && steps.create-pr.outcome != 'failure'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-        run: |
-          WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-          UPDATED_BODY="$CURRENT_BODY
-
-          # Output WORKFLOW_URL to logs for verification
-          echo "Workflow URL: $WORKFLOW_URL"
-
-          ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
-          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
-          echo "Updated Comment Body: $UPDATED_BODY"
+      # - name: Update PR comment with PR creation result
+      #   if: success()
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      #   run: |
+      #     PR_URL=${{ steps.create-pr.outputs.PR_URL }}
+      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+      #     UPDATED_BODY="$CURRENT_BODY
+
+      #     **A PR has been created with the dataset configuration: 🗺️ [PR link]($PR_URL)**"
+      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+      # - name: Update PR comment on PR creation failure
+      #   if: failure() && steps.create-pr.outcome == 'failure'
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      #   run: |
+      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+      #     UPDATED_BODY="$CURRENT_BODY
+
+      #     **Failed ❌ to create a PR with the dataset configuration. 😔 **"
+      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+      # # If the workflow fails at any point, the PR comment will be updated
+      # - name: Update PR comment on overall workflow failure
+      #   if: failure() && steps.create-pr.outcome != 'failure'
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      #   run: |
+      #     WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+      #     UPDATED_BODY="$CURRENT_BODY
+
+      #     # Output WORKFLOW_URL to logs for verification
+      #     echo "Workflow URL: $WORKFLOW_URL"
+
+      #     ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
+      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+      #     echo "Updated Comment Body: $UPDATED_BODY"
 
   publish-to-prod-on-pr-merge:
     if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true }}

From 98fd0cab8e06f2e9c542f0d92bd3c542578dc6b1 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:15:57 -0800
Subject: [PATCH 29/68] fix: add branch debugging

---
 .github/workflows/pr.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index c03db44d..42f166d4 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -238,15 +238,16 @@ jobs:
           git config --global user.name "github-actions[bot]"
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
 
-      - name: Debug Variables
+      - name: Set up Variables
         run: |
           echo "CONFIG_REPO_ORG: ${{ vars.VEDA_CONFIG_REPO_ORG }}"
           echo "CONFIG_REPO_NAME: ${{ vars.VEDA_CONFIG_REPO_NAME }}"
+          echo "VEDA_CONFIG_REPO=${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}" >> $GITHUB_ENV
 
       - name: Checkout veda-config repo
         uses: actions/checkout@v4
         with:
-          repository: ${{ vars.VEDA_CONFIG_REPO_ORG }}/${{vars.VEDA_CONFIG_REPO_NAME }}
+          repository: ${{ env.VEDA_CONFIG_REPO }}
           token: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
           ref: develop
           path: datasets
@@ -254,6 +255,8 @@ jobs:
       - name: Copy untracked mdx file to veda-config
         run: |
           echo "Copying .mdx file to veda-config repository"
+          git remote -v
+          git branch
           mkdir -p datasets
           ls ./ingestion-data/dataset-mdx/
           git status

From 44ba78b5f9b51bded37968c02c91cd339d78a96b Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:26:45 -0800
Subject: [PATCH 30/68] fix: add debugging token access step

---
 .github/workflows/pr.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 42f166d4..145fc580 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -252,6 +252,11 @@ jobs:
           ref: develop
           path: datasets
 
+      - name: Debug Token Access
+        run: |
+          curl -H "Authorization: token $VEDA_CONFIG_REPO_ACCESS_TOKEN" \
+          https://api.github.com/repos/${{ env.VEDA_CONFIG_REPO }}
+
       - name: Copy untracked mdx file to veda-config
         run: |
           echo "Copying .mdx file to veda-config repository"
@@ -299,7 +304,7 @@ jobs:
           git commit -m "Add dataset(s)"
           git push origin $NEW_BRANCH
           echo "**DEBUG**-Committed and pushed file"
-          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -R ${{ vars.VEDA_CONFIG_REPO_ORG }}/${{vars.VEDA_CONFIG_REPO_NAME }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)") --verbose)
+          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR creation succeeded"

From 87dcf4711d427d7e56139cb2cf80e784c58c0bd8 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:29:30 -0800
Subject: [PATCH 31/68] fix: try another syntax for token supplying

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 145fc580..e6266a03 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -254,7 +254,7 @@ jobs:
 
       - name: Debug Token Access
         run: |
-          curl -H "Authorization: token $VEDA_CONFIG_REPO_ACCESS_TOKEN" \
+          curl -H "Authorization: token ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}" \
           https://api.github.com/repos/${{ env.VEDA_CONFIG_REPO }}
 
       - name: Copy untracked mdx file to veda-config

From ad2682040ece32e6e9aabeb9265763847cfffd1e Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:31:52 -0800
Subject: [PATCH 32/68] fix: update token commands

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index e6266a03..cc17c8eb 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -304,7 +304,7 @@ jobs:
           git commit -m "Add dataset(s)"
           git push origin $NEW_BRANCH
           echo "**DEBUG**-Committed and pushed file"
-          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_REPO_ACCESS_TOKEN gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
+          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR creation succeeded"

From 5b1846d2b5238bf13ccfceec85f483d8e691b365 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:36:52 -0800
Subject: [PATCH 33/68] fix: more debugging target repo

---
 .github/workflows/pr.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index cc17c8eb..605865e2 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -257,6 +257,12 @@ jobs:
           curl -H "Authorization: token ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}" \
           https://api.github.com/repos/${{ env.VEDA_CONFIG_REPO }}
 
+      - name: Verify Target Repository
+        run: |
+          echo "Current Directory: $(pwd)"
+          git remote -v
+          git branch
+
       - name: Copy untracked mdx file to veda-config
         run: |
           echo "Copying .mdx file to veda-config repository"

From 618d743da14614b61f4213def21d6e2de11b4b48 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:46:47 -0800
Subject: [PATCH 34/68] fix: try checking out target repo another way

---
 .github/workflows/pr.yml | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 605865e2..42de1680 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -244,13 +244,11 @@ jobs:
           echo "CONFIG_REPO_NAME: ${{ vars.VEDA_CONFIG_REPO_NAME }}"
           echo "VEDA_CONFIG_REPO=${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}" >> $GITHUB_ENV
 
-      - name: Checkout veda-config repo
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ env.VEDA_CONFIG_REPO }}
-          token: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
-          ref: develop
-          path: datasets
+      - name: Clone the Target Repository
+        run: |
+            git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git datasets
+            git checkout develop
+            cd datasets
 
       - name: Debug Token Access
         run: |

From 54c93e26a5e81e303676c27f67a4910304f305f5 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:48:20 -0800
Subject: [PATCH 35/68] fix: typo

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 42de1680..78f220d9 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -246,7 +246,7 @@ jobs:
 
       - name: Clone the Target Repository
         run: |
-            git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git datasets
+            git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git
             git checkout develop
             cd datasets
 

From ecfc08d5d38ae4f5e73b8da8b36497ba3f08472b Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:52:42 -0800
Subject: [PATCH 36/68] fix: remove prev debugging token step

---
 .github/workflows/pr.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 78f220d9..534b2ddd 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -247,14 +247,8 @@ jobs:
       - name: Clone the Target Repository
         run: |
             git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git
-            git checkout develop
             cd datasets
 
-      - name: Debug Token Access
-        run: |
-          curl -H "Authorization: token ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}" \
-          https://api.github.com/repos/${{ env.VEDA_CONFIG_REPO }}
-
       - name: Verify Target Repository
         run: |
           echo "Current Directory: $(pwd)"

From 921bf57b3060cfb585174013d9f3bca4ba03a362 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 13:54:51 -0800
Subject: [PATCH 37/68] fix: change cloning step

---
 .github/workflows/pr.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 534b2ddd..1afe9303 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -247,7 +247,7 @@ jobs:
       - name: Clone the Target Repository
         run: |
             git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git
-            cd datasets
+            ls
 
       - name: Verify Target Repository
         run: |
@@ -258,8 +258,6 @@ jobs:
       - name: Copy untracked mdx file to veda-config
         run: |
           echo "Copying .mdx file to veda-config repository"
-          git remote -v
-          git branch
           mkdir -p datasets
           ls ./ingestion-data/dataset-mdx/
           git status
@@ -269,7 +267,6 @@ jobs:
             ls
             cp "$file" "datasets/"
           done
-          git status
 
       # Creates a PR in veda-config with the following changes:
       # 1. the mdx files for all published collections

From 8bab675c0c686753ed437822a28c415eaa6a9bb1 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 14:10:05 -0800
Subject: [PATCH 38/68] fix: cd into cloned repo

---
 .github/workflows/pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 1afe9303..d025eb67 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -248,6 +248,7 @@ jobs:
         run: |
             git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git
             ls
+            cd ${{ vars.VEDA_CONFIG_REPO_NAME }}
 
       - name: Verify Target Repository
         run: |

From 0fc433c6d18e4ab8d11f5edb7f79899a21c3d609 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 14:22:27 -0800
Subject: [PATCH 39/68] fix: simplify

---
 .github/workflows/pr.yml | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index d025eb67..4d71db91 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -246,15 +246,10 @@ jobs:
 
       - name: Clone the Target Repository
         run: |
-            git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git
-            ls
-            cd ${{ vars.VEDA_CONFIG_REPO_NAME }}
-
-      - name: Verify Target Repository
-        run: |
+          git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git
+          ls
+          cd ${{ vars.VEDA_CONFIG_REPO_NAME }}
           echo "Current Directory: $(pwd)"
-          git remote -v
-          git branch
 
       - name: Copy untracked mdx file to veda-config
         run: |

From 3ec9ea29f974491f45603f1cc92d279b3cda4ff1 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 14:33:02 -0800
Subject: [PATCH 40/68] fix: update untracked file copying step

---
 .github/workflows/pr.yml | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 4d71db91..0f4690c0 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -248,21 +248,16 @@ jobs:
         run: |
           git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git
           ls
-          cd ${{ vars.VEDA_CONFIG_REPO_NAME }}
-          echo "Current Directory: $(pwd)"
 
       - name: Copy untracked mdx file to veda-config
         run: |
           echo "Copying .mdx file to veda-config repository"
-          mkdir -p datasets
+          ls
           ls ./ingestion-data/dataset-mdx/
-          git status
-          git ls-files --others --exclude-standard | while read file; do
-            echo "Copying $file to datasets directory"
-            mkdir -p "datasets"
-            ls
-            cp "$file" "datasets/"
-          done
+          mkdir -p datasets
+          find ingestion-data/dataset-mdx/ -name '*.mdx' -exec cp {} veda-config/datasets/ \;
+          echo "Contents of the datasets directory after copying:"
+          ls veda-config-/datasets/
 
       # Creates a PR in veda-config with the following changes:
       # 1. the mdx files for all published collections

From d37fb9b6865b38bf7760a50e4b6cb62da2787275 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 14:40:52 -0800
Subject: [PATCH 41/68] fix: typo

---
 .github/workflows/pr.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 0f4690c0..bb03dbe1 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -249,15 +249,14 @@ jobs:
           git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git
           ls
 
-      - name: Copy untracked mdx file to veda-config
+      - name: Copy untracked mdx files to veda-config
         run: |
-          echo "Copying .mdx file to veda-config repository"
-          ls
+          echo "Copying untracked .mdx files to veda-config repository"
           ls ./ingestion-data/dataset-mdx/
           mkdir -p datasets
           find ingestion-data/dataset-mdx/ -name '*.mdx' -exec cp {} veda-config/datasets/ \;
           echo "Contents of the datasets directory after copying:"
-          ls veda-config-/datasets/
+          ls veda-config/datasets/
 
       # Creates a PR in veda-config with the following changes:
       # 1. the mdx files for all published collections

From b830338397d4ca5fc66d5d696ad5b8b160ff7105 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 14:44:07 -0800
Subject: [PATCH 42/68] fix: cd into cloned repo

---
 .github/workflows/pr.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index bb03dbe1..7724196f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -269,9 +269,12 @@ jobs:
           COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
         run: |
+          cd veda-config
+
           files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")
           hash=$(echo -n "$files_string" | md5sum | cut -d ' ' -f 1)
           NEW_BRANCH="add-dataset-$hash"
+
           git fetch origin
           if git ls-remote --exit-code --heads origin $NEW_BRANCH; then
             git push origin --delete $NEW_BRANCH

From b006851512180f1b0fe8e1262463522cd77de6a5 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 14:47:54 -0800
Subject: [PATCH 43/68] fix: reorder steps, see if it fixes issue

---
 .github/workflows/pr.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 7724196f..45ad020c 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -233,11 +233,6 @@ jobs:
           ls -la
           ls ./ingestion-data/dataset-mdx/
 
-      - name: Set up Git
-        run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-
       - name: Set up Variables
         run: |
           echo "CONFIG_REPO_ORG: ${{ vars.VEDA_CONFIG_REPO_ORG }}"
@@ -258,6 +253,13 @@ jobs:
           echo "Contents of the datasets directory after copying:"
           ls veda-config/datasets/
 
+      - name: Set up Git
+        run: |
+          cd veda-config
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+
       # Creates a PR in veda-config with the following changes:
       # 1. the mdx files for all published collections
       # 2. updates the stac/raster urls in .env file
@@ -269,7 +271,6 @@ jobs:
           COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
         run: |
-          cd veda-config
 
           files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")
           hash=$(echo -n "$files_string" | md5sum | cut -d ' ' -f 1)

From 378c91f45f8683ca6d40b05fe56154a4987849ba Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 14:55:50 -0800
Subject: [PATCH 44/68] fix: print working directory

---
 .github/workflows/pr.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 45ad020c..7428887d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -271,6 +271,7 @@ jobs:
           COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
         run: |
+          echo "Working directory $(pwd)"
 
           files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")
           hash=$(echo -n "$files_string" | md5sum | cut -d ' ' -f 1)
@@ -288,10 +289,9 @@ jobs:
           git remote -v
           git branch
           git status
-          git add .
-          echo "**DEBUG**-Added file for tracking"
-          git commit -m "Add dataset(s)"
-          git push origin $NEW_BRANCH
+          # git add .
+          # git commit -m "Add dataset(s)"
+          # git push origin $NEW_BRANCH
           echo "**DEBUG**-Committed and pushed file"
           PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
 

From 1ee254972a02b749b4983f6a586bf654517b799e Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:00:10 -0800
Subject: [PATCH 45/68] fix: condense steps

---
 .github/workflows/pr.yml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 7428887d..a15063ee 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -253,12 +253,6 @@ jobs:
           echo "Contents of the datasets directory after copying:"
           ls veda-config/datasets/
 
-      - name: Set up Git
-        run: |
-          cd veda-config
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-
 
       # Creates a PR in veda-config with the following changes:
       # 1. the mdx files for all published collections
@@ -271,6 +265,9 @@ jobs:
           COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
         run: |
+          cd veda-config
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
           echo "Working directory $(pwd)"
 
           files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")

From d45aa7ea7f2394b210ff945d9aff1a91d9e465b8 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:02:52 -0800
Subject: [PATCH 46/68] fix: add commands to add and commit file

---
 .github/workflows/pr.yml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index a15063ee..bb96dbaf 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -282,13 +282,10 @@ jobs:
 
           # Update the env vars to staging based on env vars
           #sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
-          ls
-          git remote -v
-          git branch
           git status
-          # git add .
-          # git commit -m "Add dataset(s)"
-          # git push origin $NEW_BRANCH
+          git add .
+          git commit -m "Add dataset(s)"
+          git push origin $NEW_BRANCH
           echo "**DEBUG**-Committed and pushed file"
           PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
 

From 578119a3306794ec633720b8982959f2b3214d38 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:07:05 -0800
Subject: [PATCH 47/68] fix: modify commit message, add logs

---
 .github/workflows/pr.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index bb96dbaf..13fa0a53 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -250,9 +250,6 @@ jobs:
           ls ./ingestion-data/dataset-mdx/
           mkdir -p datasets
           find ingestion-data/dataset-mdx/ -name '*.mdx' -exec cp {} veda-config/datasets/ \;
-          echo "Contents of the datasets directory after copying:"
-          ls veda-config/datasets/
-
 
       # Creates a PR in veda-config with the following changes:
       # 1. the mdx files for all published collections
@@ -284,9 +281,10 @@ jobs:
           #sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
           git status
           git add .
-          git commit -m "Add dataset(s)"
+          git commit -m "feat: add MDX files for dataset(s) [Automated workflow]"
+          echo "**DEBUG**-Committed file"
           git push origin $NEW_BRANCH
-          echo "**DEBUG**-Committed and pushed file"
+          echo "**DEBUG**-Pushed file"
           PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT

From 19a9c734b025f2e597de6878612b6942b438b817 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:11:03 -0800
Subject: [PATCH 48/68] fix: authenticate git

---
 .github/workflows/pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 13fa0a53..01d0162a 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -265,6 +265,7 @@ jobs:
           cd veda-config
           git config --global user.name "github-actions[bot]"
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git remote set-url origin https://${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}@github.com/${{ env.VEDA_CONFIG_REPO }}
           echo "Working directory $(pwd)"
 
           files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")

From 21bc63b0d5e95fb3a258a206808afceaddc2934a Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:16:08 -0800
Subject: [PATCH 49/68] fix: add other steps to update comment in veda-data pr

---
 .github/workflows/pr.yml | 98 ++++++++++++++++++----------------------
 1 file changed, 45 insertions(+), 53 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 01d0162a..d8a0a3c0 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -251,11 +251,7 @@ jobs:
           mkdir -p datasets
           find ingestion-data/dataset-mdx/ -name '*.mdx' -exec cp {} veda-config/datasets/ \;
 
-      # Creates a PR in veda-config with the following changes:
-      # 1. the mdx files for all published collections
-      # 2. updates the stac/raster urls in .env file
-      # This step needs a GH_TOKEN that has permissions to create a PR in veda-config
-      - name: Create PR with changes
+      - name: Create veda-config PR with changes
         id: create-pr
         env:
           GITHUB_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
@@ -278,63 +274,59 @@ jobs:
           fi
           git checkout -b $NEW_BRANCH
 
-          # Update the env vars to staging based on env vars
-          #sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
           git status
           git add .
           git commit -m "feat: add MDX files for dataset(s) [Automated workflow]"
-          echo "**DEBUG**-Committed file"
           git push origin $NEW_BRANCH
-          echo "**DEBUG**-Pushed file"
-          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
+          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset(s) [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR creation succeeded"
 
       # Updates the comment with a link to the above PR
-      # - name: Update PR comment with PR creation result
-      #   if: success()
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-      #   run: |
-      #     PR_URL=${{ steps.create-pr.outputs.PR_URL }}
-      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-      #     UPDATED_BODY="$CURRENT_BODY
-
-      #     **A PR has been created with the dataset configuration: 🗺️ [PR link]($PR_URL)**"
-      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
-      # - name: Update PR comment on PR creation failure
-      #   if: failure() && steps.create-pr.outcome == 'failure'
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-      #   run: |
-      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-      #     UPDATED_BODY="$CURRENT_BODY
-
-      #     **Failed ❌ to create a PR with the dataset configuration. 😔 **"
-      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
-      # # If the workflow fails at any point, the PR comment will be updated
-      # - name: Update PR comment on overall workflow failure
-      #   if: failure() && steps.create-pr.outcome != 'failure'
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-      #   run: |
-      #     WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-      #     UPDATED_BODY="$CURRENT_BODY
-
-      #     # Output WORKFLOW_URL to logs for verification
-      #     echo "Workflow URL: $WORKFLOW_URL"
-
-      #     ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
-      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
-      #     echo "Updated Comment Body: $UPDATED_BODY"
+      - name: Update PR comment with PR creation result
+        if: success()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+        run: |
+         PR_URL=${{ steps.create-pr.outputs.PR_URL }}
+          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+          UPDATED_BODY="$CURRENT_BODY
+
+          **A PR has been created with the dataset configuration: 🗺️ [PR link]($PR_URL)**"
+          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+      - name: Update PR comment on PR creation failure
+        if: failure() && steps.create-pr.outcome == 'failure'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+        run: |
+          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+          UPDATED_BODY="$CURRENT_BODY
+
+          **Failed ❌ to create a PR with the dataset configuration. 😔 **"
+          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+      # If the workflow fails at any point, the PR comment will be updated
+      - name: Update PR comment on overall workflow failure
+        if: failure() && steps.create-pr.outcome != 'failure'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+        run: |
+          WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+          UPDATED_BODY="$CURRENT_BODY
+
+          # Output WORKFLOW_URL to logs for verification
+          echo "Workflow URL: $WORKFLOW_URL"
+
+          ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
+          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+          echo "Updated Comment Body: $UPDATED_BODY"
 
   publish-to-prod-on-pr-merge:
     if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true }}

From 1eb663e65757a9d8a4832f2cd7a3fe7fadd95e3d Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:22:19 -0800
Subject: [PATCH 50/68] fix: add debugging to new step

---
 .github/workflows/pr.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index d8a0a3c0..e11bfedc 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -290,6 +290,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
         run: |
+         echo "COMMENT_ID: $COMMENT_ID"
+         echo "${{ github.repository }}"
          PR_URL=${{ steps.create-pr.outputs.PR_URL }}
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
           UPDATED_BODY="$CURRENT_BODY

From a84ebe9f02e4530de669a67f5ab9b0d44ed137ee Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:28:05 -0800
Subject: [PATCH 51/68] fix: update env var setting

---
 .github/workflows/pr.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index e11bfedc..74d24ca8 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -288,7 +288,7 @@ jobs:
         if: success()
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+          COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
         run: |
          echo "COMMENT_ID: $COMMENT_ID"
          echo "${{ github.repository }}"
@@ -303,7 +303,7 @@ jobs:
         if: failure() && steps.create-pr.outcome == 'failure'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+          COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
         run: |
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
           UPDATED_BODY="$CURRENT_BODY
@@ -316,7 +316,7 @@ jobs:
         if: failure() && steps.create-pr.outcome != 'failure'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+          COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
         run: |
           WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')

From a914ca27f441fdd3ad2a2bac384c0f45c27580a4 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:33:47 -0800
Subject: [PATCH 52/68] fix: update comment-id references

---
 .github/workflows/pr.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 74d24ca8..7bd66791 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -255,7 +255,7 @@ jobs:
         id: create-pr
         env:
           GITHUB_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
-          COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment-id }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
         run: |
           cd veda-config
@@ -288,7 +288,7 @@ jobs:
         if: success()
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment-id }}
         run: |
          echo "COMMENT_ID: $COMMENT_ID"
          echo "${{ github.repository }}"
@@ -303,7 +303,7 @@ jobs:
         if: failure() && steps.create-pr.outcome == 'failure'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment-id }}
         run: |
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
           UPDATED_BODY="$CURRENT_BODY
@@ -316,7 +316,7 @@ jobs:
         if: failure() && steps.create-pr.outcome != 'failure'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment-id }}
         run: |
           WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')

From ebedabde10a9e382f898d3d1db16008fd86cf941 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:39:51 -0800
Subject: [PATCH 53/68] fix: update output to include commentId

---
 .github/workflows/pr.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 7bd66791..3e4d0136 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -30,6 +30,7 @@ jobs:
     environment: staging
     outputs:
       publishedCollections: ${{ steps.publish-collections.outputs.success_collections }}
+      commentId: ${{ steps.publish-collections.outputs.COMMENT_ID }}
     steps:
       - uses: actions/checkout@v4
 
@@ -255,7 +256,7 @@ jobs:
         id: create-pr
         env:
           GITHUB_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
-          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment-id }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
         run: |
           cd veda-config
@@ -288,7 +289,7 @@ jobs:
         if: success()
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment-id }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
         run: |
          echo "COMMENT_ID: $COMMENT_ID"
          echo "${{ github.repository }}"
@@ -303,7 +304,7 @@ jobs:
         if: failure() && steps.create-pr.outcome == 'failure'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment-id }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
         run: |
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
           UPDATED_BODY="$CURRENT_BODY
@@ -316,7 +317,7 @@ jobs:
         if: failure() && steps.create-pr.outcome != 'failure'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment-id }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
         run: |
           WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')

From 3d5548a44226140a5a7c736535295870bd07d992 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:43:45 -0800
Subject: [PATCH 54/68] fix: echo comment_id

---
 .github/workflows/pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 3e4d0136..abacea96 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -215,6 +215,7 @@ jobs:
       - name: Use output from dataset-publication-and-configuration
         run: |
           echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.publishedCollections }}"
+          echo "The comment_id output from the previous step is: ${{ needs.publish-new-datasets.outputs.commentId }}"
 
       # Creates a slim dataset mdx file for each collection based on the dataset config json
       - name: Create dataset mdx for given collections

From 094c9177ad985e72dc7c02faad414e01b0cc2204 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 16:04:38 -0800
Subject: [PATCH 55/68] fix: update comment id extraction

---
 .github/workflows/pr.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index abacea96..eb90a6aa 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -30,7 +30,7 @@ jobs:
     environment: staging
     outputs:
       publishedCollections: ${{ steps.publish-collections.outputs.success_collections }}
-      commentId: ${{ steps.publish-collections.outputs.COMMENT_ID }}
+      commentId: ${{ steps.init-comment.outputs.COMMENT_ID }}
     steps:
       - uses: actions/checkout@v4
 
@@ -212,7 +212,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Use output from dataset-publication-and-configuration
+      - name: Use output from publish-new-datasets
         run: |
           echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.publishedCollections }}"
           echo "The comment_id output from the previous step is: ${{ needs.publish-new-datasets.outputs.commentId }}"

From 302c90a7466449b125fcd092d136576a2e20ed25 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 16:08:04 -0800
Subject: [PATCH 56/68] fix: add anayeaye changes

---
 .github/workflows/pr.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index eb90a6aa..baec3a29 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -129,16 +129,17 @@ jobs:
 
           # Track successful publications
           all_failed=true
-          success_collections=()
+          declare -a success_collections=()
           status_message='### Collection Publication Status
           '
 
-          for file in "${ADDED_FILES[@]}"; do
+          for file in ${ADDED_FILES}; do
             echo $file
             if [ -f "$file" ]; then
               dataset_config=$(jq '.' "$file")
               collection_id=$(jq -r '.collection' "$file")
 
+              echo "Publishing $collection_id"
               response=$(curl -s -w "%{http_code}" -o response.txt -X POST "$publish_url" \
                 -H "Content-Type: application/json" \
                 -H "Authorization: Bearer $AUTH_TOKEN" \
@@ -172,7 +173,7 @@ jobs:
           fi
 
           # Output only successful collections to be used in subsequent steps
-          echo "success_collections=$(IFS=','; echo "${success_collections[*]}")" >> $GITHUB_OUTPUT
+          echo "success_collections=${success_collections[*]}" >> $GITHUB_OUTPUT
 
           # Update PR comment
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
@@ -224,8 +225,7 @@ jobs:
         run: |
           echo $PUBLISHED_COLLECTION_FILES
           pip install -r ./scripts/requirements.txt
-          for file in "${PUBLISHED_COLLECTION_FILES[@]}"
-          do
+          for file in ${PUBLISHED_COLLECTION_FILES}; do
             python3 ./scripts/generate-mdx.py "$file"
           done
 

From 3002432f5f394d52030c436140efd90c66aa27db Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 16:12:27 -0800
Subject: [PATCH 57/68] fix: clean up logging

---
 .github/workflows/pr.yml | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index baec3a29..de3dc1fd 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -216,7 +216,6 @@ jobs:
       - name: Use output from publish-new-datasets
         run: |
           echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.publishedCollections }}"
-          echo "The comment_id output from the previous step is: ${{ needs.publish-new-datasets.outputs.commentId }}"
 
       # Creates a slim dataset mdx file for each collection based on the dataset config json
       - name: Create dataset mdx for given collections
@@ -229,16 +228,8 @@ jobs:
             python3 ./scripts/generate-mdx.py "$file"
           done
 
-      - name: List files in workspace
-        run: |
-          echo "Listing all files to verify .mdx file generation:"
-          ls -la
-          ls ./ingestion-data/dataset-mdx/
-
       - name: Set up Variables
         run: |
-          echo "CONFIG_REPO_ORG: ${{ vars.VEDA_CONFIG_REPO_ORG }}"
-          echo "CONFIG_REPO_NAME: ${{ vars.VEDA_CONFIG_REPO_NAME }}"
           echo "VEDA_CONFIG_REPO=${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}" >> $GITHUB_ENV
 
       - name: Clone the Target Repository
@@ -264,7 +255,6 @@ jobs:
           git config --global user.name "github-actions[bot]"
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
           git remote set-url origin https://${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}@github.com/${{ env.VEDA_CONFIG_REPO }}
-          echo "Working directory $(pwd)"
 
           files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")
           hash=$(echo -n "$files_string" | md5sum | cut -d ' ' -f 1)
@@ -283,7 +273,7 @@ jobs:
           PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset(s) [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
-          echo "PR creation succeeded"
+          echo "PR creation succeeded!"
 
       # Updates the comment with a link to the above PR
       - name: Update PR comment with PR creation result
@@ -292,8 +282,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
         run: |
-         echo "COMMENT_ID: $COMMENT_ID"
-         echo "${{ github.repository }}"
          PR_URL=${{ steps.create-pr.outputs.PR_URL }}
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
           UPDATED_BODY="$CURRENT_BODY

From 461380583310bf7371a86a1ef80c53b0428b9c7c Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 18 Dec 2024 16:14:56 -0800
Subject: [PATCH 58/68] fix: update step name

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index de3dc1fd..6321dc0b 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -232,7 +232,7 @@ jobs:
         run: |
           echo "VEDA_CONFIG_REPO=${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}" >> $GITHUB_ENV
 
-      - name: Clone the Target Repository
+      - name: Clone veda-config repository
         run: |
           git clone https://github.com/${{ env.VEDA_CONFIG_REPO }}.git
           ls

From d2b8503da8be1e37aef5f5590747f39d954939ee Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 19 Dec 2024 11:18:28 -0800
Subject: [PATCH 59/68] fix: update to include github actor and try to extract
 collection

---
 .github/workflows/pr.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 6321dc0b..2b4d5b51 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -205,7 +205,7 @@ jobs:
           ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
           gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
 
-  create-mdx-files:
+  create-mdx-files-and-open-pr:
     runs-on: ubuntu-latest
     environment: staging
     needs: publish-new-datasets
@@ -225,12 +225,14 @@ jobs:
           echo $PUBLISHED_COLLECTION_FILES
           pip install -r ./scripts/requirements.txt
           for file in ${PUBLISHED_COLLECTION_FILES}; do
-            python3 ./scripts/generate-mdx.py "$file"
+            collection_id=$(python3 ./scripts/generate-mdx.py "$file")
+             echo "collection_id=$collection_id" >> $GITHUB_ENV
           done
 
       - name: Set up Variables
         run: |
           echo "VEDA_CONFIG_REPO=${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}" >> $GITHUB_ENV
+          echo "collection_id=${{ env.collection_id }}"
 
       - name: Clone veda-config repository
         run: |
@@ -270,7 +272,7 @@ jobs:
           git add .
           git commit -m "feat: add MDX files for dataset(s) [Automated workflow]"
           git push origin $NEW_BRANCH
-          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset(s) [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
+          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset(s) ${{ env.collection_id }} [Automated PR by ${{ github.actor }}]' --body-file <(echo "Add datasets (Automatically created by Github action, triggered by ${{ github.actor}} )"))
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR creation succeeded!"

From 10cfebd386919ef61668321af461c0f83d31413f Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 19 Dec 2024 11:19:16 -0800
Subject: [PATCH 60/68] fix: reformat file

---
 scripts/generate-mdx.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/generate-mdx.py b/scripts/generate-mdx.py
index 8ecc478d..e8726d53 100644
--- a/scripts/generate-mdx.py
+++ b/scripts/generate-mdx.py
@@ -125,3 +125,6 @@ def safe_open_w(path):
     )
     with safe_open_w(output_filepath) as ofile:
         ofile.write(new_content)
+
+    collection_id = input_data["collection"]
+    print(collection_id)

From bd3fcd3b437b81d1d2b7f42f70c1a9df6b3065f7 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 19 Dec 2024 11:42:34 -0800
Subject: [PATCH 61/68] fix: update to handle list of collections

---
 .github/workflows/pr.yml | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 2b4d5b51..cc2e16af 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -223,16 +223,18 @@ jobs:
           PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.publishedCollections }}
         run: |
           echo $PUBLISHED_COLLECTION_FILES
+          collection_ids="[]"
           pip install -r ./scripts/requirements.txt
           for file in ${PUBLISHED_COLLECTION_FILES}; do
             collection_id=$(python3 ./scripts/generate-mdx.py "$file")
-             echo "collection_id=$collection_id" >> $GITHUB_ENV
+            collection_ids=$(echo $collection_ids | jq --arg id "$collection_id" '. + [$id]')
           done
 
+          echo "collection_ids=${collection_ids}" >> $GITHUB_ENV
+
       - name: Set up Variables
         run: |
           echo "VEDA_CONFIG_REPO=${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}" >> $GITHUB_ENV
-          echo "collection_id=${{ env.collection_id }}"
 
       - name: Clone veda-config repository
         run: |
@@ -252,6 +254,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
           COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
+          COLLECTION_IDS: ${{ env.collection_ids }}
         run: |
           cd veda-config
           git config --global user.name "github-actions[bot]"
@@ -272,7 +275,13 @@ jobs:
           git add .
           git commit -m "feat: add MDX files for dataset(s) [Automated workflow]"
           git push origin $NEW_BRANCH
-          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset(s) ${{ env.collection_id }} [Automated PR by ${{ github.actor }}]' --body-file <(echo "Add datasets (Automatically created by Github action, triggered by ${{ github.actor}} )"))
+          body="### Add dataset(s) - [Automated PR by ${{ github.actor }}]\n\n"
+          for id in $COLLECTION_IDS; do
+            body="$body- $id\n"
+          done
+
+          echo "$body"
+          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset(s) $(echo '${{ env.collection_ids }}' | jq '.[0]') [Automated PR by ${{ github.actor }}]' --body $body"))
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR creation succeeded!"

From d1d6cd2aec098d8d0390766631729eb80db563c5 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 19 Dec 2024 11:48:29 -0800
Subject: [PATCH 62/68] fix: escape special chars

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index cc2e16af..86b52df6 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -227,7 +227,7 @@ jobs:
           pip install -r ./scripts/requirements.txt
           for file in ${PUBLISHED_COLLECTION_FILES}; do
             collection_id=$(python3 ./scripts/generate-mdx.py "$file")
-            collection_ids=$(echo $collection_ids | jq --arg id "$collection_id" '. + [$id]')
+            collection_ids=$(echo $collection_ids | jq --arg id "$(echo $collection_id | sed 's/"/\\"/g')" '. + [$id]')
           done
 
           echo "collection_ids=${collection_ids}" >> $GITHUB_ENV

From 86d7422eb8696ac9af5e5ed5d36c0a92fcf8d3d9 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 19 Dec 2024 11:51:23 -0800
Subject: [PATCH 63/68] fix: add more cleaning to collection id

---
 .github/workflows/pr.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 86b52df6..7153ee68 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -227,7 +227,9 @@ jobs:
           pip install -r ./scripts/requirements.txt
           for file in ${PUBLISHED_COLLECTION_FILES}; do
             collection_id=$(python3 ./scripts/generate-mdx.py "$file")
-            collection_ids=$(echo $collection_ids | jq --arg id "$(echo $collection_id | sed 's/"/\\"/g')" '. + [$id]')
+            collection_id=$(echo "$collection_id" | sed 's/^["\s]*//;s/["\s]*$//')
+            echo "Processed collection ID: $collection_id"
+            collection_ids=$(echo $collection_ids | jq --arg id "$collection_id" '. + [$id]')
           done
 
           echo "collection_ids=${collection_ids}" >> $GITHUB_ENV

From f7ae3c5c4e7211e2f729bcf450e9b31471bf334b Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 19 Dec 2024 11:58:01 -0800
Subject: [PATCH 64/68] fix: add debugging for collection id extraction

---
 .github/workflows/pr.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 7153ee68..dc7d5360 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -232,6 +232,8 @@ jobs:
             collection_ids=$(echo $collection_ids | jq --arg id "$collection_id" '. + [$id]')
           done
 
+          collection_ids=$(echo "$collection_ids" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+          echo "Final collection_ids: $collection_ids"
           echo "collection_ids=${collection_ids}" >> $GITHUB_ENV
 
       - name: Set up Variables

From cc0d8be88cee56d133581e6a889cf7fd59da5947 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 19 Dec 2024 12:05:01 -0800
Subject: [PATCH 65/68] fix: try another approach for updating gh body

---
 .github/workflows/pr.yml | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index dc7d5360..8e4b831f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -223,16 +223,16 @@ jobs:
           PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.publishedCollections }}
         run: |
           echo $PUBLISHED_COLLECTION_FILES
-          collection_ids="[]"
+          collection_ids=""
           pip install -r ./scripts/requirements.txt
           for file in ${PUBLISHED_COLLECTION_FILES}; do
             collection_id=$(python3 ./scripts/generate-mdx.py "$file")
             collection_id=$(echo "$collection_id" | sed 's/^["\s]*//;s/["\s]*$//')
             echo "Processed collection ID: $collection_id"
-            collection_ids=$(echo $collection_ids | jq --arg id "$collection_id" '. + [$id]')
+            collection_ids="$collection_ids$collection_id,"
           done
-
-          collection_ids=$(echo "$collection_ids" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+          # Remove trailing comma
+          collection_ids=${collection_ids%,}
           echo "Final collection_ids: $collection_ids"
           echo "collection_ids=${collection_ids}" >> $GITHUB_ENV
 
@@ -258,7 +258,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }}
           COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
-          COLLECTION_IDS: ${{ env.collection_ids }}
         run: |
           cd veda-config
           git config --global user.name "github-actions[bot]"
@@ -279,11 +278,16 @@ jobs:
           git add .
           git commit -m "feat: add MDX files for dataset(s) [Automated workflow]"
           git push origin $NEW_BRANCH
-          body="### Add dataset(s) - [Automated PR by ${{ github.actor }}]\n\n"
-          for id in $COLLECTION_IDS; do
-            body="$body- $id\n"
+
+          # Convert the comma-separated list into bullet points
+          collection_bullet_points=""
+          IFS=',' read -ra IDs <<< "$collection_ids"
+          for id in "${IDs[@]}"; do
+            collection_bullet_points+="- $id\n"
           done
 
+          body="### Add dataset(s) - [Automated PR by ${{ github.actor }}]\n $collection_bullet_points\n"
+
           echo "$body"
           PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset(s) $(echo '${{ env.collection_ids }}' | jq '.[0]') [Automated PR by ${{ github.actor }}]' --body $body"))
 

From c6dfa313ab11d578fc41b1cd7e69f9467f642568 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 19 Dec 2024 12:07:59 -0800
Subject: [PATCH 66/68] fix: update pr_url

---
 .github/workflows/pr.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 8e4b831f..cdb6f75d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -289,7 +289,8 @@ jobs:
           body="### Add dataset(s) - [Automated PR by ${{ github.actor }}]\n $collection_bullet_points\n"
 
           echo "$body"
-          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title 'Add dataset(s) $(echo '${{ env.collection_ids }}' | jq '.[0]') [Automated PR by ${{ github.actor }}]' --body $body"))
+          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title "Add dataset(s) $(echo '${{ env.collection_ids }}' | jq '.[0]') [Automated PR by ${{ github.actor }}]" --body "$body")
+
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR creation succeeded!"

From 795a031e6b34fdb095cbbfd5455916e307738773 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 19 Dec 2024 12:14:12 -0800
Subject: [PATCH 67/68] fix: cleanup by extracting pr title and body

---
 .github/workflows/pr.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index cdb6f75d..3d9628f9 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -282,15 +282,18 @@ jobs:
           # Convert the comma-separated list into bullet points
           collection_bullet_points=""
           IFS=',' read -ra IDs <<< "$collection_ids"
+
+          # Extract the first collection ID
+          first_collection_id="${IDs[0]}"
           for id in "${IDs[@]}"; do
             collection_bullet_points+="- $id\n"
           done
 
-          body="### Add dataset(s) - [Automated PR by ${{ github.actor }}]\n $collection_bullet_points\n"
+          pr_title="Add dataset(s) - $first_collection_id [Automated PR by ${{ github.actor }}]"
+          body="### Add dataset(s) - $first_collection_id [Automated PR by ${{ github.actor }}]\n\n$collection_bullet_points"
 
           echo "$body"
-          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title "Add dataset(s) $(echo '${{ env.collection_ids }}' | jq '.[0]') [Automated PR by ${{ github.actor }}]" --body "$body")
-
+          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title "$pr_title" --body "$body")
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR creation succeeded!"

From 1c726097cfd387e73e9e5bdae50ce2c17bb4ed22 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 19 Dec 2024 14:45:02 -0800
Subject: [PATCH 68/68] fix: use -e flag for echo to fix newlines

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 3d9628f9..8e9baf87 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -293,7 +293,7 @@ jobs:
           body="### Add dataset(s) - $first_collection_id [Automated PR by ${{ github.actor }}]\n\n$collection_bullet_points"
 
           echo "$body"
-          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title "$pr_title" --body "$body")
+          PR_URL=$(GITHUB_TOKEN=${{ secrets.VEDA_CONFIG_REPO_ACCESS_TOKEN }} gh pr create -R ${{ env.VEDA_CONFIG_REPO }} -H $NEW_BRANCH -B develop --title "$pr_title" --body "$(echo -e "$body")")
 
           echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
           echo "PR creation succeeded!"