moved to merged bin for flasher

J-A-A-M · Dec 4, 2024 · f13ce17 · f13ce17
1 parent 126fc3d
commit f13ce17
Show file tree

Hide file tree

Showing 23 changed files with 182 additions and 48 deletions.
diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml
@@ -33,6 +33,7 @@ jobs:
           mkdir -p ${{ github.workspace }}/bin_beta/
           cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware.bin ${{ github.workspace }}/bin_beta/${{ env.BETA_VERSION }}.bin
           cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware.bin ${{ github.workspace }}/JAAM_${{ env.BETA_VERSION }}.bin
+          cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware-merged.bin ${{ github.workspace }}/JAAM_${{ env.BETA_VERSION }}_FULL.bin
       # Disabled for now
       # - name: Leave only 3 recent BETA builds
       #   run: |
@@ -42,7 +43,7 @@ jobs:
       - name: Upload flasher to Github Pages
         uses: ./.github/workflows/upload-pages
         with:
-          beta_binary_path: ${{ github.workspace }}/JAAM_${{ env.BETA_VERSION }}.bin
+          beta_binary_path: ${{ github.workspace }}/JAAM_${{ env.BETA_VERSION }}_FULL.bin
           version: ${{ inputs.release-version }}
           beta_build: ${{ github.run_number }}  
       - name: Commit changes
@@ -54,7 +55,7 @@ jobs:
       - name: Create Beta Pre Release
         uses: ncipollo/release-action@v1
         with:
-          artifacts: ${{ github.workspace }}/JAAM_${{ env.BETA_VERSION }}.bin
+          artifacts: ${{ github.workspace }}/JAAM_${{ env.BETA_VERSION }}.bin,${{ github.workspace }}/JAAM_${{ env.BETA_VERSION }}_FULL.bin
           name: ${{ env.RELEASE_TITLE }}
           commit: ${{ steps.commit_changes.outputs.commit_hash }}
           tag: ${{ env.BETA_VERSION }}

diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml
@@ -17,11 +17,17 @@
           - name: Copy bin file
             run: |
               cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware.bin ${{ github.workspace }}/firmware.bin
+              cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware-merged.bin ${{ github.workspace }}/firmware_FULL.bin
           - name: Upload firmware
             uses: actions/upload-artifact@v4
             with:
               name: firmware.bin
               path: ${{ github.workspace }}/firmware.bin
+          - name: Upload full firmware
+            uses: actions/upload-artifact@v4
+            with:
+              name: firmware_FULL.bin
+              path: ${{ github.workspace }}/firmware_FULL.bin
     compile_firmware_lite:
         runs-on: ubuntu-latest
         steps:
@@ -36,11 +42,17 @@
           - name: Copy bin file
             run: |
               cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware.bin ${{ github.workspace }}/firmware-lite.bin
+              cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware-merged.bin ${{ github.workspace }}/firmware-lite_FULL.bin
           - name: Upload firmware lite
             uses: actions/upload-artifact@v4
             with:
               name: firmware-lite.bin
               path: ${{ github.workspace }}/firmware-lite.bin
+          - name: Upload full firmware lite
+            uses: actions/upload-artifact@v4
+            with:
+              name: firmware-lite_FULL.bin
+              path: ${{ github.workspace }}/firmware-lite_FULL.bin
     compile_jaam2_test:
         runs-on: ubuntu-latest
         steps:
@@ -56,7 +68,7 @@
           - name: Copy bin file
             run: |
               cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware.bin ${{ github.workspace }}/firmware-jaam2-test.bin
-          - name: Upload firmware lite
+          - name: Upload JAAM 2 test firmware
             uses: actions/upload-artifact@v4
             with:
               name: firmware-jaam2-test.bin

diff --git a/.github/workflows/firmware-compile/action.yml b/.github/workflows/firmware-compile/action.yml
@@ -24,4 +24,4 @@ runs:
       run: pip install --upgrade platformio
     - name: Build ${{ inputs.project-folder }}
       shell: bash
-      run: pio run -d ${{ inputs.project-folder }}
+      run: pio run -d ${{ inputs.project-folder }} -t mergebin
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -26,11 +26,17 @@ jobs:
       - name: Copy bin to upload
         run: |
           cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware.bin ${{ github.workspace }}/JAAM_${{ inputs.release-version }}.bin
+          cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware-merged.bin ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_FULL.bin
       - name: Upload firmware
         uses: actions/upload-artifact@v4
         with:
           name: JAAM_${{ inputs.release-version }}.bin
           path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}.bin
+      - name: Upload full firmware
+        uses: actions/upload-artifact@v4
+        with:
+          name: JAAM_${{ inputs.release-version }}_FULL.bin
+          path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_FULL.bin
   build_lite:
     if: ${{ github.ref_name == 'master' }}
     runs-on: ubuntu-latest
@@ -46,11 +52,17 @@ jobs:
       - name: Copy bin to upload
         run: |
           cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware.bin ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite.bin
+          cp -f ${{ github.workspace }}/firmware/.pio/build/firmware/firmware-merged.bin ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite_FULL.bin
       - name: Upload firmware
         uses: actions/upload-artifact@v4
         with:
           name: JAAM_${{ inputs.release-version }}_lite.bin
           path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite.bin
+      - name: Upload full firmware
+        uses: actions/upload-artifact@v4
+        with:
+          name: JAAM_${{ inputs.release-version }}_lite_FULL.bin
+          path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite_FULL.bin
   release_firmware:
     needs: [build, build_lite]
     runs-on: ubuntu-latest
@@ -71,11 +83,21 @@ jobs:
         with:
           name: JAAM_${{ inputs.release-version }}.bin
           path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}.bin
+      - name: Download full firmware
+        uses: actions/download-artifact@v4
+        with:
+          name: JAAM_${{ inputs.release-version }}_FULL.bin
+          path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_FULL.bin
       - name: Download firmware lite
         uses: actions/download-artifact@v4
         with:
           name: JAAM_${{ inputs.release-version }}_lite.bin
           path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite.bin
+      - name: Download full firmware lite
+        uses: actions/download-artifact@v4
+        with:
+          name: JAAM_${{ inputs.release-version }}_lite_FULL.bin
+          path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite_FULL.bin
       - name: Copy release files to bin folder
         run: |
           mkdir -p ${{ github.workspace }}/bin/
@@ -93,8 +115,8 @@ jobs:
       - name: Upload flasher to Github Pages
         uses: ./.github/workflows/upload-pages
         with:
-          binary_path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}.bin/JAAM_${{ inputs.release-version }}.bin
-          lite_binary_path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite.bin/JAAM_${{ inputs.release-version }}_lite.bin
+          binary_path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_FULL.bin/JAAM_${{ inputs.release-version }}_FULL.bin
+          lite_binary_path: ${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite_FULL.bin/JAAM_${{ inputs.release-version }}_lite_FULL.bin
           version: ${{ inputs.release-version }}
       - name: Commit changes
         id: commit_changes
@@ -105,7 +127,7 @@ jobs:
       - name: Create Release
         uses: ncipollo/release-action@v1
         with:
-          artifacts: "${{ github.workspace }}/JAAM_${{ inputs.release-version }}.bin/JAAM_${{ inputs.release-version }}.bin,${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite.bin/JAAM_${{ inputs.release-version }}_lite.bin"
+          artifacts: "${{ github.workspace }}/JAAM_${{ inputs.release-version }}.bin/JAAM_${{ inputs.release-version }}.bin,${{ github.workspace }}/JAAM_${{ inputs.release-version }}_FULL.bin/JAAM_${{ inputs.release-version }}_FULL.bin,${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite.bin/JAAM_${{ inputs.release-version }}_lite.bin,${{ github.workspace }}/JAAM_${{ inputs.release-version }}_lite_FULL.bin/JAAM_${{ inputs.release-version }}_lite_FULL.bin"
           name: "${{ inputs.release-version }}: ${{ inputs.release-title }}"
           commit: ${{ steps.commit_changes.outputs.commit_hash }}
           tag: ${{ inputs.release-version }}

diff --git a/.github/workflows/upload-pages/action.yml b/.github/workflows/upload-pages/action.yml
@@ -19,11 +19,11 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    - name: Copy bin to flesher
+    - name: Copy bin to flasher
       if: ${{ inputs.binary_path != '' }}
       shell: bash
       run: |
-        cp -f ${{ inputs.binary_path }} ${{ github.workspace }}/flasher/bins/jaam.bin
+        cp -f ${{ inputs.binary_path }} ${{ github.workspace }}/flasher/bins/jaam.fw.bin
     - name: Change version in flasher
       if: ${{ inputs.binary_path != '' }}
       shell: bash
@@ -34,7 +34,7 @@ runs:
       if: ${{ inputs.beta_binary_path != '' }}
       shell: bash
       run: |
-        cp -f ${{ inputs.beta_binary_path }} ${{ github.workspace }}/flasher/bins/jaam_beta.bin
+        cp -f ${{ inputs.beta_binary_path }} ${{ github.workspace }}/flasher/bins/jaam.fw.beta.bin
     - name: Change beta version in flasher
       if: ${{ inputs.beta_binary_path != '' }}
       shell: bash
@@ -45,7 +45,7 @@ runs:
       if: ${{ inputs.lite_binary_path != '' }}
       shell: bash
       run: |
-        cp -f ${{ inputs.lite_binary_path }} ${{ github.workspace }}/flasher/bins/jaam_lite.bin
+        cp -f ${{ inputs.lite_binary_path }} ${{ github.workspace }}/flasher/bins/jaam.fw.lite.bin
     - name: Change lite version in flasher
       if: ${{ inputs.lite_binary_path != '' }}
       shell: bash

diff --git a/deploy/update_server/update_server.py b/deploy/update_server/update_server.py
@@ -38,30 +38,31 @@ async def server_error(request: Request, exc: HTTPException):
 
 exception_handlers = {404: not_found, 500: server_error}
 
+
 def bin_sort(bin):
-    if (bin.startswith("latest")):
+    if bin.startswith("latest"):
         return (100, 0, 0, 0)
     version = bin.removesuffix(".bin")
     fw_beta = version.split("-")
     fw = fw_beta[0]
-    if (len(fw_beta) == 1):
+    if len(fw_beta) == 1:
         beta = 1000
     else:
         beta = int(fw_beta[1].removeprefix("b"))
-    
+
     major_minor_patch = fw.split(".")
     major = int(major_minor_patch[0])
-    if (len(major_minor_patch) == 1):
+    if len(major_minor_patch) == 1:
         minor = 0
         patch = 0
-    elif (len(major_minor_patch) == 2):
+    elif len(major_minor_patch) == 2:
         minor = int(major_minor_patch[1])
         patch = 0
     else:
         minor = int(major_minor_patch[1])
         patch = int(major_minor_patch[2])
 
-    print(f'{bin} -> {major}.{minor}.{patch}.{beta}')
+    print(f"{bin} -> {major}.{minor}.{patch}.{beta}")
     return (major, minor, patch, beta)
 
 
@@ -88,25 +89,25 @@ async def list_beta(request):
 
 async def update(request):
     if request.path_params["filename"] == "latest":
-         filenames = sorted(
+        filenames = sorted(
             [file for file in os.listdir(shared_path) if os.path.isfile(os.path.join(shared_path, file))],
-            key=bin_sort, 
-            reverse=True
-            )
-         filenames = list(filter(lambda filename: not filename.startswith("4."), filenames))
-         return FileResponse(f'{shared_path}/{filenames[0]}.bin')
+            key=bin_sort,
+            reverse=True,
+        )
+        filenames = list(filter(lambda filename: not filename.startswith("4."), filenames))
+        return FileResponse(f"{shared_path}/{filenames[0]}.bin")
     return FileResponse(f'{shared_path}/{request.path_params["filename"]}.bin')
 
 
 async def update_beta(request):
     if request.path_params["filename"] == "latest_beta":
-         filenames = sorted(
+        filenames = sorted(
             [file for file in os.listdir(shared_beta_path) if os.path.isfile(os.path.join(shared_beta_path, file))],
-            key=bin_sort, 
-            reverse=True
-            )
-         filenames = list(filter(lambda filename: not filename.startswith("4."), filenames))
-         return FileResponse(f'{shared_path}/{filenames[0]}.bin')
+            key=bin_sort,
+            reverse=True,
+        )
+        filenames = list(filter(lambda filename: not filename.startswith("4."), filenames))
+        return FileResponse(f"{shared_path}/{filenames[0]}.bin")
     return FileResponse(f'{shared_beta_path}/{request.path_params["filename"]}.bin')
 
 

diff --git a/deploy/websocket_server/websocket_server.py b/deploy/websocket_server/websocket_server.py
@@ -86,22 +86,22 @@ class AlertVersion:
 
 
 def bin_sort(bin):
-    if (bin.startswith("latest")):
+    if bin.startswith("latest"):
         return (100, 0, 0, 0)
     version = bin.removesuffix(".bin")
     fw_beta = version.split("-")
     fw = fw_beta[0]
-    if (len(fw_beta) == 1):
+    if len(fw_beta) == 1:
         beta = 0
     else:
         beta = int(fw_beta[1].removeprefix("b"))
-    
+
     major_minor_patch = fw.split(".")
     major = int(major_minor_patch[0])
-    if (len(major_minor_patch) == 1):
+    if len(major_minor_patch) == 1:
         minor = 0
         patch = 0
-    elif (len(major_minor_patch) == 2):
+    elif len(major_minor_patch) == 2:
         minor = int(major_minor_patch[1])
         patch = 0
     else:
@@ -110,6 +110,7 @@ def bin_sort(bin):
 
     return (major, minor, patch, beta)
 
+
 async def alerts_data(websocket, client, shared_data, alert_version):
     client_ip, client_port = websocket.remote_address
     while True:
@@ -153,7 +154,11 @@ async def alerts_data(websocket, client, shared_data, alert_version):
                 client["weather"] = shared_data.weather_v1
             if client["bins"] != shared_data.bins:
                 temp_bins = list(json.loads(shared_data.bins))
-                if client["firmware"].startswith("3.") or client["firmware"].startswith("2.") or client["firmware"].startswith("1."):
+                if (
+                    client["firmware"].startswith("3.")
+                    or client["firmware"].startswith("2.")
+                    or client["firmware"].startswith("1.")
+                ):
                     temp_bins = list(filter(lambda bin: not bin.startswith("4."), temp_bins))
                     temp_bins.append("latest.bin")
                 temp_bins.sort(key=bin_sort, reverse=True)
@@ -163,7 +168,11 @@ async def alerts_data(websocket, client, shared_data, alert_version):
                 client["bins"] = shared_data.bins
             if client["test_bins"] != shared_data.test_bins:
                 temp_bins = list(json.loads(shared_data.test_bins))
-                if client["firmware"].startswith("3.") or client["firmware"].startswith("2.") or client["firmware"].startswith("1."):
+                if (
+                    client["firmware"].startswith("3.")
+                    or client["firmware"].startswith("2.")
+                    or client["firmware"].startswith("1.")
+                ):
                     temp_bins = list(filter(lambda bin: not bin.startswith("4."), temp_bins))
                     temp_bins.append("latest_beta.bin")
                 temp_bins.sort(key=bin_sort, reverse=True)

diff --git a/firmware/merge-bin.py b/firmware/merge-bin.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python3
+
+# Adds PlatformIO post-processing to merge all the ESP flash images into a single image.
+
+import os
+
+Import("env", "projenv")
+
+board_config = env.BoardConfig()
+firmware_bin = "${BUILD_DIR}/${PROGNAME}.bin"
+merged_bin = os.environ.get("MERGED_BIN_PATH", "${BUILD_DIR}/${PROGNAME}-merged.bin")
+
+
+def merge_bin_action(source, target, env):
+    flash_images = [
+        *env.Flatten(env.get("FLASH_EXTRA_IMAGES", [])),
+        "$ESP32_APP_OFFSET",
+        source[0].get_abspath(),
+    ]
+    merge_cmd = " ".join(
+        [
+            '"$PYTHONEXE"',
+            '"$OBJCOPY"',
+            "--chip",
+            board_config.get("build.mcu", "esp32"),
+            "merge_bin",
+            "-o",
+            merged_bin,
+            "--flash_mode",
+            board_config.get("build.flash_mode", "dio"),
+            "--flash_freq",
+            "${__get_board_f_flash(__env__)}",
+            "--flash_size",
+            board_config.get("upload.flash_size", "4MB"),
+            *flash_images,
+        ]
+    )
+    env.Execute(merge_cmd)
+
+
+env.AddCustomTarget(
+    name="mergebin",
+    dependencies=firmware_bin,
+    actions=merge_bin_action,
+    title="Merge binary",
+    description="Build combined image",
+    always_build=True,
+)
diff --git a/firmware/platformio.ini b/firmware/platformio.ini
@@ -14,6 +14,7 @@ board = esp32dev
 framework = arduino
 monitor_speed = 115200
 board_build.partitions = min_spiffs.csv
+extra_scripts = merge-bin.py
 lib_deps = 
 	https://github.com/MatheusAlvesA/ArduinoAsync.git
 	adafruit/Adafruit [email protected]

diff --git a/flasher/bins/jaam.boot.bin b/flasher/bins/jaam.boot.bin
diff --git a/flasher/bins/jaam.bootloader.bin b/flasher/bins/jaam.bootloader.bin
diff --git a/flasher/bins/jaam.bootloader.old.bin b/flasher/bins/jaam.bootloader.old.bin
diff --git a/flasher/bins/jaam.fw.beta.bin b/flasher/bins/jaam.fw.beta.bin
diff --git a/flasher/bins/jaam.bin → flasher/bins/jaam.fw.bin b/flasher/bins/jaam.bin → flasher/bins/jaam.fw.bin
diff --git a/flasher/bins/jaam_lite.bin → flasher/bins/jaam.fw.lite.bin b/flasher/bins/jaam_lite.bin → flasher/bins/jaam.fw.lite.bin
diff --git a/flasher/bins/jaam.partitions.bin b/flasher/bins/jaam.partitions.bin
diff --git a/flasher/bins/jaam.partitions.old.bin b/flasher/bins/jaam.partitions.old.bin
diff --git a/flasher/bins/jaam_beta.bin b/flasher/bins/jaam_beta.bin
diff --git a/flasher/manifest-beta.json b/flasher/manifest-beta.json
@@ -9,10 +9,7 @@
         "chipFamily": "ESP32",
         "improv": false,
         "parts": [
-          { "path": "bins/jaam.bootloader.bin", "offset": 4096 },
-          { "path": "bins/jaam.partitions.bin", "offset": 32768 },
-          { "path": "bins/jaam.boot.bin", "offset": 57344 },
-          { "path": "bins/jaam_beta.bin", "offset": 65536 }
+          { "path": "bins/jaam.fw.beta.bin", "offset": 0 }
         ]
       }
     ]