Add code to generate a Python baseline snapshot

.github/workflows/make-python-baseline-snapshot.yml

@@ -0,0 +1,70 @@
+name: Make Python Baseline Snapshot
+
+permissions:
+    id-token: write # This is required for requesting the JWT
+    contents: read  # This is required for actions/checkout
+
+on:
+    workflow_dispatch:
+        inputs:
+            tag:
+                description: The release tag/version string
+            dry-run:
+                description: Actually upload or just test build step?
+                default: false
+                type: boolean
+    push:
+      branches:
+        - 'hoodmane/pyodide-load-from-file'
+
+jobs:
+  version:
+    outputs:
+      version: ${{ steps.echo.outputs.version }}
+    # version job uses ubuntu 24.04, this way we don't have to install the updated clang while
+    # the build job uses 20.04 for libc compatibility.
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - id: echo
+        run: |
+          echo "::set-output name=version::1.$(cat src/workerd/io/supported-compatibility-date.txt | tr -d '-').0"
+
+  build:
+    runs-on: ubuntu-20.04
+    name: build Python runtime
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - name: Setup Linux
+        if: runner.os == 'Linux'
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          wget https://apt.llvm.org/llvm.sh
+          sed -i '/apt-get install/d' llvm.sh
+          chmod +x llvm.sh
+          sudo ./llvm.sh 15
+          sudo apt-get install -y --no-install-recommends clang-15 lld-15 libunwind-15 libc++abi1-15 libc++1-15 libc++-15-dev
+          echo "build:linux --action_env=CC=/usr/lib/llvm-15/bin/clang --action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
+          echo "build:linux --host_action_env=CC=/usr/lib/llvm-15/bin/clang --host_action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
+      - name: Configure download mirrors
+        shell: bash
+        run: |
+          if [ ! -z "${{ secrets.WORKERS_MIRROR_URL }}" ] ; then
+            # Strip comment in front of WORKERS_MIRROR_URL, then substitute secret to use it.
+            sed -e '/WORKERS_MIRROR_URL/ { s@# *@@; s@WORKERS_MIRROR_URL@${{ secrets.WORKERS_MIRROR_URL }}@; }' -i.bak WORKSPACE
+          fi
+      - name: Bazel build
+        # Strip debug info here – we don't generate debug info but some is pulled in from external
+        # static libraries, for example the Rust STL. This is equivalent to the -Wl,-S linker
+        # option, symbols will not be removed.
+        run: |
+          bazelisk build --config=release_linux @workerd//src/workerd/server:workerd
+          cd samples/pyodide
+          ../../bazel-bin/src/workerd/server/workerd test config.capnp \
+            --python-save-baseline-snapshot \
+            --pyodide-package-disk-cache-dir=save_dir \
+          | tee log.txt
+          cat log.txt | grep inputs.tag
+          cp save_dir/snapshot.bin pyodide_${{ inputs.tag }}.capnp.bin

src/workerd/io/compatibility-date.capnp

@@ -424,7 +424,7 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
   pythonWorkers @43 :Bool
       $compatEnableFlag("python_workers")
       $pythonSnapshotRelease(pyodide = "0.26.0a2", pyodideRevision = "2024-03-01",
-          packages = "2024-03-01", backport = 0)
+          packages = "2024-03-01", backport = 2)
       $impliedByAfterDate(name = "pythonWorkersDevPyodide", date = "2000-01-01");
   # Enables Python Workers. Access to this flag is not restricted, instead bundles containing
   # Python modules are restricted in EWC.

src/workerd/server/workerd-api.c++

@@ -441,6 +441,7 @@ void writePyodideBundleFileToDisk(const kj::Maybe<kj::Own<const kj::Directory>>&
 
 kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
     const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version) {
+  KJ_DBG(version);
   KJ_IF_SOME(version, pyConfig.pyodideBundleManager.getPyodideBundle(version)) {
     return version;
   }
@@ -477,12 +478,13 @@ kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
       kj::HttpHeaders headers(table);
 
       kj::String url =
-          kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide_",
-              version, ".capnp.bin");
+          kj::str("https://www.googleapis.com/storage/v1/b/cloudflare-edgeworker-python-packages/o/python-runtime-capnp-bin%2Fpyodide_",
+              version, ".capnp.bin?alt=media");
 
       auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers);
 
       auto res = req.response.wait(io.waitScope);
+      KJ_DBG(url, res.statusCode);
       auto body = res.body->readAllBytes().wait(io.waitScope);
 
       writePyodideBundleFileToDisk(pyConfig.pyodideDiskCacheRoot, version, body);