Use ReadyToRun to improve performance (#261)

Motivation ---------- JIT of the C# compiler is very expensive when running and the cost can be significantly reduced by publishing the command line app ReadyToRun. Modifications ------------- - Set PublishReadyToRun on Yardarm.CommandLine - Update Dockerfile to publish the app rather than install it as a global tool so it gets the ReadyToRun build output - Fixup .dockerignore to reduce the size of the build context - Update SDK to more easily support multiple runtime identifiers and to use the publish output so it gets the ReadyToRun build output - Split the build in GitHub actions to separately build the main app and the SDK so that the SDK variant may be built on Windows where all target runtime identifers are supported for ReadyToRun - Update the Docker build to build both x64 and arm64 variants of the image
CenterEdge · Oct 12, 2024 · 9ea2841 · 9ea2841
1 parent d27eabb
commit 9ea2841
Show file tree

Hide file tree

Showing 6 changed files with 139 additions and 87 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,4 +1,6 @@
-src/.vs
-src/*/bin
-src/*/obj
+**/.vs
+**/bin
+**/obj
+**/artifacts
+**/*.user
 .github
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -8,33 +8,43 @@ on:
     - release-*
 
 jobs:
-  build:
+  version:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.gitversion.outputs.nuGetVersionV2 }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 6.0.x
+
+      - name: Install GitVersion
+        uses: gittools/actions/gitversion/setup@v1
+        with:
+          versionSpec: "5.12.0"
+      - name: Determine Version
+        id: gitversion
+        uses: gittools/actions/gitversion/execute@v1
+        with:
+          useConfigFile: true
+          configFilePath: "GitVersion.yml"
 
+  build:
     runs-on: ubuntu-latest
+    needs: version
 
     steps:
     - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
 
     - name: Setup .NET Core
       uses: actions/setup-dotnet@v4
       with:
-        # 6.0 needed for GitVersion
-        dotnet-version: |
-          6.0.x
-          8.0.x
-
-    - name: Install GitVersion
-      uses: gittools/actions/gitversion/setup@v1
-      with:
-        versionSpec: "5.12.0"
-    - name: Determine Version
-      id: gitversion
-      uses: gittools/actions/gitversion/execute@v1
-      with:
-        useConfigFile: true
-        configFilePath: "GitVersion.yml"
+        dotnet-version: 8.0.x
 
     - uses: actions/cache@v4
       with:
@@ -45,25 +55,19 @@ jobs:
     - name: Install dependencies
       working-directory: ./src/main
       run: dotnet restore
-    - name: Install SDK dependencies
-      working-directory: ./src/sdk
-      run: dotnet restore Yardarm.Sdk.sln
 
     - name: Build
       working-directory: ./src/main
-      run: dotnet build --configuration Release -p:Version=${{ steps.gitversion.outputs.nuGetVersionV2  }} --no-restore
+      run: dotnet build --configuration Release -p:Version=${{ needs.version.outputs.version  }} --no-restore
     - name: Test
       working-directory: ./src/main
       run: dotnet test --configuration Release --no-build --verbosity normal --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true"
-    - name: Build SDK
-      working-directory: ./src/sdk
-      run: dotnet build --configuration Release -p:Version=${{ steps.gitversion.outputs.nuGetVersionV2  }} --no-restore Yardarm.Sdk.sln
     - name: Test Generate
       run: ./scripts/generate.sh
 
     - name: Pack
       working-directory: ./src/main
-      run: dotnet pack --configuration Release -p:Version=${{ steps.gitversion.outputs.nuGetVersionV2  }} --no-build
+      run: dotnet pack --configuration Release -p:Version=${{ needs.version.outputs.version  }} --no-build
       # Note: SDK is packed by build above, doesn't need to be packed here
     - name: Push to NuGet.org
       if: ${{ startsWith(github.ref, 'refs/tags/release/') }}
@@ -76,45 +80,68 @@ jobs:
       run: |
         dotnet nuget push **/*.nupkg --api-key ${{ secrets.GITHUB_TOKEN }} --source https://nuget.pkg.github.com/CenterEdge/index.json --skip-duplicate
 
-  docker:
-
-    runs-on: ubuntu-latest
-
-    needs: build
+  build-sdk:
+    # SDK must be built on Windows to support building ReadyToRun images for multiple platforms
+    runs-on: windows-latest
+    needs: version
 
     steps:
     - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
 
     - name: Setup .NET Core
       uses: actions/setup-dotnet@v4
       with:
-        # 6.0 needed for GitVersion
-        dotnet-version: |
-          6.0.x
-    - name: Install GitVersion
-      uses: gittools/actions/gitversion/setup@v1
-      with:
-        versionSpec: "5.12.0"
-    - name: Determine Version
-      id: gitversion
-      uses: gittools/actions/gitversion/execute@v1
+        dotnet-version: 8.0.x
+
+    - uses: actions/cache@v4
       with:
-        useConfigFile: true
-        configFilePath: "GitVersion.yml"
+        path: ~/.nuget/packages
+        key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }}
+        restore-keys: |
+          ${{ runner.os }}-nuget-
+    - name: Install dependencies
+      working-directory: ./src/main
+      run: dotnet restore
+    - name: Install SDK dependencies
+      working-directory: ./src/sdk
+      run: dotnet restore Yardarm.Sdk.sln
 
+    - name: Build SDK
+      working-directory: ./src/sdk
+      run: dotnet build --configuration Release -p:Version=${{ needs.version.outputs.version  }} --no-restore Yardarm.Sdk.sln
+    - name: Test Generate
+      run: ./scripts/generate-sdk.sh
+
+    # Note: SDK is packed by build above, doesn't need to be packed here
+    - name: Push to NuGet.org
+      if: ${{ startsWith(github.ref, 'refs/tags/release/') }}
+      working-directory: ./src
+      run: |
+        dotnet nuget push **/*.nupkg --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json --skip-duplicate
+    - name: Push to GitHub Packages
+      if: ${{ startsWith(github.ref, 'refs/pull/') || startsWith(github.ref, 'refs/tags/release/') }}
+      working-directory: ./src
+      run: |
+        dotnet nuget push **/*.nupkg --api-key ${{ secrets.GITHUB_TOKEN }} --source https://nuget.pkg.github.com/CenterEdge/index.json --skip-duplicate
+
+  docker:
+    runs-on: ubuntu-latest
+    needs: version
+
+    steps:
     - name: Docker meta
       id: meta
-      uses: docker/metadata-action@v4
+      uses: docker/metadata-action@v5
       with:
         images: ghcr.io/centeredge/yardarm
         tags: |
           type=ref,event=branch
           type=ref,event=pr
-          type=semver,pattern={{version}},value=${{ steps.gitversion.outputs.nuGetVersionV2 }}
+          type=semver,pattern={{version}},value=${{ needs.version.outputs.version }}
           type=sha
 
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
@@ -125,10 +152,10 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    - uses: docker/build-push-action@v5
+    - uses: docker/build-push-action@v6
       with:
-        context: .
-        build-args: VERSION=${{ steps.gitversion.outputs.nuGetVersionV2 }}
+        platforms: linux/amd64,linux/arm64
+        build-args: VERSION=${{ needs.version.outputs.version }}
         push: ${{ startsWith(github.ref, 'refs/tags/release/') }}
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}

diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,16 @@
 ARG VERSION=0.1.0-local
 
-FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
+# --platform=$BUILDPLATFORM ensures that the build runs on the actual CPU platform of the builder, without emulation
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 ARG VERSION
+ARG TARGETARCH
 WORKDIR /app
 
+# Place a properly formatted RID in /tmp/arch
+RUN arch=$TARGETARCH \
+    && if [ "$TARGETARCH" = "amd64" ]; then arch="x64"; fi \
+    && echo "linux-$arch" > /tmp/arch
+
 COPY src/main/Yardarm/*.csproj ./main/Yardarm/
 COPY src/main/Yardarm.Client/*.csproj ./main/Yardarm.Client/
 COPY src/main/Yardarm.CommandLine/*.csproj ./main/Yardarm.CommandLine/
@@ -14,20 +21,21 @@ COPY src/main/Yardarm.NewtonsoftJson.Client/*.csproj ./main/Yardarm.NewtonsoftJs
 COPY src/main/Yardarm.SystemTextJson/*.csproj ./main/Yardarm.SystemTextJson/
 COPY src/main/Yardarm.SystemTextJson.Client/*.csproj ./main/Yardarm.SystemTextJson.Client/
 COPY ["src/*.props", "src/*.targets", "src/*.snk", "./"]
-RUN dotnet restore ./main/Yardarm.CommandLine/Yardarm.CommandLine.csproj
+RUN dotnet restore -r $(cat /tmp/arch) ./main/Yardarm.CommandLine/Yardarm.CommandLine.csproj
 
 COPY ./src ./
-RUN dotnet pack -c Release -p:VERSION=${VERSION} ./main/Yardarm.CommandLine/Yardarm.CommandLine.csproj
+RUN dotnet publish --no-restore -c Release -r $(cat /tmp/arch) -p:VERSION=${VERSION} -o /publish ./main/Yardarm.CommandLine/Yardarm.CommandLine.csproj
 
-FROM mcr.microsoft.com/dotnet/sdk:8.0
+# No --platform here so we get the base image for the target platform
+FROM mcr.microsoft.com/dotnet/runtime:8.0
 ARG VERSION
 WORKDIR /app
 
+COPY --from=build /publish/ ./
 RUN groupadd -g 1000 -r yardarm && useradd --no-log-init -u 1000 -r -g yardarm yardarm && \
     mkdir -p /home/yardarm && \
-    chown yardarm:yardarm /home/yardarm
-ENV HOME=/home/yardarm PATH=/home/yardarm/.dotnet/tools:${PATH}
-USER yardarm
-
-COPY --from=build /app/main/Yardarm.CommandLine/bin/Release/Yardarm.CommandLine.*.nupkg ./
-RUN dotnet tool install --global --add-source /app --version ${VERSION} Yardarm.CommandLine
+    chown yardarm:yardarm /home/yardarm && \
+    ln -s /app/Yardarm.CommandLine /app/yardarm
+ENV HOME=/home/yardarm PATH=/app:${PATH}
+USER 1000
+CMD ["yardarm"]
diff --git a/scripts/generate-sdk.sh b/scripts/generate-sdk.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+mkdir -p ./bin
+curl -sSL https://api.adv.centeredge.io/v1/swagger/api/swagger.json -o ./bin/mashtub.json
+
+# Basic test of the SDK
+dotnet build -c Release src/sdk/Yardarm.Sdk.Test/Yardarm.Sdk.Test.csproj
diff --git a/scripts/generate.sh b/scripts/generate.sh
@@ -14,6 +14,3 @@ dotnet run --no-build --no-launch-profile -c Release --project src/main/Yardarm.
     restore -n TestNJ -x src/main/Yardarm.NewtonsoftJson/bin/Release/net8.0/Yardarm.NewtonsoftJson.dll -f netstandard2.0 net6.0 net8.0 --intermediate-dir ./obj/
 dotnet run --no-build --no-launch-profile -c Release --project src/main/Yardarm.CommandLine -- \
     generate --no-restore -n TestNJ -x src/main/Yardarm.NewtonsoftJson/bin/Release/net8.0/Yardarm.NewtonsoftJson.dll -f netstandard2.0 net6.0 net8.0 --embed --intermediate-dir ./obj/ --nupkg ./bin/ -v 1.0.0 -i ./bin/mashtub.json
-
-# Basic test of the SDK
-dotnet build -c Release src/sdk/Yardarm.Sdk.Test/Yardarm.Sdk.Test.csproj
diff --git a/src/sdk/Yardarm.Sdk/Yardarm.Sdk.csproj b/src/sdk/Yardarm.Sdk/Yardarm.Sdk.csproj
@@ -48,34 +48,43 @@
     Publish and collect the Yardarm command-line application to include as a tool.
     Only do this once as part of the outer multi-targeting run.
   -->
-  <PropertyGroup>
-    <GenerateNuspecDependsOn>
-      CollectYardarmCommandLine;
-      $(GenerateNuspecDependsOn)
-    </GenerateNuspecDependsOn>
-  </PropertyGroup>
-  <Target Name="CollectYardarmCommandLine">
+  <ItemGroup>
+    <YardarmCommandLineRuntimeIdentifier Include="win-x64" />
+    <YardarmCommandLineRuntimeIdentifier Include="linux-x64" />
+  </ItemGroup>
+
+  <Target Name="DefineYardarmCommandLineRids">
     <ItemGroup>
       <_YardarmCommandLine Include="..\..\main\Yardarm.CommandLine\Yardarm.CommandLine.csproj">
-        <AdditionalProperties>RuntimeIdentifier=win-x64</AdditionalProperties>
-      </_YardarmCommandLine>
-      <_YardarmCommandLine Include="..\..\main\Yardarm.CommandLine\Yardarm.CommandLine.csproj">
-        <AdditionalProperties>RuntimeIdentifier=linux-x64</AdditionalProperties>
+        <AdditionalProperties>RuntimeIdentifier=%(YardarmCommandLineRuntimeIdentifier.Identity)</AdditionalProperties>
+        <RuntimeIdentifier>%(YardarmCommandLineRuntimeIdentifier.Identity)</RuntimeIdentifier>
       </_YardarmCommandLine>
     </ItemGroup>
+  </Target>
 
+  <Target Name="BuildYardarmCommandLine"
+          DependsOnTargets="DefineYardarmCommandLineRids">
     <!--
       Ensure that the CommandLine build has been run for both RuntimeIdentifier values
     -->
-    <MSBuild Projects="@(_YardarmCommandLine)" Properties="Configuration=$(Configuration);TargetFramework=net8.0;SelfContained=False" Targets="Build" BuildInParallel="$(BuildInParallel)" SkipNonexistentProjects="false" ContinueOnError="false">
+    <MSBuild Projects="@(_YardarmCommandLine)" Properties="Configuration=$(Configuration);TargetFramework=net8.0;SelfContained=False" Targets="Publish" BuildInParallel="$(BuildInParallel)" SkipNonexistentProjects="false" ContinueOnError="false">
     </MSBuild>
+  </Target>
 
+  <PropertyGroup>
+    <GenerateNuspecDependsOn>
+      CollectYardarmCommandLine;
+      $(GenerateNuspecDependsOn)
+    </GenerateNuspecDependsOn>
+  </PropertyGroup>
+  <Target Name="CollectYardarmCommandLine"
+          DependsOnTargets="DefineYardarmCommandLineRids;BuildYardarmCommandLine">
     <!--
       Collect the files to be published. By using ComputeFilesToPublish and PublishItemsOutputGroup we skip
       the cost of actually copying the files to a publish directory, instead it collects the returned items
       and we can load them directly into the NuGet package.
     -->
-    <MSBuild Projects="@(_YardarmCommandLine)" Properties="Configuration=$(Configuration);TargetFramework=net8.0;SelfContained=False" Targets="ComputeFilesToPublish;PublishItemsOutputGroup" BuildInParallel="$(BuildInParallel)" SkipNonexistentProjects="false" ContinueOnError="false">
+    <MSBuild Projects="@(_YardarmCommandLine)" Properties="Configuration=$(Configuration);TargetFramework=net8.0;SelfContained=False;PublishReadyToRun=true" Targets="PublishItemsOutputGroup" BuildInParallel="$(BuildInParallel)" SkipNonexistentProjects="false" ContinueOnError="false">
       <Output TaskParameter="TargetOutputs" ItemName="_YardarmCommandLineFiles" />
     </MSBuild>
 
@@ -88,7 +97,7 @@
       <_FilteredYardarmCommandLineFiles Include="@(_YardarmCommandLineFiles)" Condition=" '%(TargetPath)' == 'Yardarm.CommandLine.deps.json' " />
       <_FilteredYardarmCommandLineFiles Include="@(_YardarmCommandLineFiles)" Condition=" '%(TargetPath)' == 'Yardarm.CommandLine.runtimeconfig.json' " />
 
-      <_YardarmCommandLineLinuxExecutable Include="@(_YardarmCommandLineFiles-&gt;ClearMetadata())" Condition=" '%(TargetPath)' == 'Yardarm.CommandLine' " />
+      <_YardarmCommandLineLinuxExecutable Include="@(_YardarmCommandLineFiles-&gt;ClearMetadata())" Condition=" '%(TargetPath)' == 'Yardarm.CommandLine' " RuntimeIdentifier="%(_YardarmCommandLineFiles.RuntimeIdentifier)" />
     </ItemGroup>
 
     <!--
@@ -97,21 +106,21 @@
       "Yardarm.CommandLine" from Linux will get put in a subdirectory (it's original name is "apphost"). Therefore, we must copy it
       to a temporary directory with a matching name and no TargetPath metadata.
     -->
-    <Copy SourceFiles="@(_YardarmCommandLineLinuxExecutable)" DestinationFiles="@(_YardarmCommandLineLinuxExecutable->'$(BaseIntermediateOutputPath)pubtemp\Yardarm.CommandLine')" SkipUnchangedFiles="true">
-      <Output TaskParameter="CopiedFiles" ItemName="_CopiedYardarmCommandLineLinuxExecutable" />
+    <ItemGroup>
+      <_CopiedYardarmCommandLineLinuxExecutable Include="@(_YardarmCommandLineLinuxExecutable->'$(BaseIntermediateOutputPath)pubtemp\%(RuntimeIdentifier)\Yardarm.CommandLine')" />
+    </ItemGroup>
+    <Copy SourceFiles="@(_YardarmCommandLineLinuxExecutable)" DestinationFiles="@(_CopiedYardarmCommandLineLinuxExecutable)" SkipUnchangedFiles="true">
     </Copy>
     <ItemGroup>
-      <_FilteredYardarmCommandLineFiles Include="@(_CopiedYardarmCommandLineLinuxExecutable)">
-        <AdditionalProperties>RuntimeIdentifier=linux-x64</AdditionalProperties>
-      </_FilteredYardarmCommandLineFiles>
+      <_FilteredYardarmCommandLineFiles Include="@(_CopiedYardarmCommandLineLinuxExecutable)" />
+      <FileWrites Include="@(_CopiedYardarmCommandLineLinuxExecutable)" />
     </ItemGroup>
 
     <!--
       Convert the collected _FilteredYardarmCommandLineFiles into _PackageFiles items marked for pack in the appropriate directories.
     -->
     <ItemGroup>
-      <_PackageFiles Include="@(_FilteredYardarmCommandLineFiles)" Condition=" '%(_FilteredYardarmCommandLineFiles.AdditionalProperties)' == 'RuntimeIdentifier=win-x64' " BuildAction="None" Pack="true" PackagePath="tools\net8.0\win-x64\yardarm\%(_FilteredYardarmCommandLineFiles.TargetPath)" />
-      <_PackageFiles Include="@(_FilteredYardarmCommandLineFiles)" Condition=" '%(_FilteredYardarmCommandLineFiles.AdditionalProperties)' == 'RuntimeIdentifier=linux-x64' " BuildAction="None" Pack="true" PackagePath="tools\net8.0\linux-x64\yardarm\%(_FilteredYardarmCommandLineFiles.TargetPath)" />
+      <_PackageFiles Include="@(_FilteredYardarmCommandLineFiles)" BuildAction="None" Pack="true" PackagePath="tools\net8.0\%(_FilteredYardarmCommandLineFiles.RuntimeIdentifier)\yardarm\%(_FilteredYardarmCommandLineFiles.TargetPath)" />
     </ItemGroup>
   </Target>