Bump Spark version to v3.5.3

Signed-off-by: Yi Chen <[email protected]>

.dockerignore

@@ -0,0 +1,16 @@
+.bsp/
+.git/
+.metals/
+.vscode/
+benchmarks/
+charts/
+docs/
+kubernetes/
+target/
+terraform/
+.dockerignore
+.gitignore
+.scalafmt.conf
+Dockerfile
+LICENSE
+README.md

Dockerfile

@@ -1,23 +1,37 @@
-ARG SPARK_IMAGE=apache/spark:v3.3.2
+ARG SPARK_IMAGE=spark:3.5.3
 
-FROM debian:bullseye-slim as kit-builder
+ARG SBT_IMAGE=sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.10_7_1.10.4_2.12.20
 
-ENV GIT_COMMIT_ID "1b7fb7529edae091684201fab142d956d6afd881"
+FROM debian:bullseye-slim AS tpcds-kit-builder
+
+ENV GIT_COMMIT_ID=1b7fb7529edae091684201fab142d956d6afd881
 
 WORKDIR /app
 
 RUN set -eux && \
     apt-get update && \
-    apt-get install -y gcc make flex bison byacc git && \
+    apt-get install -y gcc make flex bison byacc git
+
+RUN set -eux && \
     git clone https://github.com/databricks/tpcds-kit.git && \
     cd tpcds-kit && \
     git checkout ${GIT_COMMIT_ID} && \
     cd tools && \
     make OS=LINUX
 
+FROM ${SBT_IMAGE} AS benchmark-builder
+
+WORKDIR /app
+
+COPY . .
+
+RUN set -eux && \
+    sbt assembly
 
 FROM ${SPARK_IMAGE}
 
-COPY --from=kit-builder /app/tpcds-kit/tools /opt/tpcds-kit/tools
+COPY --from=tpcds-kit-builder /app/tpcds-kit/tools /opt/tpcds-kit/tools
+
+COPY --from=benchmark-builder /app/target/scala-2.12/*.jar /opt/spark/jars/
 
-COPY target/scala-2.12/spark-tpcds-benchmark-assembly-0.1.jar /opt/spark/jars/
+COPY lib /opt/spark/jars/

build.sbt

@@ -1,8 +1,8 @@
 ThisBuild / organization := "com.aliyun.ack"
 ThisBuild / version := "0.1"
-ThisBuild / scalaVersion := "2.12.18"
+ThisBuild / scalaVersion := "2.12.20"
 
-val sparkVersion = "3.3.2"
+val sparkVersion = "3.5.3"
 
 lazy val benchmark = (project in file("."))
   .settings(