[AMORO-2895] Add more image related properties for the amoro optimize…

…rs (#2896) * [AMORO-2895] Add more image related properties for the amoro optimizers * bugfix * modify docs
apache · Jun 12, 2024 · f964b59 · f964b59
1 parent 89dd028
commit f964b59
Show file tree

Hide file tree

Showing 6 changed files with 116 additions and 24 deletions.
diff --git a/...ms-server/src/main/java/org/apache/amoro/server/manager/KubernetesOptimizerContainer.java b/...ms-server/src/main/java/org/apache/amoro/server/manager/KubernetesOptimizerContainer.java
@@ -18,6 +18,8 @@
 
 package org.apache.amoro.server.manager;
 
+import io.fabric8.kubernetes.api.model.LocalObjectReference;
+import io.fabric8.kubernetes.api.model.LocalObjectReferenceBuilder;
 import io.fabric8.kubernetes.api.model.Quantity;
 import io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder;
 import io.fabric8.kubernetes.api.model.apps.Deployment;
@@ -37,7 +39,10 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /** Kubernetes Optimizer Container with Standalone Optimizer */
 public class KubernetesOptimizerContainer extends AbstractResourceContainer {
@@ -48,6 +53,9 @@ public class KubernetesOptimizerContainer extends AbstractResourceContainer {
   public static final String CPU_FACTOR_PROPERTY = "cpu.factor";
   public static final String NAMESPACE = "namespace";
   public static final String IMAGE = "image";
+  public static final String PULL_POLICY = "pullPolicy";
+
+  public static final String PULL_SECRETS = "imagePullSecrets";
   public static final String KUBE_CONFIG_PATH = "kube-config-path";
 
   private static final String NAME_PREFIX = "amoro-optimizer-";
@@ -83,14 +91,21 @@ protected Map<String, String> doScaleOut(Resource resource) {
 
     String namespace = groupProperties.getOrDefault(NAMESPACE, "default");
     String image = checkAndGetProperty(groupProperties, IMAGE);
+    String pullPolicy = checkAndGetProperty(groupProperties, PULL_POLICY);
+    String pullSecrets = groupProperties.getOrDefault(PULL_SECRETS, "");
     String cpuLimitFactorString = groupProperties.getOrDefault(CPU_FACTOR_PROPERTY, "1.0");
     double cpuLimitFactor = Double.parseDouble(cpuLimitFactorString);
     int cpuLimit = (int) (Math.ceil(cpuLimitFactor * resource.getThreadCount()));
 
+    List<LocalObjectReference> imagePullSecretsList =
+        Arrays.stream(pullSecrets.split(";"))
+            .map(secret -> new LocalObjectReferenceBuilder().withName(secret).build())
+            .collect(Collectors.toList());
+
     String resourceId = resource.getResourceId();
     String groupName = resource.getGroupName();
     String kubernetesName = NAME_PREFIX + resourceId;
-    Deployment deployment =
+    DeploymentBuilder deploymentBuilder =
         new DeploymentBuilder()
             .withNewMetadata()
             .withName(NAME_PREFIX + resourceId)
@@ -107,6 +122,7 @@ protected Map<String, String> doScaleOut(Resource resource) {
             .addNewContainer()
             .withName("optimizer")
             .withImage(image)
+            .withImagePullPolicy(pullPolicy)
             .withCommand("sh", "-c", startUpArgs)
             .withResources(
                 new ResourceRequirementsBuilder()
@@ -129,8 +145,19 @@ protected Map<String, String> doScaleOut(Resource resource) {
             .withNewSelector()
             .addToMatchLabels("app", NAME_PREFIX + resourceId)
             .endSelector()
-            .endSpec()
-            .build();
+            .endSpec();
+    if (!imagePullSecretsList.isEmpty()) {
+      deploymentBuilder
+          .editSpec()
+          .editTemplate()
+          .editSpec()
+          .withImagePullSecrets(imagePullSecretsList)
+          .endSpec()
+          .endTemplate()
+          .endSpec();
+    }
+    Deployment deployment = deploymentBuilder.build();
+
     client.apps().deployments().inNamespace(namespace).resource(deployment).create();
     Map<String, String> startupProperties = Maps.newHashMap();
     startupProperties.put(NAMESPACE, namespace);

diff --git a/charts/amoro/Chart.yaml b/charts/amoro/Chart.yaml
@@ -20,7 +20,7 @@ name: amoro
 description: A Helm chart for Amoro
 type: application
 version: 0.1.0
-appVersion: "0.6.0"
+appVersion: "0.7"
 home: https://amoro.apache.org/
 icon: https://amoro.apache.org/img/amoro-logo-icon.png
 sources:

diff --git a/charts/amoro/templates/_optimizer.tpl b/charts/amoro/templates/_optimizer.tpl
@@ -15,6 +15,42 @@
   limitations under the License.
 */}}
 
+{{/*Flink Optimizer Image Pull Secrets*/}}
+{{- define "amoro.optimizer.container.flink.pull.secrets" -}}
+  {{- if .Values.imagePullSecrets -}}
+    {{- $secrets := .Values.imagePullSecrets -}}
+    {{- $secretNames := list -}}
+    {{- range $secrets }}
+      {{- $secretNames = append $secretNames .name -}}
+    {{- end }}
+    {{- join ";" $secretNames -}}
+  {{- end -}}
+{{- end -}}
+
+{{/*Spark Optimizer Image Pull Secrets*/}}
+{{- define "amoro.optimizer.container.spark.pull.secrets" -}}
+  {{- if .Values.imagePullSecrets -}}
+    {{- $secrets := .Values.imagePullSecrets -}}
+    {{- $secretNames := list -}}
+    {{- range $secrets }}
+      {{- $secretNames = append $secretNames .name -}}
+    {{- end }}
+    {{- join "," $secretNames -}}
+  {{- end -}}
+{{- end -}}
+
+{{/*Kubernetes Optimizer Image Pull Secrets*/}}
+{{- define "amoro.optimizer.container.kubernetes.pull.secrets" -}}
+  {{- if .Values.imagePullSecrets -}}
+    {{- $secrets := .Values.imagePullSecrets -}}
+    {{- $secretNames := list -}}
+    {{- range $secrets }}
+      {{- $secretNames = append $secretNames .name -}}
+    {{- end }}
+    {{- join ";" $secretNames -}}
+  {{- end -}}
+{{- end -}}
+
 {{/*Flink Optimizer Image Tag*/}}
 {{- define "amoro.optimizer.container.flink.tag" -}}
 {{- if .Values.optimizer.flink.image.tag -}}
@@ -54,7 +90,11 @@ properties:
   flink-home: /opt/flink
   export.FLINK_HOME: /opt/flink
   flink-conf.kubernetes.container.image: {{ include "amoro.optimizer.container.flink.image" .  | quote }}
+  flink-conf.kubernetes.container.image.pull-policy: {{ .Values.optimizer.flink.image.pullPolicy | quote }}
   flink-conf.kubernetes.service-account: {{ include "amoro.sa.name" . | quote }}
+  {{- if .Values.imagePullSecrets }}
+  flink-conf.kubernetes.container.image.pull-secrets: {{ include "amoro.optimizer.container.flink.pull.secrets" . }}
+  {{- end }}
   {{- with .Values.optimizer.flink.properties -}}
     {{- toYaml . | nindent 2 }}
   {{- end -}}
@@ -68,7 +108,11 @@ properties:
   spark-home: /opt/spark
   export.SPARK_HOME: /opt/spark
   spark-conf.spark.kubernetes.container.image: {{ include "amoro.optimizer.container.spark.image" .  | quote }}
+  spark-conf.spark.kubernetes.container.image.pullPolicy: {{ .Values.optimizer.spark.image.pullPolicy | quote }}
   spark-conf.spark.kubernetes.authenticate.driver.serviceAccountName: {{ include "amoro.sa.name" . | quote }}
+  {{- if .Values.imagePullSecrets }}
+  spark-conf.spark.kubernetes.container.image.pullSecrets: {{ include "amoro.optimizer.container.spark.pull.secrets" . }}
+  {{- end }}
   {{- with .Values.optimizer.spark.properties -}}
     {{- toYaml . | nindent 2 }}
   {{- end -}}
@@ -90,5 +134,8 @@ properties:
   {{- with .Values.optimizer.kubernetes.properties -}}
     {{- toYaml . | nindent 2 }}
   {{- end -}}
+  {{- if .Values.imagePullSecrets }}
+  imagePullSecrets: {{ include "amoro.optimizer.container.kubernetes.pull.secrets" . }}
+  {{- end }}
 {{- end -}}
 
diff --git a/charts/amoro/values.yaml b/charts/amoro/values.yaml
@@ -201,6 +201,7 @@ optimizer:
       ## @param optimizer.flink.image.tag the image tag, if not set, the default value is the same with amoro image tag
       ##
       tag: ~
+      pullPolicy: IfNotPresent
       jobUri: "local:///opt/flink/usrlib/optimizer-job.jar"
     properties: {
       "flink-conf.taskmanager.memory.managed.size": "32mb",
@@ -226,6 +227,7 @@ optimizer:
       ## @param optimizer.spark.image.tag the image tag, if not set, the default value is the same with amoro image tag
       ##
       tag: ~
+      pullPolicy: IfNotPresent
       jobUri: "local:///opt/spark/usrlib/optimizer-job.jar"
     properties: {
       "master": "k8s://https://kubernetes.default.svc",
@@ -254,7 +256,8 @@ optimizer:
     properties:
       namespace: "default"
       kube-config-path: "～/.kube/config"
-      image: "apache/amoro:0.6"
+      image: "apache/amoro:latest"
+      pullPolicy: "IfNotPresent"
   extra: []
 
 ## Configure the plugin that allows you to access the metric reporters and events listeners.

diff --git a/docs/admin-guides/deployment-on-kubernetes.md b/docs/admin-guides/deployment-on-kubernetes.md
@@ -33,6 +33,10 @@ This is an image built based on the Amoro binary distribution package for deploy
 
 This is an image built based on the official version of Flink for deploying the Flink optimizer.
 
+**apache/amoro-spark-optimizer**
+
+This is an image built based on the official version of Spark for deploying the Spark optimizer.
+
 ## Build AMS Docker Image
 
 If you want to build images locally, you can find the `build.sh` script in the docker folder of the project and pass the following command:
@@ -41,12 +45,17 @@ If you want to build images locally, you can find the `build.sh` script in the d
 ./docker/build.sh amoro
 ```
 
-or build the `optimizer-flink` image by:
+or build the `amoro-flink-optimizer` image by:
 
 ```shell
-./docker/build.sh optimizer-flink --flink-version <flink-version>
+./docker/build.sh amoro-flink-optimizer --flink-version <flink-version>
 ```
 
+or build the `amoro-spark-optimizer` image by:
+
+```shell
+./docker/build.sh amoro-spark-optimizer --spark-version <spark-version>
+```
 
 ## Get Helm Charts
 
@@ -56,7 +65,7 @@ You can obtain the latest official release chart by adding the official Helm rep
 $ helm repo add amoro https://netease.github.io/amoro/charts
 $ helm search repo amoro 
 NAME           CHART VERSION    APP VERSION        DESCRIPTION           
-amoro/amoro    0.1.0            0.6.0              A Helm chart for Amoro 
+amoro/amoro    0.1.0            0.7.0              A Helm chart for Amoro 
 
 $ helm pull amoro/amoro 
 $ tar zxvf amoro-*.tgz
@@ -156,6 +165,7 @@ image:
   repository: <your repository>
   pullPolicy: IfNotPresent
   tag: <your tag>
+imagePullSecrets: [ ]
 ```
 
 ### Configure the Flink Optimizer Container
@@ -174,6 +184,7 @@ optimizer:
       repository: apache/amoro-flink-optimizer
       ## the image tag, if not set, the default value is the same with amoro image tag.
       tag: ~
+      pullPolicy: IfNotPresent
       ## the location of flink optimizer jar in image.
       jobUri: "local:///opt/flink/usrlib/optimizer-job.jar"
     properties: {