[SKIP-1224] Add PodSettings (#349)

* add podsettings for adding annotations and terminationgraceperiodsettings on pod templates

* add some docs

* not yaml yaml

api/v1alpha1/application_types.go

@@ -218,6 +218,12 @@ type ApplicationSpec struct {
 	//
 	//+kubebuilder:validation:Optional
 	AuthorizationSettings *AuthorizationSettings `json:"authorizationSettings,omitempty"`
+
+	// PodSettings are used to apply specific settings to the Pod Template used by Skiperator to create Deployments. This allows you to set
+	// things like annotations on the Pod to change the behaviour of sidecars, and set relevant Pod options such as TerminationGracePeriodSeconds.
+	//
+	//+kubebuilder:validation:Optional
+	PodSettings *podtypes.PodSettings `json:"podSettings,omitempty"`
 }
 
 // AuthorizationSettings Settings for overriding the default deny of all actuator endpoints. AllowAll will allow any

api/v1alpha1/podtypes/pod_settings.go

@@ -0,0 +1,18 @@
+package podtypes
+
+// PodSettings
+//
+// +kubebuilder:object:generate=true
+type PodSettings struct {
+	// Annotations that are set on Pods created by Skiperator. These annotations can for example be used to change the behaviour of sidecars and similar.
+	//
+	//+kubebuilder:validation:Optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+
+	// TerminationGracePeriodSeconds determines how long Kubernetes waits after a SIGTERM signal sent to a Pod before terminating the pod. If your application uses longer than
+	// 30 seconds to terminate, you should increase TerminationGracePeriodSeconds.
+	//
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:default:=30
+	TerminationGracePeriodSeconds int64 `json:"terminationGracePeriodSeconds,omitempty"`
+}

api/v1alpha1/skipjob_types.go

@@ -106,6 +106,9 @@ type ContainerSettings struct {
 	// +kubebuilder:default="Never"
 	// +kubebuilder:validation:Optional
 	RestartPolicy *corev1.RestartPolicy `json:"restartPolicy"`
+
+	//+kubebuilder:validation:Optional
+	PodSettings *podtypes.PodSettings `json:"podSettings,omitempty"`
 }
 
 // +kubebuilder:object:generate=true

config/crd/skiperator.kartverket.no_applications.yaml

@@ -492,6 +492,21 @@ spec:
                 - path
                 - port
                 type: object
+              podSettings:
+                description: PodSettings are used to apply specific settings to the
+                  Pod Template used by Skiperator to create Deployments. This allows
+                  you to set things like annotations on the Pod to change the behaviour
+                  of sidecars, and set relevant Pod options such as TerminationGracePeriodSeconds.
+                properties:
+                  annotations:
+                    additionalProperties:
+                      type: string
+                    type: object
+                  terminationGracePeriodSeconds:
+                    default: 30
+                    format: int64
+                    type: integer
+                type: object
               port:
                 description: The port the deployment exposes
                 type: integer

config/crd/skiperator.kartverket.no_skipjobs.yaml

@@ -436,6 +436,18 @@ spec:
                     - path
                     - port
                     type: object
+                  podSettings:
+                    description: PodSettings
+                    properties:
+                      annotations:
+                        additionalProperties:
+                          type: string
+                        type: object
+                      terminationGracePeriodSeconds:
+                        default: 30
+                        format: int64
+                        type: integer
+                    type: object
                   priority:
                     default: medium
                     enum:

controllers/application/deployment.go

@@ -9,6 +9,7 @@ import (
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/core"
 	"github.com/kartverket/skiperator/pkg/resourcegenerator/gcp"
 	"github.com/kartverket/skiperator/pkg/util"
+	"golang.org/x/exp/maps"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -97,6 +98,10 @@ func (r *ApplicationReconciler) defineDeployment(ctx context.Context, applicatio
 		generatedSpecAnnotations["prometheus.io/path"] = application.Spec.Prometheus.Path
 	}
 
+	if application.Spec.PodSettings != nil && len(application.Spec.PodSettings.Annotations) > 0 {
+		maps.Copy(generatedSpecAnnotations, application.Spec.PodSettings.Annotations)
+	}
+
 	podForDeploymentTemplate := corev1.Pod{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Pod",
@@ -106,31 +111,14 @@ func (r *ApplicationReconciler) defineDeployment(ctx context.Context, applicatio
 			Labels:      labels,
 			Annotations: generatedSpecAnnotations,
 		},
-		Spec: corev1.PodSpec{
-			Containers: []corev1.Container{
-				skiperatorContainer,
-			},
-
-			// TODO: Make this as part of operator in a safe way
-			ImagePullSecrets: []corev1.LocalObjectReference{{Name: "github-auth"}},
-			SecurityContext: &corev1.PodSecurityContext{
-				SupplementalGroups: []int64{util.SkiperatorUser},
-				FSGroup:            util.PointTo(util.SkiperatorUser),
-				SeccompProfile: &corev1.SeccompProfile{
-					Type: corev1.SeccompProfileTypeRuntimeDefault,
-				},
-			},
-			ServiceAccountName: application.Name,
-			// The resulting kubernetes object includes the ServiceAccount field, and thus it's required in order
-			// to not create a diff for the hash of existing and wanted spec
-			DeprecatedServiceAccount:      application.Name,
-			Volumes:                       podVolumes,
-			PriorityClassName:             fmt.Sprintf("skip-%s", application.Spec.Priority),
-			RestartPolicy:                 corev1.RestartPolicyAlways,
-			TerminationGracePeriodSeconds: util.PointTo(int64(corev1.DefaultTerminationGracePeriodSeconds)),
-			DNSPolicy:                     corev1.DNSClusterFirst,
-			SchedulerName:                 corev1.DefaultSchedulerName,
-		},
+		Spec: core.CreatePodSpec(
+			skiperatorContainer,
+			podVolumes,
+			application.Name,
+			application.Spec.Priority,
+			util.PointTo(corev1.RestartPolicyAlways),
+			application.Spec.PodSettings,
+		),
 	}
 
 	r.SetLabelsFromApplication(&podForDeploymentTemplate, *application)

controllers/skipjob/job.go

@@ -267,7 +267,14 @@ func getJobSpec(skipJob *skiperatorv1alpha1.SKIPJob, selector *metav1.LabelSelec
 		Selector:              nil,
 		ManualSelector:        nil,
 		Template: corev1.PodTemplateSpec{
-			Spec: core.CreatePodSpec(core.CreateJobContainer(skipJob, containerVolumeMounts, envVars), podVolumes, skipJob.KindPostFixedName(), skipJob.Spec.Container.Priority, skipJob.Spec.Container.RestartPolicy),
+			Spec: core.CreatePodSpec(
+				core.CreateJobContainer(skipJob, containerVolumeMounts, envVars),
+				podVolumes,
+				skipJob.KindPostFixedName(),
+				skipJob.Spec.Container.Priority,
+				skipJob.Spec.Container.RestartPolicy,
+				skipJob.Spec.Container.PodSettings,
+			),
 			ObjectMeta: metav1.ObjectMeta{
 				Labels: GetJobLabels(skipJob, nil),
 			},

pkg/resourcegenerator/core/pod.go

@@ -12,15 +12,21 @@ type PodOpts struct {
 	IstioEnabled bool
 }
 
-func CreatePodSpec(container corev1.Container, volumes []corev1.Volume, serviceAccountName string, priority string, policy *corev1.RestartPolicy) corev1.PodSpec {
+func CreatePodSpec(container corev1.Container, volumes []corev1.Volume, serviceAccountName string, priority string, policy *corev1.RestartPolicy, podSettings *podtypes.PodSettings) corev1.PodSpec {
+	if podSettings == nil {
+		podSettings = &podtypes.PodSettings{
+			TerminationGracePeriodSeconds: int64(30),
+		}
+	}
+
 	return corev1.PodSpec{
 		Volumes: volumes,
 		Containers: []corev1.Container{
 			container,
 		},
 		RestartPolicy:                 *policy,
-		TerminationGracePeriodSeconds: util.PointTo(int64(30)),
-		DNSPolicy:                     "ClusterFirst",
+		TerminationGracePeriodSeconds: util.PointTo(podSettings.TerminationGracePeriodSeconds),
+		DNSPolicy:                     corev1.DNSClusterFirst,
 		ServiceAccountName:            serviceAccountName,
 		DeprecatedServiceAccount:      serviceAccountName,
 		NodeName:                      "",
@@ -35,7 +41,7 @@ func CreatePodSpec(container corev1.Container, volumes []corev1.Volume, serviceA
 			},
 		},
 		ImagePullSecrets:  []corev1.LocalObjectReference{{Name: "github-auth"}},
-		SchedulerName:     "default-scheduler",
+		SchedulerName:     corev1.DefaultSchedulerName,
 		PriorityClassName: fmt.Sprintf("skip-%s", priority),
 	}
 

tests/application/pod-settings/00-application.yaml

@@ -0,0 +1,10 @@
+apiVersion: skiperator.kartverket.no/v1alpha1
+kind: Application
+metadata:
+  name: podsettings
+spec:
+  image: image
+  port: 8080
+  podSettings:
+    annotations:
+      testLabel: "testing"

tests/application/pod-settings/00-assert.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: podsettings
+spec:
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        argocd.argoproj.io/sync-options: "Prune=false"
+        testLabel: "testing"
+    spec:
+      terminationGracePeriodSeconds: 30

tests/application/pod-settings/01-delete-application.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: skiperator.kartverket.no/v1alpha1
+    kind: Application
+    name: podsettings