Merge openebs#693

693: ci(helm): clean up deploy yamls and improve helm templates r=jkryl a=jkryl List of changes: * use defined storage class from storage-class.yaml in pvc.yaml * use develop tags on develop branch * generate pool.yaml from flexible template * add cpu-count, hugepages and image-pull-policy chart parameters * avoid hardcoded namespace in yamls * remove unused mayastor-daemonset-config.yaml * remove grafana monitor files that aren't maintained anymore * remove namespace.yaml and use "kubectl create namespace mayastor" instead Future directions: 1. We can use the new helm template to generate pool yaml according to our needs in e2e install test avoiding duplicate pool yaml. 2. scripts/generate-deploy-yamls.sh should be a highlevel tool with profile name argument (i.e. release, develop, test, ...) to make it really useful. For test profile we can generate config with two cpus per mayastor, for release we can use just one, etc. The point being that this is the recommended config for given env. Co-authored-by: Jan Kryl <[email protected]>
vharsh · Feb 10, 2021 · 7c2e9e9 · 7c2e9e9
2 parents 9c028b2 + a14faff
commit 7c2e9e9
Show file tree

Hide file tree

Showing 27 changed files with 96 additions and 902 deletions.
diff --git a/chart/README.md b/chart/README.md
@@ -23,7 +23,7 @@ kubectl delete namespace mayastor
 
 ## templating
 
-[ ] templatize namespace properly - mayastor namespace is hardcoded in yaml templates
+[x] templatize namespace properly - mayastor namespace is hardcoded in yaml templates
   - use Release.Namespace
   - use Release.Name
 [ ] allow pulling image from authenticated repository

diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl
@@ -6,3 +6,13 @@
 {{- "" }}
 {{- end }}
 {{- end }}
+
+{{/* Generate CPU list specification based on CPU count (-l param of mayastor) */}}
+{{- define "mayastorCpuSpec" -}}
+{{- range $i, $e := until (int .Values.mayastorCpuCount) }}
+{{- if gt $i 0 }}
+{{- printf "," }}
+{{- end }}
+{{- printf "%d" (add $i 1) }}
+{{- end }}
+{{- end }}
diff --git a/chart/templates/csi-daemonset.yaml b/chart/templates/csi-daemonset.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  namespace: mayastor
+  namespace: {{ .Release.Namespace }}
   name: mayastor-csi
   labels:
     openebs/engine: mayastor
@@ -29,7 +29,7 @@ spec:
       containers:
       - name: mayastor-csi
         image: {{ include "mayastorImagesPrefix" . }}mayadata/mayastor-csi:{{ .Values.mayastorImagesTag }}
-        imagePullPolicy: Always
+        imagePullPolicy: {{ .Values.mayastorImagePullPolicy }}
         # we need privileged because we mount filesystems and use mknod
         securityContext:
           privileged: true

diff --git a/chart/templates/mayastor-daemonset.yaml b/chart/templates/mayastor-daemonset.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  namespace: mayastor
+  namespace: {{ .Release.Namespace }}
   name: mayastor
   labels:
     openebs/engine: mayastor
@@ -20,7 +20,7 @@ spec:
         app: mayastor
     spec:
       hostNetwork: true
-      # To resolve services from mayastor namespace
+      # To resolve services from mayastor's namespace
       dnsPolicy: ClusterFirstWithHostNet
       nodeSelector:
         openebs.io/engine: mayastor
@@ -32,7 +32,7 @@ spec:
       containers:
       - name: mayastor
         image: {{ include "mayastorImagesPrefix" . }}mayadata/mayastor:{{ .Values.mayastorImagesTag }}
-        imagePullPolicy: Always
+        imagePullPolicy: {{ .Values.mayastorImagePullPolicy }}
         env:
         - name: MY_NODE_NAME
           valueFrom:
@@ -59,7 +59,7 @@ spec:
         - "-g$(MY_POD_IP)"
         - "-nnats"
         - "-y/var/local/mayastor/config.yaml"
-        - "-m0x3"
+        - "-l{{ include "mayastorCpuSpec" . }}"
         securityContext:
           privileged: true
         volumeMounts:
@@ -78,13 +78,13 @@ spec:
           # belong to Guaranteed QoS class, hence can never get evicted in case of
           # pressure unless they exceed those limits. limits and requests must be the same.
           limits:
-            cpu: "2"
-            memory: "500Mi"
-            hugepages-2Mi: "1Gi"
+            cpu: "{{ .Values.mayastorCpuCount }}"
+            memory: "512Mi"
+            hugepages-2Mi: "{{ .Values.mayastorHugePagesGiB }}Gi"
           requests:
-            cpu: "2"
-            memory: "500Mi"
-            hugepages-2Mi: "1Gi"
+            cpu: "{{ .Values.mayastorCpuCount }}"
+            memory: "512Mi"
+            hugepages-2Mi: "{{ .Values.mayastorHugePagesGiB }}Gi"
         ports:
         - containerPort: 10124
           protocol: TCP

diff --git a/chart/templates/moac-deployment.yaml b/chart/templates/moac-deployment.yaml
@@ -1,9 +1,8 @@
----
 kind: Deployment
 apiVersion: apps/v1
 metadata:
   name: moac
-  namespace: mayastor
+  namespace: {{ .Release.Namespace }}
 spec:
   replicas: 1
   selector:
@@ -45,7 +44,7 @@ spec:
 
         - name: moac
           image: {{ include "mayastorImagesPrefix" . }}mayadata/moac:{{ .Values.mayastorImagesTag }}
-          imagePullPolicy: Always
+          imagePullPolicy: {{ .Values.mayastorImagePullPolicy }}
           args:
             - "--csi-address=$(CSI_ENDPOINT)"
             - "--namespace=$(MY_POD_NAMESPACE)"
@@ -62,23 +61,6 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-          ports:
-            - containerPort: 4000
-              protocol: TCP
-              name: "rest-api"
       volumes:
         - name: socket-dir
-          emptyDir:
----
-kind: Service
-apiVersion: v1
-metadata:
-  name: moac
-  namespace: mayastor
-spec:
-  selector:
-    app: moac
-  ports:
-  - protocol: TCP
-    port: 4000
-    targetPort: 4000
+          emptyDir:
diff --git a/chart/templates/moac-rbac.yaml b/chart/templates/moac-rbac.yaml
@@ -3,7 +3,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: moac
-  namespace: mayastor
+  namespace: {{ .Release.Namespace }}
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -86,7 +86,7 @@ metadata:
 subjects:
 - kind: ServiceAccount
   name: moac
-  namespace: mayastor
+  namespace: {{ .Release.Namespace }}
 roleRef:
   kind: ClusterRole
   name: moac

diff --git a/chart/templates/nats-deployment.yaml b/chart/templates/nats-deployment.yaml
@@ -3,7 +3,7 @@ kind: Deployment
 apiVersion: apps/v1
 metadata:
   name: nats
-  namespace: mayastor
+  namespace: {{ .Release.Namespace }}
 spec:
   replicas: 1
   selector:
@@ -27,7 +27,7 @@ kind: Service
 apiVersion: v1
 metadata:
   name: nats
-  namespace: mayastor
+  namespace: {{ .Release.Namespace }}
 spec:
   selector:
     app: nats

diff --git a/chart/templates/pool.yaml b/chart/templates/pool.yaml
@@ -0,0 +1,15 @@
+{{- range .Values.mayastorPools }}
+---
+apiVersion: "openebs.io/v1alpha1"
+kind: MayastorPool
+metadata:
+  # Name can be anything as long as it is unique
+  name: pool-on-{{ .node }}
+  # or let k8s to generate a unique pool name
+  #generateName: pool-
+  namespace: {{ $.Release.Namespace }}
+spec:
+  node: {{ .node }}
+  # ATM only one disk device is supported (i.e. /dev/nvme1n1)
+  disks: ["{{ .device }}"]
+{{- end }}
diff --git a/chart/values.yaml b/chart/values.yaml
@@ -1,2 +1,8 @@
-mayastorImagesTag: latest
+mayastorImagesTag: develop
+mayastorImagePullPolicy: Always
+mayastorCpuCount: "2"
+mayastorHugePagesGiB: "1"
 mayastorImagesRepo: ""
+mayastorPools:
+  - node: "NODE_NAME"
+    device: "DEVICE"
diff --git a/deploy/README.md b/deploy/README.md
@@ -8,6 +8,7 @@ It is now hosted at [GitBook](https://mayastor.gitbook.io/introduction/).
 
 # Deployment Files
 
-The files in this directory are provided as a convenience, a template for the successful test deployment of Mayastor to a cluster.
+The files in this directory are provided as a convenience, a template for the
+successful test deployment of Mayastor to a cluster.
 
-**NOTE:** ALL versions of deployment files from v0.4.0 specify the 'latest' tag on Mayastor container images.  If you wish to deploy earlier versions, your own builds, or a nightly build, you must change the image tags accordingly.
+Most of them are generated from helm templates in `chart/` directory.
diff --git a/deploy/csi-daemonset.yaml b/deploy/csi-daemonset.yaml
@@ -30,7 +30,7 @@ spec:
       # the same.
       containers:
       - name: mayastor-csi
-        image: mayadata/mayastor-csi:latest
+        image: mayadata/mayastor-csi:develop
         imagePullPolicy: Always
         # we need privileged because we mount filesystems and use mknod
         securityContext:

diff --git a/deploy/fio.yaml b/deploy/fio.yaml
@@ -17,3 +17,6 @@ spec:
       volumeMounts:
         - mountPath: "/volume"
           name: ms-volume
+      #volumeDevices:
+      #  - devicePath: /dev/xvda
+      #    name: ms-volume
diff --git a/deploy/mayastor-daemonset-config.yaml b/deploy/mayastor-daemonset-config.yaml
diff --git a/deploy/mayastor-daemonset.yaml b/deploy/mayastor-daemonset.yaml
@@ -22,7 +22,7 @@ spec:
         app: mayastor
     spec:
       hostNetwork: true
-      # To resolve services from mayastor namespace
+      # To resolve services from mayastor's namespace
       dnsPolicy: ClusterFirstWithHostNet
       nodeSelector:
         openebs.io/engine: mayastor
@@ -33,7 +33,7 @@ spec:
         command: ['sh', '-c', 'until nc -vz nats 4222; do echo "Waiting for message bus..."; sleep 1; done;']
       containers:
       - name: mayastor
-        image: mayadata/mayastor:latest
+        image: mayadata/mayastor:develop
         imagePullPolicy: Always
         env:
         - name: MY_NODE_NAME
@@ -61,7 +61,7 @@ spec:
         - "-g$(MY_POD_IP)"
         - "-nnats"
         - "-y/var/local/mayastor/config.yaml"
-        - "-m0x3"
+        - "-l1,2"
         securityContext:
           privileged: true
         volumeMounts: