From a14faffc0fe34f55313b2ee82b084de9d7786c1e Mon Sep 17 00:00:00 2001 From: Jan Kryl Date: Thu, 28 Jan 2021 21:18:05 +0000 Subject: [PATCH] ci(helm): clean up deploy yamls and improve helm templates List of changes: * use defined storage class from storage-class.yaml in pvc.yaml * use develop tags on develop branch * generate pool.yaml from flexible template * add cpu-count, hugepages and image-pull-policy chart parameters * avoid hardcoded namespace in yamls * remove unused mayastor-daemonset-config.yaml * remove grafana monitor files that aren't maintained anymore * remove namespace.yaml and use "kubectl create namespace mayastor" instead Future directions: 1. We can use the new helm template to generate pool yaml according to our needs in e2e install test avoiding duplicate pool yaml. 2. scripts/generate-deploy-yamls.sh should be a highlevel tool with profile name argument (i.e. release, develop, test, ...) to make it really useful. For test profile we can generate config with two cpus per mayastor, for release we can use just one, etc. The point being that this is the recommended config for given env. --- chart/README.md | 2 +- chart/templates/_helpers.tpl | 10 + chart/templates/csi-daemonset.yaml | 4 +- chart/templates/mayastor-daemonset.yaml | 20 +- chart/templates/moac-deployment.yaml | 24 +- chart/templates/moac-rbac.yaml | 4 +- chart/templates/nats-deployment.yaml | 4 +- chart/templates/pool.yaml | 15 + chart/values.yaml | 8 +- deploy/README.md | 5 +- deploy/csi-daemonset.yaml | 2 +- deploy/fio.yaml | 3 + deploy/mayastor-daemonset-config.yaml | 106 ----- deploy/mayastor-daemonset.yaml | 6 +- deploy/moac-deployment.yaml | 20 +- deploy/monitor/README.md | 52 --- deploy/monitor/grafana/dashboards.yaml | 11 - deploy/monitor/grafana/datasources.yaml | 26 -- .../monitor/grafana/mayastor-dashboard.json | 440 ------------------ deploy/monitor/monitor-deployment.yaml | 106 ----- deploy/monitor/telegraf-config.yaml | 78 ---- deploy/namespace.yaml | 6 - deploy/pool.yaml | 14 +- deploy/pvc.yaml | 4 +- scripts/generate-deploy-yamls.sh | 8 +- test/e2e/install/install_test.go | 9 +- test/e2e/uninstall/uninstall_test.go | 11 +- 27 files changed, 96 insertions(+), 902 deletions(-) create mode 100644 chart/templates/pool.yaml delete mode 100644 deploy/mayastor-daemonset-config.yaml delete mode 100644 deploy/monitor/README.md delete mode 100644 deploy/monitor/grafana/dashboards.yaml delete mode 100644 deploy/monitor/grafana/datasources.yaml delete mode 100644 deploy/monitor/grafana/mayastor-dashboard.json delete mode 100644 deploy/monitor/monitor-deployment.yaml delete mode 100644 deploy/monitor/telegraf-config.yaml delete mode 100644 deploy/namespace.yaml diff --git a/chart/README.md b/chart/README.md index f9c8a4730..3bc39eb22 100644 --- a/chart/README.md +++ b/chart/README.md @@ -23,7 +23,7 @@ kubectl delete namespace mayastor ## templating -[ ] templatize namespace properly - mayastor namespace is hardcoded in yaml templates +[x] templatize namespace properly - mayastor namespace is hardcoded in yaml templates - use Release.Namespace - use Release.Name [ ] allow pulling image from authenticated repository diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index 296354258..7b4753649 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -6,3 +6,13 @@ {{- "" }} {{- end }} {{- end }} + +{{/* Generate CPU list specification based on CPU count (-l param of mayastor) */}} +{{- define "mayastorCpuSpec" -}} +{{- range $i, $e := until (int .Values.mayastorCpuCount) }} +{{- if gt $i 0 }} +{{- printf "," }} +{{- end }} +{{- printf "%d" (add $i 1) }} +{{- end }} +{{- end }} diff --git a/chart/templates/csi-daemonset.yaml b/chart/templates/csi-daemonset.yaml index 3e88e3aac..9c2072d15 100644 --- a/chart/templates/csi-daemonset.yaml +++ b/chart/templates/csi-daemonset.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - namespace: mayastor + namespace: {{ .Release.Namespace }} name: mayastor-csi labels: openebs/engine: mayastor @@ -29,7 +29,7 @@ spec: containers: - name: mayastor-csi image: {{ include "mayastorImagesPrefix" . }}mayadata/mayastor-csi:{{ .Values.mayastorImagesTag }} - imagePullPolicy: Always + imagePullPolicy: {{ .Values.mayastorImagePullPolicy }} # we need privileged because we mount filesystems and use mknod securityContext: privileged: true diff --git a/chart/templates/mayastor-daemonset.yaml b/chart/templates/mayastor-daemonset.yaml index 9ee09fbe3..c84957d64 100644 --- a/chart/templates/mayastor-daemonset.yaml +++ b/chart/templates/mayastor-daemonset.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - namespace: mayastor + namespace: {{ .Release.Namespace }} name: mayastor labels: openebs/engine: mayastor @@ -20,7 +20,7 @@ spec: app: mayastor spec: hostNetwork: true - # To resolve services from mayastor namespace + # To resolve services from mayastor's namespace dnsPolicy: ClusterFirstWithHostNet nodeSelector: openebs.io/engine: mayastor @@ -32,7 +32,7 @@ spec: containers: - name: mayastor image: {{ include "mayastorImagesPrefix" . }}mayadata/mayastor:{{ .Values.mayastorImagesTag }} - imagePullPolicy: Always + imagePullPolicy: {{ .Values.mayastorImagePullPolicy }} env: - name: MY_NODE_NAME valueFrom: @@ -59,7 +59,7 @@ spec: - "-g$(MY_POD_IP)" - "-nnats" - "-y/var/local/mayastor/config.yaml" - - "-m0x3" + - "-l{{ include "mayastorCpuSpec" . }}" securityContext: privileged: true volumeMounts: @@ -78,13 +78,13 @@ spec: # belong to Guaranteed QoS class, hence can never get evicted in case of # pressure unless they exceed those limits. limits and requests must be the same. limits: - cpu: "2" - memory: "500Mi" - hugepages-2Mi: "1Gi" + cpu: "{{ .Values.mayastorCpuCount }}" + memory: "512Mi" + hugepages-2Mi: "{{ .Values.mayastorHugePagesGiB }}Gi" requests: - cpu: "2" - memory: "500Mi" - hugepages-2Mi: "1Gi" + cpu: "{{ .Values.mayastorCpuCount }}" + memory: "512Mi" + hugepages-2Mi: "{{ .Values.mayastorHugePagesGiB }}Gi" ports: - containerPort: 10124 protocol: TCP diff --git a/chart/templates/moac-deployment.yaml b/chart/templates/moac-deployment.yaml index d1f2af816..40d0d6cbb 100644 --- a/chart/templates/moac-deployment.yaml +++ b/chart/templates/moac-deployment.yaml @@ -1,9 +1,8 @@ ---- kind: Deployment apiVersion: apps/v1 metadata: name: moac - namespace: mayastor + namespace: {{ .Release.Namespace }} spec: replicas: 1 selector: @@ -45,7 +44,7 @@ spec: - name: moac image: {{ include "mayastorImagesPrefix" . }}mayadata/moac:{{ .Values.mayastorImagesTag }} - imagePullPolicy: Always + imagePullPolicy: {{ .Values.mayastorImagePullPolicy }} args: - "--csi-address=$(CSI_ENDPOINT)" - "--namespace=$(MY_POD_NAMESPACE)" @@ -62,23 +61,6 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - ports: - - containerPort: 4000 - protocol: TCP - name: "rest-api" volumes: - name: socket-dir - emptyDir: ---- -kind: Service -apiVersion: v1 -metadata: - name: moac - namespace: mayastor -spec: - selector: - app: moac - ports: - - protocol: TCP - port: 4000 - targetPort: 4000 + emptyDir: \ No newline at end of file diff --git a/chart/templates/moac-rbac.yaml b/chart/templates/moac-rbac.yaml index 464799af5..ae271ca2d 100644 --- a/chart/templates/moac-rbac.yaml +++ b/chart/templates/moac-rbac.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: moac - namespace: mayastor + namespace: {{ .Release.Namespace }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -86,7 +86,7 @@ metadata: subjects: - kind: ServiceAccount name: moac - namespace: mayastor + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: moac diff --git a/chart/templates/nats-deployment.yaml b/chart/templates/nats-deployment.yaml index a64702a5e..b44e44785 100644 --- a/chart/templates/nats-deployment.yaml +++ b/chart/templates/nats-deployment.yaml @@ -3,7 +3,7 @@ kind: Deployment apiVersion: apps/v1 metadata: name: nats - namespace: mayastor + namespace: {{ .Release.Namespace }} spec: replicas: 1 selector: @@ -27,7 +27,7 @@ kind: Service apiVersion: v1 metadata: name: nats - namespace: mayastor + namespace: {{ .Release.Namespace }} spec: selector: app: nats diff --git a/chart/templates/pool.yaml b/chart/templates/pool.yaml new file mode 100644 index 000000000..91ebae3e9 --- /dev/null +++ b/chart/templates/pool.yaml @@ -0,0 +1,15 @@ +{{- range .Values.mayastorPools }} +--- +apiVersion: "openebs.io/v1alpha1" +kind: MayastorPool +metadata: + # Name can be anything as long as it is unique + name: pool-on-{{ .node }} + # or let k8s to generate a unique pool name + #generateName: pool- + namespace: {{ $.Release.Namespace }} +spec: + node: {{ .node }} + # ATM only one disk device is supported (i.e. /dev/nvme1n1) + disks: ["{{ .device }}"] +{{- end }} \ No newline at end of file diff --git a/chart/values.yaml b/chart/values.yaml index 427b5b163..dcd79b1a9 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,2 +1,8 @@ -mayastorImagesTag: latest +mayastorImagesTag: develop +mayastorImagePullPolicy: Always +mayastorCpuCount: "2" +mayastorHugePagesGiB: "1" mayastorImagesRepo: "" +mayastorPools: + - node: "NODE_NAME" + device: "DEVICE" \ No newline at end of file diff --git a/deploy/README.md b/deploy/README.md index 8aab60cf3..31819f4ef 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -8,6 +8,7 @@ It is now hosted at [GitBook](https://mayastor.gitbook.io/introduction/). # Deployment Files -The files in this directory are provided as a convenience, a template for the successful test deployment of Mayastor to a cluster. +The files in this directory are provided as a convenience, a template for the +successful test deployment of Mayastor to a cluster. -**NOTE:** ALL versions of deployment files from v0.4.0 specify the 'latest' tag on Mayastor container images. If you wish to deploy earlier versions, your own builds, or a nightly build, you must change the image tags accordingly. +Most of them are generated from helm templates in `chart/` directory. diff --git a/deploy/csi-daemonset.yaml b/deploy/csi-daemonset.yaml index f5f8cecae..43b8a4cc1 100644 --- a/deploy/csi-daemonset.yaml +++ b/deploy/csi-daemonset.yaml @@ -30,7 +30,7 @@ spec: # the same. containers: - name: mayastor-csi - image: mayadata/mayastor-csi:latest + image: mayadata/mayastor-csi:develop imagePullPolicy: Always # we need privileged because we mount filesystems and use mknod securityContext: diff --git a/deploy/fio.yaml b/deploy/fio.yaml index 44a3dbfd0..f754a1c2c 100644 --- a/deploy/fio.yaml +++ b/deploy/fio.yaml @@ -17,3 +17,6 @@ spec: volumeMounts: - mountPath: "/volume" name: ms-volume + #volumeDevices: + # - devicePath: /dev/xvda + # name: ms-volume \ No newline at end of file diff --git a/deploy/mayastor-daemonset-config.yaml b/deploy/mayastor-daemonset-config.yaml deleted file mode 100644 index a5b4ed83f..000000000 --- a/deploy/mayastor-daemonset-config.yaml +++ /dev/null @@ -1,106 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - namespace: mayastor - name: mayastor - labels: - openebs/engine: mayastor -spec: - selector: - matchLabels: - app: mayastor - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - minReadySeconds: 10 - template: - metadata: - labels: - app: mayastor - spec: - hostNetwork: true - # To resolve services from mayastor namespace - dnsPolicy: ClusterFirstWithHostNet - nodeSelector: - openebs.io/engine: mayastor - kubernetes.io/arch: amd64 - # NOTE: Each container must have mem/cpu limits defined in order to - # belong to Guaranteed QoS class, hence can never get evicted in case of - # pressure unless they exceed those limits. limits and requests must be - # the same. - initContainers: - - name: message-bus-probe - image: busybox:latest - command: ['sh', '-c', 'until nc -vz nats 4222; do echo "Waiting for message bus..."; sleep 1; done;'] - containers: - - name: mayastor - image: mayadata/mayastor:latest - imagePullPolicy: Always - env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: IMPORT_NEXUSES - value: "false" - args: - - "-N$(MY_NODE_NAME)" - - "-g$(MY_POD_IP)" - - "-nnats" - - "-y/var/local/mayastor/config.yaml" - - "-C/var/local/mayastor/child-status-config.yaml" - securityContext: - privileged: true - volumeMounts: - - name: device - mountPath: /dev - - name: dshm - mountPath: /dev/shm - - name: configlocation - mountPath: /var/local/mayastor/ - - name: config - mountPath: /var/local/mayastor/config.yaml - - name: child-status-config - mountPath: /var/local/mayastor/child-status-config.yaml - resources: - limits: - cpu: "1" - memory: "512Mi" - hugepages-2Mi: "1Gi" - requests: - cpu: "1" - memory: "512Mi" - hugepages-2Mi: "1Gi" - ports: - - containerPort: 10124 - protocol: TCP - name: mayastor - volumes: - - name: device - hostPath: - path: /dev - type: Directory - - name: dshm - emptyDir: - medium: Memory - sizeLimit: "1Gi" - - name: hugepage - emptyDir: - medium: HugePages - - name: configlocation - hostPath: - path: /var/local/mayastor/ - type: DirectoryOrCreate - - name: config - hostPath: - path: /var/local/mayastor/config.yaml - type: FileOrCreate - - name: child-status-config - hostPath: - path: /var/local/mayastor/child-status-config.yaml - type: FileOrCreate diff --git a/deploy/mayastor-daemonset.yaml b/deploy/mayastor-daemonset.yaml index 3faf66fef..bc2b85e41 100644 --- a/deploy/mayastor-daemonset.yaml +++ b/deploy/mayastor-daemonset.yaml @@ -22,7 +22,7 @@ spec: app: mayastor spec: hostNetwork: true - # To resolve services from mayastor namespace + # To resolve services from mayastor's namespace dnsPolicy: ClusterFirstWithHostNet nodeSelector: openebs.io/engine: mayastor @@ -33,7 +33,7 @@ spec: command: ['sh', '-c', 'until nc -vz nats 4222; do echo "Waiting for message bus..."; sleep 1; done;'] containers: - name: mayastor - image: mayadata/mayastor:latest + image: mayadata/mayastor:develop imagePullPolicy: Always env: - name: MY_NODE_NAME @@ -61,7 +61,7 @@ spec: - "-g$(MY_POD_IP)" - "-nnats" - "-y/var/local/mayastor/config.yaml" - - "-m0x3" + - "-l1,2" securityContext: privileged: true volumeMounts: diff --git a/deploy/moac-deployment.yaml b/deploy/moac-deployment.yaml index 22e24a1bb..61450ae78 100644 --- a/deploy/moac-deployment.yaml +++ b/deploy/moac-deployment.yaml @@ -1,19 +1,5 @@ --- # Source: mayastor/templates/moac-deployment.yaml -kind: Service -apiVersion: v1 -metadata: - name: moac - namespace: mayastor -spec: - selector: - app: moac - ports: - - protocol: TCP - port: 4000 - targetPort: 4000 ---- -# Source: mayastor/templates/moac-deployment.yaml kind: Deployment apiVersion: apps/v1 metadata: @@ -59,7 +45,7 @@ spec: mountPath: /var/lib/csi/sockets/pluginproxy/ - name: moac - image: mayadata/moac:latest + image: mayadata/moac:develop imagePullPolicy: Always args: - "--csi-address=$(CSI_ENDPOINT)" @@ -77,10 +63,6 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - ports: - - containerPort: 4000 - protocol: TCP - name: "rest-api" volumes: - name: socket-dir emptyDir: diff --git a/deploy/monitor/README.md b/deploy/monitor/README.md deleted file mode 100644 index 3729ebab1..000000000 --- a/deploy/monitor/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Monitoring extension for MayaStor - -Currently it shows two graphs: iops and bandwidth for arbitrary replica. -The monitoring stack consists of: - -* telegraf: gathering stats from mayastor REST API endpoint -* influxdb: database for the stats -* grafana: graphical frontend - -Note that this is just a proof of concept for showing "something" at -events like KubeCon. Monitoring for MayaStor will need to be designed -from scratch at some point in future when requirements are clear. - -Metrics in influxDB don't reside on persistent volume so when the pod -is restarted, all saved measurements are gone. - -# Deployment - -We assume that mayastor (including moac) has been already deployed to -`mayastor` namespace. - -1. Create configmap holding configuration files for grafana: - ```bash - kubectl -n mayastor create configmap grafana-config \ - --from-file=datasources.yaml=grafana/datasources.yaml \ - --from-file=dashboards.yaml=grafana/dashboards.yaml \ - --from-file=mayastor-dashboard.json=grafana/mayastor-dashboard.json - ``` - -2. Create configmap holding configuration of telegraf: - ```bash - kubectl create -f telegraf-config.yaml - ``` - -3. Deploy all three components: telegraf, influxdb and grafana: - ```bash - kubectl create -f monitor-deployment.yaml - ``` - -4. Get port of grafana to be used for external access (in this case 30333): - ```bash - kubectl -n mayastor get svc - ``` - ``` - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - grafana NodePort 10.0.0.88 80:30333/TCP 3m10s - moac ClusterIP 10.0.0.184 4000/TCP 113m - ``` - -5. Put URL in following form to your web browser: - `http://:/` (user/password is "admin"). - Choose mayastor dashboard. diff --git a/deploy/monitor/grafana/dashboards.yaml b/deploy/monitor/grafana/dashboards.yaml deleted file mode 100644 index 3b978e625..000000000 --- a/deploy/monitor/grafana/dashboards.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: 1 - -providers: -- name: 'default' - orgId: 1 - folder: '' - type: file - disableDeletion: false - updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards - options: - path: /var/lib/grafana/dashboards diff --git a/deploy/monitor/grafana/datasources.yaml b/deploy/monitor/grafana/datasources.yaml deleted file mode 100644 index e8053ac00..000000000 --- a/deploy/monitor/grafana/datasources.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# config file version -apiVersion: 1 - -# list of datasources to insert/update depending -# what's available in the database -datasources: - # name of the datasource. Required -- name: influxdb - # datasource type. Required - type: influxdb - # access mode. proxy or direct (Server or Browser in the UI). Required - access: proxy - # org id. will default to orgId 1 if not specified - orgId: 1 - # url - url: http://127.0.0.1:8086 - # database password, if used - password: telegraf - # database user, if used - user: telegraf - # database name, if used - database: mayastor - # version - version: 1 - # allow users to edit datasources from the UI. - editable: true diff --git a/deploy/monitor/grafana/mayastor-dashboard.json b/deploy/monitor/grafana/mayastor-dashboard.json deleted file mode 100644 index 60df97c50..000000000 --- a/deploy/monitor/grafana/mayastor-dashboard.json +++ /dev/null @@ -1,440 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "iteration": 1578992203638, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "influxdb", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "repeatDirection": "h", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "alias": "read", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT non_negative_derivative(max(\"num_read_ops\"), 1s) FROM \"replica\" WHERE (\"uuid\" =~ /^$uuid$/ AND \"node\" =~ /^$node$/) AND time >= now() - 15m GROUP BY time(500ms) fill(null)", - "rawQuery": true, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - }, - { - "alias": "write", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT non_negative_derivative(max(\"num_write_ops\"), 1s) FROM \"replica\" WHERE (\"uuid\" =~ /^$uuid$/ AND \"node\" =~ /^$node$/) AND time >= now() - 15m GROUP BY time(500ms) fill(null);", - "rawQuery": true, - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "IOPS", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transparent": true, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "influxdb", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeatDirection": "h", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "alias": "read", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT non_negative_derivative(max(\"bytes_read\"), 1s) FROM \"replica\" WHERE (\"uuid\" =~ /^$uuid$/ AND \"node\" =~ /^$node$/) AND time >= now() - 15m GROUP BY time(500ms) fill(null)", - "rawQuery": true, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - }, - { - "alias": "write", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT non_negative_derivative(max(\"bytes_written\"), 1s) FROM \"replica\" WHERE (\"uuid\" =~ /^$uuid$/ AND \"node\" =~ /^$node$/) AND time >= now() - 15m GROUP BY time(500ms) fill(null);", - "rawQuery": true, - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Bandwidth", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transparent": true, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "2s", - "schemaVersion": 21, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "tags": [], - "text": "node1", - "value": "node1" - }, - "datasource": "influxdb", - "definition": "SHOW TAG VALUES WITH KEY = \"node\"", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "node", - "options": [], - "query": "SHOW TAG VALUES WITH KEY = \"node\"", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "1b232114-7c71-40a9-8f35-004fc3878187", - "value": "1b232114-7c71-40a9-8f35-004fc3878187" - }, - "datasource": "influxdb", - "definition": "SHOW TAG VALUES WITH KEY = \"uuid\"", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "uuid", - "options": [], - "query": "SHOW TAG VALUES WITH KEY = \"uuid\"", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "2s", - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "MayaStor dashboard", - "uid": "EXv5v7PWk", - "version": 1 -} diff --git a/deploy/monitor/monitor-deployment.yaml b/deploy/monitor/monitor-deployment.yaml deleted file mode 100644 index abaf84ec9..000000000 --- a/deploy/monitor/monitor-deployment.yaml +++ /dev/null @@ -1,106 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: influxdb-secrets - namespace: mayastor -type: Opaque -stringData: - INFLUXDB_DB: mayastor - INFLUXDB_USER: telegraf - INFLUXDB_USER_PASSWORD: telegraf ---- -apiVersion: v1 -kind: Secret -metadata: - name: grafana-secrets - namespace: mayastor -type: Opaque -stringData: - GF_SECURITY_ADMIN_USER: admin - GF_SECURITY_ADMIN_PASSWORD: admin ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: monitor - namespace: mayastor - labels: - app: monitor -spec: - replicas: 1 - selector: - matchLabels: - app: monitor - template: - metadata: - labels: - app: monitor - spec: - containers: - - name: telegraf - image: telegraf:latest - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: /etc/telegraf/telegraf.conf - name: telegraf-config - subPath: telegraf.conf - readOnly: true - envFrom: - - secretRef: - name: influxdb-secrets - - name: influxdb - image: influxdb:latest - imagePullPolicy: IfNotPresent - envFrom: - - secretRef: - name: influxdb-secrets - - name: grafana - image: grafana/grafana:latest - imagePullPolicy: IfNotPresent - envFrom: - - secretRef: - name: grafana-secrets - env: - - name: INFLUXDB_HOST - value: "127.0.0.1" - - name: GF_SERVER_HTTP_PORT - value: "3000" - ports: - - containerPort: 3000 - protocol: TCP - volumeMounts: - - mountPath: /etc/grafana/provisioning/datasources/datasources.yaml - name: grafana-config - readOnly: true - subPath: datasources.yaml - - mountPath: /etc/grafana/provisioning/dashboards/dashboards.yaml - name: grafana-config - readOnly: true - subPath: dashboards.yaml - - mountPath: /var/lib/grafana/dashboards/mayastor-dashboard.json - name: grafana-config - readOnly: true - subPath: mayastor-dashboard.json - volumes: - - name: grafana-config - configMap: - name: grafana-config - - name: telegraf-config - configMap: - name: telegraf-config ---- -kind: Service -apiVersion: v1 -metadata: - name: grafana - namespace: mayastor -spec: - selector: - app: monitor - type: NodePort - ports: - - protocol: TCP - port: 80 - targetPort: 3000 - nodePort: 30333 diff --git a/deploy/monitor/telegraf-config.yaml b/deploy/monitor/telegraf-config.yaml deleted file mode 100644 index 1c449d251..000000000 --- a/deploy/monitor/telegraf-config.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: telegraf-config - namespace: mayastor -data: - telegraf.conf: |+ - # Configuration for telegraf agent - [agent] - ## Default data collection interval for all inputs - interval = "5s" - ## Rounds collection interval to 'interval' - ## ie, if interval="10s" then always collect on :00, :10, :20, etc. - round_interval = true - ## Telegraf will send metrics to outputs in batches of at most - ## metric_batch_size metrics. - ## This controls the size of writes that Telegraf sends to output plugins. - metric_batch_size = 1000 - ## For failed writes, telegraf will cache metric_buffer_limit metrics for each - ## output, and will flush this buffer on a successful write. Oldest metrics - ## are dropped first when this buffer fills. - ## This buffer only fills when writes fail to output plugin(s). - metric_buffer_limit = 10000 - ## Collection jitter is used to jitter the collection by a random amount. - ## Each plugin will sleep for a random time within jitter before collecting. - ## This can be used to avoid many plugins querying things like sysfs at the - ## same time, which can have a measurable effect on the system. - collection_jitter = "0s" - ## Default flushing interval for all outputs. Maximum flush_interval will be - ## flush_interval + flush_jitter - flush_interval = "5s" - ## Jitter the flush interval by a random amount. This is primarily to avoid - ## large write spikes for users running a large number of telegraf instances. - ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s - flush_jitter = "0s" - ## By default or when set to "0s", precision will be set to the same - ## timestamp order as the collection interval, with the maximum being 1s. - ## ie, when interval = "10s", precision will be "1s" - ## when interval = "250ms", precision will be "1ms" - ## Precision will NOT be used for service inputs. It is up to each individual - ## service input to set the timestamp at the appropriate precision. - ## Valid time units are "ns", "us" (or "µs"), "ms", "s". - precision = "" - ## Logging configuration: - ## Run telegraf with debug log messages. - debug = false - ## Run telegraf in quiet mode (error log messages only). - quiet = false - ## Specify the log file name. The empty string means to log to stderr. - logfile = "" - ## Override default hostname, if empty use os.Hostname() - hostname = "" - ## If set to true, do no set the "host" tag in the telegraf agent. - omit_hostname = true - - # Configuration for sending metrics to InfluxDB - [[outputs.influxdb]] - ## The full HTTP or UDP URL for your InfluxDB instance. - urls = ["http://127.0.0.1:8086"] - ## The target database for metrics; will be created as needed. - database = "$INFLUXDB_DB" - timeout = "5s" - ## HTTP Basic Auth - username = "$INFLUXDB_USER" - password = "$INFLUXDB_USER_PASSWORD" - - # Configuration for gathering stats data from mayastor - [[inputs.http]] - name_override = "replica" - urls = ["http://moac:4000/stats"] - timeout = "5s" - data_format = "json" - json_query = "" - tag_keys = ["uuid", "pool", "node"] - json_time_key = "timestamp" - json_time_format = "2006-01-02T15:04:05Z07:00" - # default is UTC - json_timezone = "" diff --git a/deploy/namespace.yaml b/deploy/namespace.yaml deleted file mode 100644 index 1e426a7eb..000000000 --- a/deploy/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Namespace -apiVersion: v1 -metadata: - name: mayastor - labels: - name: mayastor diff --git a/deploy/pool.yaml b/deploy/pool.yaml index 6886ad33f..04ad4387d 100644 --- a/deploy/pool.yaml +++ b/deploy/pool.yaml @@ -1,10 +1,14 @@ +--- +# Source: mayastor/templates/pool.yaml apiVersion: "openebs.io/v1alpha1" kind: MayastorPool metadata: - #name: unique - generateName: pool- + # Name can be anything as long as it is unique + name: pool-on-NODE_NAME + # or let k8s to generate a unique pool name + #generateName: pool- namespace: mayastor spec: - node: MYHOSTNAME - disks: ["MYBLOCKDEVICE"] - + node: NODE_NAME + # ATM only one disk device is supported (i.e. /dev/nvme1n1) + disks: ["DEVICE"] diff --git a/deploy/pvc.yaml b/deploy/pvc.yaml index 769ffdad7..92eb66b6a 100644 --- a/deploy/pvc.yaml +++ b/deploy/pvc.yaml @@ -5,7 +5,9 @@ metadata: spec: accessModes: - ReadWriteOnce + # Change to "Block" for raw block device + volumeMode: Filesystem resources: requests: storage: 64Mi - storageClassName: mayastor + storageClassName: mayastor-nvmf diff --git a/scripts/generate-deploy-yamls.sh b/scripts/generate-deploy-yamls.sh index 21fb5d7b5..ea2e47b58 100755 --- a/scripts/generate-deploy-yamls.sh +++ b/scripts/generate-deploy-yamls.sh @@ -43,10 +43,10 @@ tmpd=$(mktemp -d /tmp/generate-deploy.sh.XXXXXXXX) # shellcheck disable=SC2064 trap "rm -fr '$tmpd'" HUP QUIT EXIT TERM INT -if [ "$mayastor_images_repo" = "NONE" ]; then - helm template --set "mayastorImagesTag=$1" mayastor "$SCRIPTDIR/../chart" --output-dir="$tmpd" --namespace mayastor -else - helm template --set "mayastorImagesTag=$1,mayastorImagesRepo=$mayastor_images_repo" mayastor "$SCRIPTDIR/../chart" --output-dir="$tmpd" --namespace mayastor +template_params="mayastorImagesTag=$1" +if [ "$mayastor_images_repo" != "NONE" ]; then + template_params="$template_params,mayastorImagesRepo=$mayastor_images_repo" fi +helm template --set "$template_params" mayastor "$SCRIPTDIR/../chart" --output-dir="$tmpd" --namespace mayastor mv "$tmpd"/mayastor/templates/*.yaml "$TARGET_DIR" diff --git a/test/e2e/install/install_test.go b/test/e2e/install/install_test.go index cb8ca6642..7e5906318 100644 --- a/test/e2e/install/install_test.go +++ b/test/e2e/install/install_test.go @@ -114,6 +114,13 @@ func getDeployYamlDir() string { return path.Clean(filename + "/../../../../deploy") } +// Create mayastor namespace +func createNamespace() { + cmd := exec.Command("kubectl", "create", "namespace", "mayastor") + out, err := cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred(), "%s", out) +} + // Helper for passing yaml from the deploy directory to kubectl func applyDeployYaml(filename string) { cmd := exec.Command("kubectl", "apply", "-f", filename) @@ -228,7 +235,7 @@ func installMayastor() { fmt.Printf("tag %v, registry %v, # of mayastor instances=%v\n", imageTag, registryAddress, numMayastorInstances) // FIXME use absolute paths, do not depend on CWD - applyDeployYaml("namespace.yaml") + createNamespace() applyDeployYaml("storage-class.yaml") applyDeployYaml("moac-rbac.yaml") applyDeployYaml("mayastorpoolcrd.yaml") diff --git a/test/e2e/uninstall/uninstall_test.go b/test/e2e/uninstall/uninstall_test.go index 084955602..3d5aa389b 100644 --- a/test/e2e/uninstall/uninstall_test.go +++ b/test/e2e/uninstall/uninstall_test.go @@ -38,6 +38,13 @@ func deleteCRD(crdName string) { _ = cmd.Run() } +// Create mayastor namespace +func deleteNamespace() { + cmd := exec.Command("kubectl", "delete", "namespace", "mayastor") + out, err := cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred(), "%s", out) +} + // Teardown mayastor on the cluster under test. // We deliberately call out to kubectl, rather than constructing the client-go // objects, so that we can verfiy the local deploy yamls are correct. @@ -129,7 +136,7 @@ func teardownMayastor() { if forceDeleted { logf.Log.Info("WARNING: Mayastor pods were force deleted at uninstall!!!") } - deleteDeployYaml("namespace.yaml") + deleteNamespace() // delete the namespace prior to possibly failing the uninstall // to yield a reusable cluster on fail. Expect(podsDeleted).To(BeTrue()) @@ -147,7 +154,7 @@ func teardownMayastor() { // More verbose here as deleting the namespace is often where this // test hangs. logf.Log.Info("Deleting the mayastor namespace") - deleteDeployYaml("namespace.yaml") + deleteNamespace() logf.Log.Info("Deleted the mayastor namespace") } }