-
Notifications
You must be signed in to change notification settings - Fork 57
Installing OCS 4 (OpenShift Container Storage)
cd into the setup-dir of the cluster where you want to install OCS 4
cd <setup-dir>
Add three new nodes to the cluster with two additional disks on each. The first will be used for filesystem type storage and the second one will be used for block type storage. Set the values (in GB) accordingly. Also remember to use the same values when creating storage cluster.
./add_node.sh --cpu 4 --memory 16000 --add-disk 50 --add-disk 100 --name storage-1
./add_node.sh --cpu 4 --memory 16000 --add-disk 50 --add-disk 100 --name storage-2
./add_node.sh --cpu 4 --memory 16000 --add-disk 50 --add-disk 100 --name storage-3
Wait for the nodes CSR and approve them (2 csr per node):
while true; do
for x in $(oc get csr | grep Pending | awk '{print $1}'); do
oc adm certificate approve $x;
done;
sleep 5;
done
Wait for the new nodes to become ready:
watch oc get nodes
Load the env file of the cluster (we need $CLUSTER_NAME and $BASE_DOM)
source ./env
oc label node storage-1.${CLUSTER_NAME}.${BASE_DOM} cluster.ocs.openshift.io/openshift-storage=''
oc label node storage-2.${CLUSTER_NAME}.${BASE_DOM} cluster.ocs.openshift.io/openshift-storage=''
oc label node storage-3.${CLUSTER_NAME}.${BASE_DOM} cluster.ocs.openshift.io/openshift-storage=''
Note: If the operator is already setup, you can skip this.
oc new-project local-storage
cat <<EOF | oc create -f -
apiVersion: operators.coreos.com/v1
kind: OperatorGroup
metadata:
name: local-storage-opg
namespace: local-storage
spec:
targetNamespaces:
- local-storage
EOF
Create the Subscription (Note: Set the channel to the required version)
cat <<EOF | oc create -f -
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
name: local-storage-operator
namespace: local-storage
spec:
channel: "4.4"
name: local-storage-operator
source: redhat-operators
sourceNamespace: openshift-marketplace
EOF
Wait for the operator installation to finish
watch oc get csv -n local-storage
First disk on each node (vdb) will be used for Filesystem type storage.
cat <<EOF | oc create -f -
apiVersion: "local.storage.openshift.io/v1"
kind: "LocalVolume"
metadata:
name: "local-disks-fs"
namespace: "local-storage"
spec:
nodeSelector:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- storage-1.${CLUSTER_NAME}.${BASE_DOM}
- storage-2.${CLUSTER_NAME}.${BASE_DOM}
- storage-3.${CLUSTER_NAME}.${BASE_DOM}
storageClassDevices:
- storageClassName: "lo-fs"
volumeMode: Filesystem
devicePaths:
- /dev/vdb
EOF
Second disk on each node (vdc) will be used for Block type storage.
cat <<EOF | oc create -f -
apiVersion: "local.storage.openshift.io/v1"
kind: "LocalVolume"
metadata:
name: "local-disks"
namespace: "local-storage"
spec:
nodeSelector:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- storage-1.${CLUSTER_NAME}.${BASE_DOM}
- storage-2.${CLUSTER_NAME}.${BASE_DOM}
- storage-3.${CLUSTER_NAME}.${BASE_DOM}
storageClassDevices:
- storageClassName: "lo-blk"
volumeMode: Block
devicePaths:
- /dev/vdc
EOF
Wait for the pods in local-storage namespace to become ready
watch oc get pods -n local-storage
Ensure that the lo-fs lo-blk storage classes have been created
oc get sc
Create the openshift-storage namespace/project
cat <<EOF | oc create -f -
apiVersion: v1
kind: Namespace
metadata:
labels:
openshift.io/cluster-monitoring: "true"
name: openshift-storage
spec: {}
EOF
Add the OperatorGroup
cat <<EOF | oc create -f -
apiVersion: operators.coreos.com/v1
kind: OperatorGroup
metadata:
name: openshift-storage-opg
namespace: openshift-storage
spec:
targetNamespaces:
- openshift-storage
EOF
Create the Subscription (Note: Set the channel to the required version)
cat <<EOF | oc create -f -
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
name: ocs-operator
namespace: openshift-storage
spec:
channel: "stable-4.4"
name: ocs-operator
source: redhat-operators
sourceNamespace: openshift-marketplace
EOF
Wait for the operator installation to finish
watch oc get csv -n openshift-storage
Set the requests.storage values according to the disks
cat <<EOF | oc create -f -
apiVersion: ocs.openshift.io/v1
kind: StorageCluster
metadata:
name: ocs-storagecluster
namespace: openshift-storage
spec:
manageNodes: false
monPVCTemplate:
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 50Gi
storageClassName: 'lo-fs'
volumeMode: Filesystem
resources:
mon:
requests: {}
limits: {}
mds:
requests: {}
limits: {}
rgw:
requests: {}
limits: {}
mgr:
requests: {}
limits: {}
noobaa-core:
requests: {}
limits: {}
noobaa-db:
requests: {}
limits: {}
storageDeviceSets:
- count: 1
dataPVCTemplate:
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 100Gi
storageClassName: 'lo-blk'
volumeMode: Block
name: ocs-deviceset
placement: {}
portable: true
replica: 3
resources:
requests: {}
limits: {}
EOF
Wait for pods in openshift-storage to get ready
watch oc get pods -n openshift-storage
Three new storage classes should be created
oc get sc
# oc get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
lo-blk kubernetes.io/no-provisioner Delete WaitForFirstConsumer false 18m
lo-fs kubernetes.io/no-provisioner Delete WaitForFirstConsumer false 19m
ocs-storagecluster-ceph-rbd openshift-storage.rbd.csi.ceph.com Delete Immediate false 15m
ocs-storagecluster-cephfs openshift-storage.cephfs.csi.ceph.com Delete Immediate false 15m
openshift-storage.noobaa.io openshift-storage.noobaa.io/obc Delete Immediate false 3m2s