HCP creation demo (#947)

Signed-off-by: Gerd Oberlechner <[email protected]>

demo/.gitignore

@@ -0,0 +1,2 @@
+cluster.json
+node_pool.json

demo/01-register-sub.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+curl -i -X PUT "localhost:8443/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b?api-version=2.0" --json '{"state":"Registered", "registrationDate": "now", "properties": { "tenantId": "64dc69e4-d083-49fc-9569-ebece1dd1408"}}'

demo/02-customer-infra.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+source env_vars
+
+az group create --name "${CUSTOMER_RG_NAME}" --location ${LOCATION}
+
+az network nsg create -g ${CUSTOMER_RG_NAME} --name ${CUSTOMER_NSG} --location ${LOCATION}
+NSG_ID=$(az network nsg list --query "[?name=='${CUSTOMER_NSG}'].id" -g ${CUSTOMER_RG_NAME} -o tsv)
+
+az network vnet create \
+  --name ${CUSTOMER_VNET_NAME} \
+  --resource-group ${CUSTOMER_RG_NAME} \
+  --address-prefix 10.0.0.0/16 \
+  --subnet-name ${CUSTOMER_VNET_SUBNET1} \
+  --subnet-prefixes 10.0.0.0/24 \
+  --nsg "${NSG_ID}"  --location ${LOCATION}

demo/03-create-cluster.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+source env_vars
+
+CURRENT_DATE=$(date -u +"%Y-%m-%dT%H:%M:%S+00:00")
+
+CLUSTER_TMPL_FILE="cluster.tmpl.json"
+CLUSTER_FILE="cluster.json"
+
+NSG_ID=$(az network nsg list --query "[?name=='${CUSTOMER_NSG}'].id" -o tsv)
+SUBNET_ID=$(az network vnet subnet show -g ${CUSTOMER_RG_NAME} --vnet-name ${CUSTOMER_VNET_NAME} --name ${CUSTOMER_VNET_SUBNET1} --query id -o tsv)
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+
+jq \
+  --arg managed_rg "$MANAGED_RESOURCE_GROUP" \
+  --arg subnet_id "$SUBNET_ID" \
+  --arg nsg_id "$NSG_ID" \
+  '.properties.spec.platform.managedResourceGroup = $managed_rg | .properties.spec.platform.subnetId = $subnet_id | .properties.spec.platform.networkSecurityGroupId = $nsg_id' "${CLUSTER_TMPL_FILE}" > ${CLUSTER_FILE}
+
+curl -i -X PUT "localhost:8443/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${CUSTOMER_RG_NAME}/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/${CLUSTER_NAME}?api-version=2024-06-10-preview" \
+  --header "X-Ms-Arm-Resource-System-Data: {\"createdBy\": \"${USER}\", \"createdByType\": \"User\", \"createdAt\": \"${CURRENT_DATE}\"}" \
+  --json @${CLUSTER_FILE}

demo/04-create-nodepool.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+source env_vars
+
+CURRENT_DATE=$(date -u +"%Y-%m-%dT%H:%M:%S+00:00")
+
+NODEPOOL_TMPL_FILE="node_pool.tmpl.json"
+NODEPOOL_FILE="node_pool.json"
+
+SUBNET_ID=$(az network vnet subnet show -g ${CUSTOMER_RG_NAME} --vnet-name ${CUSTOMER_VNET_NAME} --name ${CUSTOMER_VNET_SUBNET1} --query id -o tsv)
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+
+jq \
+  --arg managed_rg "$MANAGED_RESOURCE_GROUP" \
+  --arg subnet_id "$SUBNET_ID" \
+  --arg nsg_id "$NSG_ID" \
+  '.properties.spec.platform.subnetId = $subnet_id' "${NODEPOOL_TMPL_FILE}" > ${NODEPOOL_FILE}
+
+
+curl -i -X PUT "localhost:8443/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${CUSTOMER_RG_NAME}/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/${CLUSTER_NAME}/nodePools/${NP_NAME}?api-version=2024-06-10-preview" \
+  --header "X-Ms-Arm-Resource-System-Data: {\"createdBy\": \"${USER}\", \"createdByType\": \"User\", \"createdAt\": \"${CURRENT_DATE}\"}" \
+  --json @${NODEPOOL_FILE}

demo/05-delete-cluster.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source env_vars
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+
+curl -i -X DELETE "localhost:8443/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${CUSTOMER_RG_NAME}/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/${CLUSTER_NAME}?api-version=2024-06-10-preview"

demo/README.md

@@ -0,0 +1,148 @@
+# Create an HCP
+
+## Prepare
+
+* have a `KUBECONFIG` for a SC and MC, e.g. for the [integrated DEV environment](../dev-infrastructure/docs/development-setup.md#access-integrated-dev-environment)
+* port-forward RP running on SC: `kubectl port-forward -n aro-hcp svc/aro-hcp-frontend 8443:8443`
+* (optional but useful) port-forward CS running on SC: `kubectl port-forward -n cluster-service svc/clusters-service 8001:8000`
+* (optional but useful) port-forward Maestro running on SC: `kubectl port-forward -n maestro svc/maestro 8002:8000`
+
+## Register the subscription with the RP
+
+The RP needs to know the subscription in order to be able to create cluters in it.
+Run the following command `once` to make the subscription known to the RP.
+
+```bash
+./01-register-sub.sh
+```
+
+## Create VNET and NSG
+
+We provision HCPs with BYO VNET, so we need to create the VNET, Subnet and NSG upfront.
+
+```bash
+./02-customer-infra.sh
+```
+
+The resources are created in a resourcegroup named `$USER-net-rg`.
+
+## Create cluster
+
+Create an HCP by sending a request to the port-forwarded RP. You can find a payload template in `cluster.tmpl.json`.
+To fill that template with some user specific defaults and send it to the RP, run the following command:
+
+```bash
+./03-create-cluster.sh
+```
+
+This creates an HCP named `$USER`. If you want to use a different name, run
+
+```bash
+CLUSTER_NAME=abc ./03-create-cluster.sh
+```
+
+Observe the cluster creation with `./query-cluster-rp.sh` until `properties.provisioningState` is (hopefully) `Succeeded`.
+`properties.spec.api.url` holds the URL to the API server of the HCP.
+
+See [Get the kubeconfig for an HCP](#get-the-kubeconfig-for-an-hcp) on how to get the kubeconfig for the HCP.
+
+## Create nodepool
+
+```bash
+./04-create-nodepool.sh
+```
+
+This creates a nodepool named `np-1` for the previously created cluster.
+
+To check progress on
+
+* the RP, run `./query-nodepool-rp.sh`
+* the MC, run `kubectl get nodepool -A` and `kubectl get azuremachine -A`
+* within the HCP, run `kubectl get nodes` while using your HCP kubeconfig
+
+## Delete cluster
+
+```bash
+./05-delete-cluster.sh
+```
+
+## Observe and debug
+
+### Check RP pod logs
+
+```bash
+kubectl logs deployment/aro-hcp-frontend -c aro-hcp-frontend -n aro-hcp -f
+kubectl logs deployment/aro-hcp-backend -c aro-hcp-backend -n aro-hcp -f
+```
+
+### Check cluster state in RP
+
+```bash
+./query-cluster-rp.sh
+```
+
+### Check CS pod logs
+
+```bash
+kubectl logs deployment/clusters-service -c service -f
+```
+
+### Check cluster state in CS
+
+```bash
+curl localhost:8001/api/clusters_mgmt/v1/clusters | jq
+```
+
+### Check Maestro logs
+
+```bash
+kubectl logs deployment/maestro -n maestro -c service -f
+```
+
+### Check MC consumer registration in Maestro
+
+```bash
+curl localhost:8002/api/maestro/v1/consumers | jq
+```
+
+### Check MC resource bundles in Maestro
+
+```bash
+curl localhost:8002/api/maestro/v1/resource-bundles | jq
+```
+
+### Check for manifestwork on MC
+
+```bash
+CS_CLUSTER_ID=$(curl localhost:8001/api/clusters_mgmt/v1/clusters | jq .items[0].id -r)
+kubectl get manifestwork -n local-cluster | grep ^${CS_CLUSTER_ID}
+```
+
+### Check for `HostedCluster` CR on MC
+
+```bash
+kubectl get hostedcluster -A
+```
+
+### Check for namespaces on MC
+
+```bash
+kubectl get ns | grep "ocm.*${CS_CLUSTER_ID}"
+```
+
+### Two namespaces should show up
+
+* `ocm-xxx-${CS_CLUSTER_ID}` - contains Hypershift CRs and secrets for an HCP
+* `ocm-xxx-${CS_CLUSTER_ID}-${CLUSTER_NAME}` - contains the hosted controlplane (e.g. pods, secrets, ...)
+
+### Get the kubeconfig for an HCP
+
+```bash
+kubectl get secret -n ocm-aro-hcp-dev-${CS_CLUSTER_ID} ${CLUSTER_NAME}-admin-kubeconfig -o json | jq .data.kubeconfig -r | base64 -d > my.kubeconfig
+```
+
+### Check nodepool on RP
+
+```bash
+./query-nodepool-rp.sh
+```

demo/cluster.tmpl.json

@@ -0,0 +1,30 @@
+{
+  "properties": {
+    "spec": {
+      "version": {
+        "id": "openshift-v4.17.0",
+        "channelGroup": "stable"
+      },
+      "dns": {},
+      "network": {
+        "networkType": "OVNKubernetes",
+        "podCidr": "10.128.0.0/14",
+        "serviceCidr": "172.30.0.0/16",
+        "machineCidr": "10.0.0.0/16",
+        "hostPrefix": 23
+      },
+      "console": {},
+      "api": {
+        "visibility": "public"
+      },
+      "proxy": {},
+      "platform": {
+        "managedResourceGroup": "$managed-resource-group",
+        "subnetId": "/subscriptions/$sub/resourceGroups/$customer-rg/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1",
+        "outboundType": "loadBalancer",
+        "networkSecurityGroupId": "/subscriptions/$sub/resourceGroups/$customer-rg/providers/Microsoft.Network/networkSecurityGroups/customer-nsg"
+      },
+      "externalAuth": {}
+    }
+  }
+}

demo/env_vars

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+
+export CUSTOMER_RG_NAME="$USER-net-rg"
+export CUSTOMER_VNET_NAME="customer-vnet"
+export CUSTOMER_VNET_SUBNET1="customer-subnet-1"
+export CUSTOMER_NSG="customer-nsg"
+export LOCATION="westus3"
+if [ -z "$CLUSTER_NAME" ]; then
+    export CLUSTER_NAME="$USER"
+fi
+MANAGED_RESOURCE_GROUP="$CLUSTER_NAME-rg"
+
+if [ -z "$NP_NAME" ]; then
+    export NP_NAME="np-1"
+fi

demo/node_pool.tmpl.json

@@ -0,0 +1,17 @@
+{
+  "properties": {
+    "spec": {
+      "version": {
+        "id": "openshift-v4.17.0",
+        "channelGroup": "stable"
+      },
+      "platform": {
+        "subnetId": "/subscriptions/$sub/resourceGroups/$customer-rg/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1",
+        "vmSize": "Standard_D8s_v3",
+        "diskSizeGiB": 30,
+        "diskStorageAccountType": "StandardSSD_LRS"
+      },
+      "replicas": 2
+    }
+  }
+}

demo/query-cluster-rp.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+source env_vars
+
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+
+curl "localhost:8443/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${CUSTOMER_RG_NAME}/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/${CLUSTER_NAME}?api-version=2024-06-10-preview" | jq

demo/query-nodepool-rp.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+source env_vars
+
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+
+curl "localhost:8443/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${CUSTOMER_RG_NAME}/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/${CLUSTER_NAME}/nodePools/${NP_NAME}?api-version=2024-06-10-preview" | jq