diff --git a/demo/.gitignore b/demo/.gitignore new file mode 100644 index 000000000..427501537 --- /dev/null +++ b/demo/.gitignore @@ -0,0 +1,2 @@ +cluster.json +node_pool.json diff --git a/demo/01-register-sub.sh b/demo/01-register-sub.sh new file mode 100755 index 000000000..7f87d3b89 --- /dev/null +++ b/demo/01-register-sub.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +curl -i -X PUT "localhost:8443/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b?api-version=2.0" --json '{"state":"Registered", "registrationDate": "now", "properties": { "tenantId": "64dc69e4-d083-49fc-9569-ebece1dd1408"}}' diff --git a/demo/02-customer-infra.sh b/demo/02-customer-infra.sh new file mode 100755 index 000000000..fcc59648c --- /dev/null +++ b/demo/02-customer-infra.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +source env_vars + +az group create --name "${CUSTOMER_RG_NAME}" --location ${LOCATION} + +az network nsg create -g ${CUSTOMER_RG_NAME} --name ${CUSTOMER_NSG} --location ${LOCATION} +NSG_ID=$(az network nsg list --query "[?name=='${CUSTOMER_NSG}'].id" -g ${CUSTOMER_RG_NAME} -o tsv) + +az network vnet create \ + --name ${CUSTOMER_VNET_NAME} \ + --resource-group ${CUSTOMER_RG_NAME} \ + --address-prefix 10.0.0.0/16 \ + --subnet-name ${CUSTOMER_VNET_SUBNET1} \ + --subnet-prefixes 10.0.0.0/24 \ + --nsg "${NSG_ID}" --location ${LOCATION} diff --git a/demo/03-create-cluster.sh b/demo/03-create-cluster.sh new file mode 100755 index 000000000..b996c1c60 --- /dev/null +++ b/demo/03-create-cluster.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +source env_vars + +CURRENT_DATE=$(date -u +"%Y-%m-%dT%H:%M:%S+00:00") + +CLUSTER_TMPL_FILE="cluster.tmpl.json" +CLUSTER_FILE="cluster.json" + +NSG_ID=$(az network nsg list --query "[?name=='${CUSTOMER_NSG}'].id" -o tsv) +SUBNET_ID=$(az network vnet subnet show -g ${CUSTOMER_RG_NAME} --vnet-name ${CUSTOMER_VNET_NAME} --name ${CUSTOMER_VNET_SUBNET1} --query id -o tsv) +SUBSCRIPTION_ID=$(az account show --query id -o tsv) + +jq \ + --arg managed_rg "$MANAGED_RESOURCE_GROUP" \ + --arg subnet_id "$SUBNET_ID" \ + --arg nsg_id "$NSG_ID" \ + '.properties.spec.platform.managedResourceGroup = $managed_rg | .properties.spec.platform.subnetId = $subnet_id | .properties.spec.platform.networkSecurityGroupId = $nsg_id' "${CLUSTER_TMPL_FILE}" > ${CLUSTER_FILE} + +curl -i -X PUT "localhost:8443/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${CUSTOMER_RG_NAME}/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/${CLUSTER_NAME}?api-version=2024-06-10-preview" \ + --header "X-Ms-Arm-Resource-System-Data: {\"createdBy\": \"${USER}\", \"createdByType\": \"User\", \"createdAt\": \"${CURRENT_DATE}\"}" \ + --json @${CLUSTER_FILE} diff --git a/demo/04-create-nodepool.sh b/demo/04-create-nodepool.sh new file mode 100755 index 000000000..53e4b5d33 --- /dev/null +++ b/demo/04-create-nodepool.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +source env_vars + +CURRENT_DATE=$(date -u +"%Y-%m-%dT%H:%M:%S+00:00") + +NODEPOOL_TMPL_FILE="node_pool.tmpl.json" +NODEPOOL_FILE="node_pool.json" + +SUBNET_ID=$(az network vnet subnet show -g ${CUSTOMER_RG_NAME} --vnet-name ${CUSTOMER_VNET_NAME} --name ${CUSTOMER_VNET_SUBNET1} --query id -o tsv) +SUBSCRIPTION_ID=$(az account show --query id -o tsv) + +jq \ + --arg managed_rg "$MANAGED_RESOURCE_GROUP" \ + --arg subnet_id "$SUBNET_ID" \ + --arg nsg_id "$NSG_ID" \ + '.properties.spec.platform.subnetId = $subnet_id' "${NODEPOOL_TMPL_FILE}" > ${NODEPOOL_FILE} + + +curl -i -X PUT "localhost:8443/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${CUSTOMER_RG_NAME}/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/${CLUSTER_NAME}/nodePools/${NP_NAME}?api-version=2024-06-10-preview" \ + --header "X-Ms-Arm-Resource-System-Data: {\"createdBy\": \"${USER}\", \"createdByType\": \"User\", \"createdAt\": \"${CURRENT_DATE}\"}" \ + --json @${NODEPOOL_FILE} diff --git a/demo/05-delete-cluster.sh b/demo/05-delete-cluster.sh new file mode 100755 index 000000000..435686c86 --- /dev/null +++ b/demo/05-delete-cluster.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source env_vars +SUBSCRIPTION_ID=$(az account show --query id -o tsv) + +curl -i -X DELETE "localhost:8443/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${CUSTOMER_RG_NAME}/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/${CLUSTER_NAME}?api-version=2024-06-10-preview" diff --git a/demo/README.md b/demo/README.md new file mode 100644 index 000000000..fda2910ac --- /dev/null +++ b/demo/README.md @@ -0,0 +1,148 @@ +# Create an HCP + +## Prepare + +* have a `KUBECONFIG` for a SC and MC, e.g. for the [integrated DEV environment](../dev-infrastructure/docs/development-setup.md#access-integrated-dev-environment) +* port-forward RP running on SC: `kubectl port-forward -n aro-hcp svc/aro-hcp-frontend 8443:8443` +* (optional but useful) port-forward CS running on SC: `kubectl port-forward -n cluster-service svc/clusters-service 8001:8000` +* (optional but useful) port-forward Maestro running on SC: `kubectl port-forward -n maestro svc/maestro 8002:8000` + +## Register the subscription with the RP + +The RP needs to know the subscription in order to be able to create cluters in it. +Run the following command `once` to make the subscription known to the RP. + +```bash +./01-register-sub.sh +``` + +## Create VNET and NSG + +We provision HCPs with BYO VNET, so we need to create the VNET, Subnet and NSG upfront. + +```bash +./02-customer-infra.sh +``` + +The resources are created in a resourcegroup named `$USER-net-rg`. + +## Create cluster + +Create an HCP by sending a request to the port-forwarded RP. You can find a payload template in `cluster.tmpl.json`. +To fill that template with some user specific defaults and send it to the RP, run the following command: + +```bash +./03-create-cluster.sh +``` + +This creates an HCP named `$USER`. If you want to use a different name, run + +```bash +CLUSTER_NAME=abc ./03-create-cluster.sh +``` + +Observe the cluster creation with `./query-cluster-rp.sh` until `properties.provisioningState` is (hopefully) `Succeeded`. +`properties.spec.api.url` holds the URL to the API server of the HCP. + +See [Get the kubeconfig for an HCP](#get-the-kubeconfig-for-an-hcp) on how to get the kubeconfig for the HCP. + +## Create nodepool + +```bash +./04-create-nodepool.sh +``` + +This creates a nodepool named `np-1` for the previously created cluster. + +To check progress on + +* the RP, run `./query-nodepool-rp.sh` +* the MC, run `kubectl get nodepool -A` and `kubectl get azuremachine -A` +* within the HCP, run `kubectl get nodes` while using your HCP kubeconfig + +## Delete cluster + +```bash +./05-delete-cluster.sh +``` + +## Observe and debug + +### Check RP pod logs + +```bash +kubectl logs deployment/aro-hcp-frontend -c aro-hcp-frontend -n aro-hcp -f +kubectl logs deployment/aro-hcp-backend -c aro-hcp-backend -n aro-hcp -f +``` + +### Check cluster state in RP + +```bash +./query-cluster-rp.sh +``` + +### Check CS pod logs + +```bash +kubectl logs deployment/clusters-service -c service -f +``` + +### Check cluster state in CS + +```bash +curl localhost:8001/api/clusters_mgmt/v1/clusters | jq +``` + +### Check Maestro logs + +```bash +kubectl logs deployment/maestro -n maestro -c service -f +``` + +### Check MC consumer registration in Maestro + +```bash +curl localhost:8002/api/maestro/v1/consumers | jq +``` + +### Check MC resource bundles in Maestro + +```bash +curl localhost:8002/api/maestro/v1/resource-bundles | jq +``` + +### Check for manifestwork on MC + +```bash +CS_CLUSTER_ID=$(curl localhost:8001/api/clusters_mgmt/v1/clusters | jq .items[0].id -r) +kubectl get manifestwork -n local-cluster | grep ^${CS_CLUSTER_ID} +``` + +### Check for `HostedCluster` CR on MC + +```bash +kubectl get hostedcluster -A +``` + +### Check for namespaces on MC + +```bash +kubectl get ns | grep "ocm.*${CS_CLUSTER_ID}" +``` + +### Two namespaces should show up + +* `ocm-xxx-${CS_CLUSTER_ID}` - contains Hypershift CRs and secrets for an HCP +* `ocm-xxx-${CS_CLUSTER_ID}-${CLUSTER_NAME}` - contains the hosted controlplane (e.g. pods, secrets, ...) + +### Get the kubeconfig for an HCP + +```bash +kubectl get secret -n ocm-aro-hcp-dev-${CS_CLUSTER_ID} ${CLUSTER_NAME}-admin-kubeconfig -o json | jq .data.kubeconfig -r | base64 -d > my.kubeconfig +``` + +### Check nodepool on RP + +```bash +./query-nodepool-rp.sh +``` diff --git a/demo/cluster.tmpl.json b/demo/cluster.tmpl.json new file mode 100644 index 000000000..07f3e415a --- /dev/null +++ b/demo/cluster.tmpl.json @@ -0,0 +1,30 @@ +{ + "properties": { + "spec": { + "version": { + "id": "openshift-v4.17.0", + "channelGroup": "stable" + }, + "dns": {}, + "network": { + "networkType": "OVNKubernetes", + "podCidr": "10.128.0.0/14", + "serviceCidr": "172.30.0.0/16", + "machineCidr": "10.0.0.0/16", + "hostPrefix": 23 + }, + "console": {}, + "api": { + "visibility": "public" + }, + "proxy": {}, + "platform": { + "managedResourceGroup": "$managed-resource-group", + "subnetId": "/subscriptions/$sub/resourceGroups/$customer-rg/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1", + "outboundType": "loadBalancer", + "networkSecurityGroupId": "/subscriptions/$sub/resourceGroups/$customer-rg/providers/Microsoft.Network/networkSecurityGroups/customer-nsg" + }, + "externalAuth": {} + } + } +} diff --git a/demo/env_vars b/demo/env_vars new file mode 100644 index 000000000..ccfc2092d --- /dev/null +++ b/demo/env_vars @@ -0,0 +1,16 @@ +#!/bin/bash + + +export CUSTOMER_RG_NAME="$USER-net-rg" +export CUSTOMER_VNET_NAME="customer-vnet" +export CUSTOMER_VNET_SUBNET1="customer-subnet-1" +export CUSTOMER_NSG="customer-nsg" +export LOCATION="westus3" +if [ -z "$CLUSTER_NAME" ]; then + export CLUSTER_NAME="$USER" +fi +MANAGED_RESOURCE_GROUP="$CLUSTER_NAME-rg" + +if [ -z "$NP_NAME" ]; then + export NP_NAME="np-1" +fi diff --git a/demo/node_pool.tmpl.json b/demo/node_pool.tmpl.json new file mode 100644 index 000000000..def911ab7 --- /dev/null +++ b/demo/node_pool.tmpl.json @@ -0,0 +1,17 @@ +{ + "properties": { + "spec": { + "version": { + "id": "openshift-v4.17.0", + "channelGroup": "stable" + }, + "platform": { + "subnetId": "/subscriptions/$sub/resourceGroups/$customer-rg/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1", + "vmSize": "Standard_D8s_v3", + "diskSizeGiB": 30, + "diskStorageAccountType": "StandardSSD_LRS" + }, + "replicas": 2 + } + } +} diff --git a/demo/query-cluster-rp.sh b/demo/query-cluster-rp.sh new file mode 100755 index 000000000..44b9bfaf2 --- /dev/null +++ b/demo/query-cluster-rp.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source env_vars + +SUBSCRIPTION_ID=$(az account show --query id -o tsv) + +curl "localhost:8443/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${CUSTOMER_RG_NAME}/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/${CLUSTER_NAME}?api-version=2024-06-10-preview" | jq diff --git a/demo/query-nodepool-rp.sh b/demo/query-nodepool-rp.sh new file mode 100755 index 000000000..4893922a3 --- /dev/null +++ b/demo/query-nodepool-rp.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source env_vars + +SUBSCRIPTION_ID=$(az account show --query id -o tsv) + +curl "localhost:8443/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${CUSTOMER_RG_NAME}/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/${CLUSTER_NAME}/nodePools/${NP_NAME}?api-version=2024-06-10-preview" | jq