Skip to content

Commit

Permalink
find pod owners via stale pods
Browse files Browse the repository at this point in the history
use the list of stale pods and group by owner, then process each owner
and wait for finished rollout

Signed-off-by: Gerd Oberlechner <[email protected]>
  • Loading branch information
geoberle authored and Jonathan Chang committed Dec 19, 2024
1 parent 69163d5 commit d532024
Showing 1 changed file with 22 additions and 24 deletions.
46 changes: 22 additions & 24 deletions dev-infrastructure/scripts/istio.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,31 +74,29 @@ istioctl tag set "$TAG" --revision "${NEWVERSION}" --istioNamespace ${ISTIO_NAME
for namespace in $(kubectl get namespaces --selector=istio.io/rev="$TAG" -o jsonpath='{.items[*].metadata.name}'); do
echo "in namespace $namespace"
pods="$(kubectl get pods --namespace "${namespace}" -o json)"
for pod in $(jq <<<"${pods}" --raw-output --arg NEWVERSION "${NEWVERSION}" '.items[] | select(.metadata.annotations["sidecar.istio.io/status"] | fromjson.revision != $NEWVERSION) | .metadata.name'); do
owner_kind=$(jq <<<"${pods}" --raw-output --arg NAME "${pod}" '.items[] | select(.metadata.name == $NAME) | .metadata.ownerReferences[0].kind')
owner_name=$(jq <<<"${pods}" --raw-output --arg NAME "${pod}" '.items[] | select(.metadata.name == $NAME) | .metadata.ownerReferences[0].name')
case "$owner_kind" in
"ReplicaSet")
deployment=$(kubectl get replicaset "$owner_name" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[0].name}')
if [[ -n "$deployment" ]]; then
echo "in ReplicaSet restart deployment $deployment"
kubectl rollout restart deployment "$deployment" -n "$namespace"
continue 2
else
echo "in ReplicaSet delete pod $pod"
kubectl delete pod "$pod" -n "$namespace"
fi
;;
"StatefulSet")
echo "in StatefulSet restart deployment $deployment"
deployment=$(kubectl get replicaset "$owner_name" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[0].name}')
for owner in $(kubectl get pods --namespace "${namespace}" -o json | jq -r --arg NEWVERSION ${NEWVERSION} '.items[] | select(.metadata.annotations["sidecar.istio.io/status"] | fromjson.revision != $NEWVERSION) | "\(.metadata.ownerReferences[0].kind)/\(.metadata.ownerReferences[0].name)"' | sort | uniq); do
echo "process pod owner ${owner}"
case "$owner" in
"ReplicaSet"*)
deployment=$(kubectl get ${owner} -n "$namespace" -o jsonpath='{.metadata.ownerReferences[0].name}')
if [[ -n "$deployment" ]]; then
echo "in ReplicaSet restart deployment $deployment"
kubectl rollout restart deployment "$deployment" -n "$namespace"
continue 2
;;
*)
# Don't do anything for (Cron)Job, or no owner pod for now.
;;
esac
kubectl rollout status deploy/${deployment}
else
echo "in ReplicaSet delete pod $pod"
kubectl delete pod "$pod" -n "$namespace"
fi
;;
"StatefulSet"*)
echo "restart statefulset $owner"
kubectl rollout restart ${owner} -n "$namespace"
kubectl rollout status ${owner} -n "$namespace"
;;
*)
# Don't do anything for (Cron)Job, or no owner pod for now.
;;
esac
# etc
done
done
Expand Down

0 comments on commit d532024

Please sign in to comment.