diff --git a/ciao-scheduler/scheduler.go b/ciao-scheduler/scheduler.go index 4d6a96933..4282ce7ef 100644 --- a/ciao-scheduler/scheduler.go +++ b/ciao-scheduler/scheduler.go @@ -746,7 +746,6 @@ func (sched *ssntpSchedulerServer) CommandForward(controllerUUID string, command fallthrough case ssntp.ReleasePublicIP: dest = sched.fwdCmdToCNCI(command, payload) - default: dest.SetDecision(ssntp.Discard) } diff --git a/testutil/singlevm/verify.sh b/testutil/singlevm/verify.sh index 492ebc56b..49539606e 100755 --- a/testutil/singlevm/verify.sh +++ b/testutil/singlevm/verify.sh @@ -2,75 +2,188 @@ ciao_bin="$HOME/local" ciao_gobin="$GOPATH"/bin +event_counter=0 -# Read cluster env variables +#Utility functions +function exitOnError { + local exit_code="$1" + local error_string="$2" + + if [ $1 -ne 0 ] + then + echo "FATAL ERROR exiting: " "$error_string" "$exit_code" + exit 1 + fi +} + +#Checks that no network artifacts are left behind +function checkForNetworkArtifacts() { + + #Verify that there are no ciao related artifacts left behind + ciao_networks=`sudo docker network ls --filter driver=ciao -q | wc -l` + + if [ $ciao_networks -ne 0 ] + then + echo "FATAL ERROR: ciao docker networks not cleaned up" + sudo docker network ls --filter driver=ciao + exit 1 + fi + + + #The only ciao interfaces left behind should be CNCI VNICs + #Once we can delete tenants we should not even have them around + cnci_vnics=`ip -d link | grep alias | grep cnci | wc -l` + ciao_vnics=`ip -d link | grep alias | wc -l` + + if [ $cnci_vnics -ne $ciao_vnics ] + then + echo "FATAL ERROR: ciao network interfaces not cleaned up" + ip -d link | grep alias + exit 1 + fi +} + +#There are too many failsafes in the CNCI. Hence just disable iptables utility to trigger failure +#This also ensures that the CNCI is always left in a consistent state (sans the permission) +function triggerIPTablesFailure { + ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" <<-EOF + sudo chmod -x /usr/bin/iptables + EOF +} + +#Restore the iptables so that the cluster is usable +function restoreIPTables { + ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" <<-EOF + sudo chmod +x /usr/bin/iptables + EOF +} + +function clearAllEvents() { + #Clear out all prior events + "$ciao_gobin"/ciao-cli event delete + + #Wait for the event count to drop to 0 + retry=0 + ciao_events=0 + + until [ $retry -ge 6 ] + do + ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'` + + if [ $ciao_events -eq 0 ] + then + break + fi + let retry=retry+1 + sleep 1 + done + + exitOnError $ciao_events "ciao events not deleted properly" +} + +function checkEventStatus { + local event_index="$1" + local event_code="$2" + local retry=0 + local ciao_events=0 + local total_events=0 + local code="" + + #We only need to wait for as many events as the index + total_events=$((event_index + 1)) + + until [ $retry -ge 6 ] + do + ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'` + + if [ $ciao_events -eq $total_events ] + then + break + fi + let retry=retry+1 + sleep 1 + done + + if [ $ciao_events -ne $total_events ] + then + echo "FATAL ERROR: ciao event not reported. Events seen =" $ciao_events + "$ciao_gobin"/ciao-cli event list + exit 1 + fi + + code=$("$ciao_gobin"/ciao-cli event list -f "{{(index . ${event_index}).Message}}" | cut -d ' ' -f 1) + + if [ "$event_code" != "$code" ] + then + echo "FATAL ERROR: Unknown event $code. Looking for $event_code" + "$ciao_gobin"/ciao-cli event list + exit 1 + fi + + "$ciao_gobin"/ciao-cli event list +} + +function createExternalIPPool() { + # first create a new external IP pool and add a subnet to it. + # this is an admin only operation, so make sure our env variables + # are set accordingly. Since user admin might belong to more than one + # tenant, make sure to specify that we are logging in as part of the + # "admin" tenant/project. + ciao_user=$CIAO_USERNAME + ciao_passwd=$CIAO_PASSWORD + export CIAO_USERNAME=$CIAO_ADMIN_USERNAME + export CIAO_PASSWORD=$CIAO_ADMIN_PASSWORD + "$ciao_gobin"/ciao-cli -tenant-name admin pool create -name test + "$ciao_gobin"/ciao-cli -tenant-name admin pool add -subnet 203.0.113.0/24 -name test + export CIAO_USERNAME=$ciao_user + export CIAO_PASSWORD=$ciao_passwd +} + +function deleteExternalIPPool() { + #Cleanup the pool + export CIAO_USERNAME=$CIAO_ADMIN_USERNAME + export CIAO_PASSWORD=$CIAO_ADMIN_PASSWORD + + "$ciao_gobin"/ciao-cli -tenant-name admin pool delete -name test + exitOnError $? "Unable to delete pool" + + export CIAO_USERNAME=$ciao_user + export CIAO_PASSWORD=$ciao_passwd +} + +# Read cluster env variables . $ciao_bin/demo.sh "$ciao_gobin"/ciao-cli workload list - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to list workloads" - exit 1 -fi +exitOnError $? "Unable to list workloads" "$ciao_gobin"/ciao-cli instance add --workload=e35ed972-c46c-4aad-a1e7-ef103ae079a2 --instances=2 - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to launch VMs" - exit 1 -fi +exitOnError $? "Unable to launch VMs" "$ciao_gobin"/ciao-cli instance list - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to list instances" - exit 1 -fi +exitOnError $? "Unable to list instances" #Launch containers - #Pre-cache the image to reduce the start latency sudo docker pull debian "$ciao_gobin"/ciao-cli instance add --workload=ca957444-fa46-11e5-94f9-38607786d9ec --instances=1 - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to launch containers" - exit 1 -fi +exitOnError $? "Unable to launch containers" sleep 5 "$ciao_gobin"/ciao-cli instance list -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to list instances" - exit 1 -fi +exitOnError $? "Unable to list instances" container_1=`sudo docker ps -q -l` container_1_ip=`sudo docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_1` "$ciao_gobin"/ciao-cli instance add --workload=ca957444-fa46-11e5-94f9-38607786d9ec --instances=1 - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to launch containers" - exit 1 -fi - +exitOnError $? "Unable to launch containers" sleep 5 "$ciao_gobin"/ciao-cli instance list -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to list instances" - exit 1 -fi +exitOnError $? "Unable to list instances" container_2=`sudo docker ps -q -l` @@ -118,93 +231,22 @@ fi echo "Checking Docker Networking" sudo docker exec $container_2 /bin/ping -c 3 $container_1_ip -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to ping across containers" - exit 1 -else - echo "Container connectivity verified" -fi +exitOnError $? "Unable to ping across containers" +echo "Container connectivity verified" #Clear out all prior events -"$ciao_gobin"/ciao-cli event delete - -#Wait for the event count to drop to 0 -retry=0 -ciao_events=0 - -until [ $retry -ge 6 ] -do - ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'` - - if [ $ciao_events -eq 0 ] - then - break - fi - - let retry=retry+1 - sleep 1 -done - -if [ $ciao_events -ne 0 ] -then - echo "FATAL ERROR: ciao events not deleted properly" - "$ciao_gobin"/ciao-cli event list - exit 1 -fi +clearAllEvents #Test External IP Assignment support #Pick the first instance which is a VM, as we can even SSH into it #We have already checked that the VM is up. -# first create a new external IP pool and add a subnet to it. -# this is an admin only operation, so make sure our env variables -# are set accordingly. Since user admin might belong to more than one -# tenant, make sure to specify that we are logging in as part of the -# "admin" tenant/project. -ciao_user=$CIAO_USERNAME -ciao_passwd=$CIAO_PASSWORD -export CIAO_USERNAME=$CIAO_ADMIN_USERNAME -export CIAO_PASSWORD=$CIAO_ADMIN_PASSWORD -"$ciao_gobin"/ciao-cli -tenant-name admin pool create -name test -"$ciao_gobin"/ciao-cli -tenant-name admin pool add -subnet 203.0.113.0/24 -name test -export CIAO_USERNAME=$ciao_user -export CIAO_PASSWORD=$ciao_passwd +createExternalIPPool testinstance=`"$ciao_gobin"/ciao-cli instance list -f '{{with index . 0}}{{.ID}}{{end}}'` - "$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test #Wait for the CNCI to report successful map -retry=0 -ciao_events=0 - -until [ $retry -ge 6 ] -do - ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'` - - if [ $ciao_events -eq 1 ] - then - break - fi - - let retry=retry+1 - sleep 1 -done - -if [ $ciao_events -ne 1 ] -then - echo "FATAL ERROR: ciao external IP event not reported" - "$ciao_gobin"/ciao-cli event list - exit 1 -fi - -mapped=`ciao-cli event list -f '{{if eq (len .) 1}}{{(index . 0).Message}}{{end}}' | cut -d " " -f 1` -if [ $mapped != "Mapped" ] -then - echo "FATAL ERROR: Unknown event" - "$ciao_gobin"/ciao-cli event list - exit 1 -fi +checkEventStatus $event_counter "Mapped" "$ciao_gobin"/ciao-cli event list "$ciao_gobin"/ciao-cli external-ip list @@ -220,13 +262,7 @@ ping_result=$? test_hostname=`ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$testip" hostname` sudo ip route del 203.0.113.0/24 dev ciaovlan -if [ $ping_result -ne 0 ] -then - echo "FATAL ERROR: Unable to ping external IP" - exit 1 -else - echo "Container external connectivity verified" -fi +exitOnError $ping_result "Unable to ping external IP" if [ "$test_hostname" == "$test_instance" ] then @@ -239,170 +275,49 @@ fi "$ciao_gobin"/ciao-cli external-ip unmap -address $testip #Wait for the CNCI to report successful unmap -retry=0 -ciao_events=0 - -until [ $retry -ge 6 ] -do - ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'` - - if [ $ciao_events -eq 2 ] - then - break - fi - - let retry=retry+1 - sleep 1 -done - -if [ $ciao_events -ne 2 ] -then - echo "FATAL ERROR: ciao external IP event not reported" - "$ciao_gobin"/ciao-cli event list - exit 1 -fi +event_counter=$((event_counter+1)) +checkEventStatus $event_counter "Unmapped" -unmapped=`"$ciao_gobin"/ciao-cli event list -f '{{if eq (len .) 2}}{{(index . 1).Message}}{{end}}' | cut -d " " -f 1` -if [ $unmapped != "Unmapped" ] -then - echo "FATAL ERROR: Unknown event" - "$ciao_gobin"/ciao-cli event list - exit 1 -fi - -"$ciao_gobin"/ciao-cli event list "$ciao_gobin"/ciao-cli external-ip list -#Negative test case for external IP -#Clear out a ciao chains to trigger a failure -ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" << EOF -sudo iptables-save > /tmp/rules.save -sudo iptables -t nat -D PREROUTING -j ciao-floating-ip-pre -sudo iptables -t nat -X ciao-floating-ip-pre -EOF +#Test for External IP Failures +#Map failure +triggerIPTablesFailure "$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test - #Wait for the CNCI to report unsuccessful map -retry=0 -ciao_events=0 - -until [ $retry -ge 6 ] -do - ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'` +event_counter=$((event_counter+1)) +checkEventStatus $event_counter "Failed" +restoreIPTables - if [ $ciao_events -eq 3 ] - then - break - fi - - let retry=retry+1 - sleep 1 -done - -if [ $ciao_events -ne 3 ] -then - echo "FATAL ERROR: ciao external IP error not reported" - "$ciao_gobin"/ciao-cli event list - exit 1 -fi - -mapped=`ciao-cli event list -f '{{if eq (len .) 3}}{{(index . 2).Message}}{{end}}' | cut -d " " -f 1,2,3` -if [ "$mapped" != "Failed to map" ] -then - echo "FATAL ERROR: Unknown event" - "$ciao_gobin"/ciao-cli event list - exit 1 -fi +#Unmap failure +"$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test +event_counter=$((event_counter+1)) +checkEventStatus $event_counter "Mapped" -#Verify that we see the error -"$ciao_gobin"/ciao-cli event list +triggerIPTablesFailure +"$ciao_gobin"/ciao-cli external-ip unmap -address $testip +event_counter=$((event_counter+1)) +checkEventStatus $event_counter "Failed" +restoreIPTables -#Restore the iptables so that the cluster is usable -ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" << EOF -sudo iptables-restore /tmp/rules.save -EOF - -#Cleanup the pool -export CIAO_USERNAME=$CIAO_ADMIN_USERNAME -export CIAO_PASSWORD=$CIAO_ADMIN_PASSWORD -"$ciao_gobin"/ciao-cli -tenant-name admin pool delete -name test -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to delete pool" - exit 1 -fi -export CIAO_USERNAME=$ciao_user -export CIAO_PASSWORD=$ciao_passwd +#Cleanup +"$ciao_gobin"/ciao-cli external-ip unmap -address $testip +event_counter=$((event_counter+1)) +checkEventStatus $event_counter "Unmapped" +#Cleanup pools +deleteExternalIPPool #Now delete all instances "$ciao_gobin"/ciao-cli instance delete --all - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to delete instances" - exit 1 -fi +exitOnError $? "Unable to delete instances" "$ciao_gobin"/ciao-cli instance list #Wait for all the instance deletions to be reported back -retry=0 -ciao_events=0 - -until [ $retry -ge 6 ] -do - ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'` - - if [ $ciao_events -eq 6 ] - then - break - fi +event_counter=$((event_counter+4)) +checkEventStatus $event_counter "Deleted" - let retry=retry+1 - sleep 1 -done - -if [ $ciao_events -ne 6 ] -then - echo "FATAL ERROR: ciao instances not deleted properly" - "$ciao_gobin"/ciao-cli event list - exit 1 -fi - -#Wait around a bit as instance delete is asynchronous -retry=0 -ciao_networks=0 -until [ $retry -ge 6 ] -do - #Verify that there are no ciao related artifacts left behind - ciao_networks=`sudo docker network ls --filter driver=ciao -q | wc -l` - - if [ $ciao_networks -eq 0 ] - then - break - fi - let retry=retry+1 - sleep 1 -done - -if [ $ciao_networks -ne 0 ] -then - echo "FATAL ERROR: ciao docker networks not cleaned up" - sudo docker network ls --filter driver=ciao - exit 1 -fi - - -#The only ciao interfaces left behind should be CNCI VNICs -#Once we can delete tenants we should not even have them around -cnci_vnics=`ip -d link | grep alias | grep cnci | wc -l` -ciao_vnics=`ip -d link | grep alias | wc -l` - -if [ $cnci_vnics -ne $ciao_vnics ] -then - echo "FATAL ERROR: ciao network interfaces not cleaned up" - ip -d link | grep alias - exit 1 -fi +#Verify that there are no ciao related artifacts left behind +checkForNetworkArtifacts