From 6b01ccbca74aa11edeb009fcce937a39ef173063 Mon Sep 17 00:00:00 2001
From: Manohar Castelino <manohar.r.castelino@intel.com>
Date: Tue, 8 Nov 2016 15:52:51 -0800
Subject: [PATCH 1/3] ciao-scheduler: Enable command forwarding to CNCI's
 Commands such as AssignPublicIP (and the eventual peer removal command) need
 passed by the scheduler to the correct CNCI, based on the CNCIAgent uuid in
 the command payload.

Signed-off-by: Tim Pepper <timothy.c.pepper@linux.intel.com>
Signed-off-by: Manohar Castelino <manohar.r.castelino@intel.com>
---
 ciao-scheduler/scheduler.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ciao-scheduler/scheduler.go b/ciao-scheduler/scheduler.go
index 4d6a96933..4282ce7ef 100644
--- a/ciao-scheduler/scheduler.go
+++ b/ciao-scheduler/scheduler.go
@@ -746,7 +746,6 @@ func (sched *ssntpSchedulerServer) CommandForward(controllerUUID string, command
 		fallthrough
 	case ssntp.ReleasePublicIP:
 		dest = sched.fwdCmdToCNCI(command, payload)
-
 	default:
 		dest.SetDecision(ssntp.Discard)
 	}

From 201315742e3516c40b205fa48ba368d810abf053 Mon Sep 17 00:00:00 2001
From: Manohar Castelino <manohar.r.castelino@intel.com>
Date: Tue, 29 Nov 2016 14:36:08 -0800
Subject: [PATCH 2/3] Single VM: Cleanup verify.sh

Make verify.sh modular. This will allow the addition of more complex
test cases.

Signed-off-by: Manohar Castelino <manohar.r.castelino@intel.com>
---
 testutil/singlevm/verify.sh | 453 ++++++++++++++----------------------
 1 file changed, 175 insertions(+), 278 deletions(-)

diff --git a/testutil/singlevm/verify.sh b/testutil/singlevm/verify.sh
index 492ebc56b..ae5e90b24 100755
--- a/testutil/singlevm/verify.sh
+++ b/testutil/singlevm/verify.sh
@@ -3,74 +3,187 @@
 ciao_bin="$HOME/local"
 ciao_gobin="$GOPATH"/bin
 
-# Read cluster env variables
+#Utility functions
+function exitOnError {
+	local exit_code="$1"
+	local error_string="$2"
+
+	if [ $1 -ne 0 ]
+	then
+		echo "FATAL ERROR exiting: " "$error_string" "$exit_code"
+		exit 1
+	fi
+}
+
+#Checks that no network artifacts are left behind
+function checkForNetworkArtifacts() {
+
+	#Verify that there are no ciao related artifacts left behind
+	ciao_networks=`sudo docker network ls --filter driver=ciao -q | wc -l`
+
+	if [ $ciao_networks -ne 0 ]
+	then
+		echo "FATAL ERROR: ciao docker networks not cleaned up"
+		sudo docker network ls --filter driver=ciao
+		exit 1
+	fi
+
+
+	#The only ciao interfaces left behind should be CNCI VNICs
+	#Once we can delete tenants we should not even have them around
+	cnci_vnics=`ip -d link | grep alias | grep cnci | wc -l`
+	ciao_vnics=`ip -d link | grep alias | wc -l`
+
+	if [ $cnci_vnics -ne $ciao_vnics ]
+	then
+		echo "FATAL ERROR: ciao network interfaces not cleaned up"
+		ip -d link | grep alias
+		exit 1
+	fi
+}
+
+#Clear out a ciao chains to trigger a failure
+function triggerIPTablesFailure {
+	ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" <<-EOF
+	sudo iptables-save > /tmp/rules.save
+	sudo iptables -t nat -D PREROUTING -j ciao-floating-ip-pre
+	sudo iptables -t nat -X ciao-floating-ip-pre
+	EOF
+}
+
+#Restore the iptables so that the cluster is usable
+function restoreIPTables {
+	ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" <<-EOF
+	sudo iptables-restore  /tmp/rules.save
+	EOF
+}
+
+function clearAllEvents() {
+	#Clear out all prior events
+	"$ciao_gobin"/ciao-cli event delete
+
+	#Wait for the event count to drop to 0
+	retry=0
+	ciao_events=0
+
+	until [ $retry -ge 6 ]
+	do
+		ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'`
+
+		if [ $ciao_events -eq 0 ]
+		then
+			break
+		fi
+		let retry=retry+1
+		sleep 1
+	done
+
+	exitOnError $ciao_events "ciao events not deleted properly"
+}
+
+function checkEventStatus {
+	local event_index="$1"
+	local event_code="$2"
+	local retry=0
+	local ciao_events=0
+	local total_events=0
+	local code=""
+
+	#We only need to wait for as many events as the index
+	total_events=$((event_index + 1))
+
+	until [ $retry -ge 6 ]
+	do
+		ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'`
 
+		if [ $ciao_events -eq $total_events ]
+		then
+			break
+		fi
+
+		let retry=retry+1
+		sleep 1
+	done
+
+	if [ $ciao_events -ne $total_events ]
+	then
+		echo "FATAL ERROR: ciao event not reported. Events seen =" $ciao_events
+		"$ciao_gobin"/ciao-cli event list
+		exit 1
+	fi
+
+	code=$("$ciao_gobin"/ciao-cli event list -f "{{(index . ${event_index}).Message}}" | cut -d ' ' -f 1)
+
+	if [ "$event_code" != "$code" ]
+	then
+		echo "FATAL ERROR: Unknown event $code. Looking for $event_code"
+		"$ciao_gobin"/ciao-cli event list
+		exit 1
+	fi
+
+	"$ciao_gobin"/ciao-cli event list
+}
+
+function createExternalIPPool() {
+	# first create a new external IP pool and add a subnet to it.
+	# this is an admin only operation, so make sure our env variables
+	# are set accordingly. Since user admin might belong to more than one
+	# tenant, make sure to specify that we are logging in as part of the
+	# "admin" tenant/project.
+	ciao_user=$CIAO_USERNAME
+	ciao_passwd=$CIAO_PASSWORD
+	export CIAO_USERNAME=$CIAO_ADMIN_USERNAME
+	export CIAO_PASSWORD=$CIAO_ADMIN_PASSWORD
+	"$ciao_gobin"/ciao-cli -tenant-name admin pool create -name test
+	"$ciao_gobin"/ciao-cli -tenant-name admin pool add -subnet 203.0.113.0/24 -name test
+	export CIAO_USERNAME=$ciao_user
+	export CIAO_PASSWORD=$ciao_passwd
+}
+
+function deleteExternalIPPool() {
+	#Cleanup the pool
+	export CIAO_USERNAME=$CIAO_ADMIN_USERNAME
+	export CIAO_PASSWORD=$CIAO_ADMIN_PASSWORD
+
+	"$ciao_gobin"/ciao-cli -tenant-name admin pool delete -name test
+	exitOnError $?  "Unable to delete pool"
+
+	export CIAO_USERNAME=$ciao_user
+	export CIAO_PASSWORD=$ciao_passwd
+}
+
+# Read cluster env variables
 . $ciao_bin/demo.sh
 
 "$ciao_gobin"/ciao-cli workload list
-
-if [ $? -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to list workloads"
-	exit 1
-fi
+exitOnError $?  "Unable to list workloads"
 
 "$ciao_gobin"/ciao-cli instance add --workload=e35ed972-c46c-4aad-a1e7-ef103ae079a2 --instances=2
-
-if [ $? -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to launch VMs"
-	exit 1
-fi
+exitOnError $?  "Unable to launch VMs"
 
 "$ciao_gobin"/ciao-cli instance list
-
-if [ $? -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to list instances"
-	exit 1
-fi
+exitOnError $? "Unable to list instances"
 
 #Launch containers
-
 #Pre-cache the image to reduce the start latency
 sudo docker pull debian
 "$ciao_gobin"/ciao-cli instance add --workload=ca957444-fa46-11e5-94f9-38607786d9ec --instances=1
-
-if [ $? -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to launch containers"
-	exit 1
-fi
+exitOnError $? "Unable to launch containers"
 
 sleep 5
 
 "$ciao_gobin"/ciao-cli instance list
-if [ $? -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to list instances"
-	exit 1
-fi
+exitOnError $? "Unable to list instances"
 
 container_1=`sudo docker ps -q -l`
 container_1_ip=`sudo docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_1`
 
 "$ciao_gobin"/ciao-cli instance add --workload=ca957444-fa46-11e5-94f9-38607786d9ec --instances=1
-
-if [ $? -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to launch containers"
-	exit 1
-fi
-
+exitOnError $?  "Unable to launch containers"
 sleep 5
 
 "$ciao_gobin"/ciao-cli instance list
-if [ $? -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to list instances"
-	exit 1
-fi
+exitOnError $?  "Unable to list instances"
 
 container_2=`sudo docker ps -q -l`
 
@@ -118,93 +231,22 @@ fi
 echo "Checking Docker Networking"
 sudo docker exec $container_2 /bin/ping -c 3 $container_1_ip
 
-if [ $? -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to ping across containers"
-	exit 1
-else
-	echo "Container connectivity verified"
-fi
+exitOnError $?  "Unable to ping across containers"
+echo "Container connectivity verified"
 
 #Clear out all prior events
-"$ciao_gobin"/ciao-cli event delete
-
-#Wait for the event count to drop to 0
-retry=0
-ciao_events=0
-
-until [ $retry -ge 6 ]
-do
-	ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'`
-
-	if [ $ciao_events -eq 0 ]
-	then
-		break
-	fi
-
-	let retry=retry+1
-	sleep 1
-done
-
-if [ $ciao_events -ne 0 ]
-then
-	echo "FATAL ERROR: ciao events not deleted properly"
-	"$ciao_gobin"/ciao-cli event list
-	exit 1
-fi
+clearAllEvents
 
 #Test External IP Assignment support
 #Pick the first instance which is a VM, as we can even SSH into it
 #We have already checked that the VM is up.
-# first create a new external IP pool and add a subnet to it.
-# this is an admin only operation, so make sure our env variables
-# are set accordingly. Since user admin might belong to more than one
-# tenant, make sure to specify that we are logging in as part of the
-# "admin" tenant/project.
-ciao_user=$CIAO_USERNAME
-ciao_passwd=$CIAO_PASSWORD
-export CIAO_USERNAME=$CIAO_ADMIN_USERNAME
-export CIAO_PASSWORD=$CIAO_ADMIN_PASSWORD
-"$ciao_gobin"/ciao-cli -tenant-name admin pool create -name test
-"$ciao_gobin"/ciao-cli -tenant-name admin pool add -subnet 203.0.113.0/24 -name test
-export CIAO_USERNAME=$ciao_user
-export CIAO_PASSWORD=$ciao_passwd
+createExternalIPPool
 
 testinstance=`"$ciao_gobin"/ciao-cli instance list -f '{{with index . 0}}{{.ID}}{{end}}'`
-
 "$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test
 
 #Wait for the CNCI to report successful map
-retry=0
-ciao_events=0
-
-until [ $retry -ge 6 ]
-do
-	ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'`
-
-	if [ $ciao_events -eq 1 ]
-	then
-		break
-	fi
-
-	let retry=retry+1
-	sleep 1
-done
-
-if [ $ciao_events -ne 1 ]
-then
-	echo "FATAL ERROR: ciao external IP event not reported"
-	"$ciao_gobin"/ciao-cli event list
-	exit 1
-fi
-
-mapped=`ciao-cli event list -f '{{if eq (len .) 1}}{{(index . 0).Message}}{{end}}' | cut -d " " -f 1`
-if [ $mapped != "Mapped" ]
-then
-	echo "FATAL ERROR: Unknown event"
-	"$ciao_gobin"/ciao-cli event list
-	exit 1
-fi
+checkEventStatus 0 "Mapped"
 
 "$ciao_gobin"/ciao-cli event list
 "$ciao_gobin"/ciao-cli external-ip list
@@ -220,13 +262,7 @@ ping_result=$?
 test_hostname=`ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$testip" hostname`
 sudo ip route del 203.0.113.0/24 dev ciaovlan
 
-if [ $ping_result -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to ping external IP"
-	exit 1
-else
-	echo "Container external connectivity verified"
-fi
+exitOnError $ping_result "Unable to ping external IP"
 
 if [ "$test_hostname" == "$test_instance" ]
 then
@@ -239,170 +275,31 @@ fi
 "$ciao_gobin"/ciao-cli external-ip unmap -address $testip
 
 #Wait for the CNCI to report successful unmap
-retry=0
-ciao_events=0
-
-until [ $retry -ge 6 ]
-do
-	ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'`
-
-	if [ $ciao_events -eq 2 ]
-	then
-		break
-	fi
-
-	let retry=retry+1
-	sleep 1
-done
-
-if [ $ciao_events -ne 2 ]
-then
-	echo "FATAL ERROR: ciao external IP event not reported"
-	"$ciao_gobin"/ciao-cli event list
-	exit 1
-fi
+checkEventStatus 1 "Unmapped"
 
-unmapped=`"$ciao_gobin"/ciao-cli event list -f '{{if eq (len .) 2}}{{(index . 1).Message}}{{end}}' | cut -d " " -f 1`
-if [ $unmapped != "Unmapped" ]
-then
-	echo "FATAL ERROR: Unknown event"
-	"$ciao_gobin"/ciao-cli event list
-	exit 1
-fi
-
-"$ciao_gobin"/ciao-cli event list
 "$ciao_gobin"/ciao-cli external-ip list
 
-#Negative test case for external IP
-#Clear out a ciao chains to trigger a failure
-ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" << EOF
-sudo iptables-save > /tmp/rules.save
-sudo iptables -t nat -D PREROUTING -j ciao-floating-ip-pre
-sudo iptables -t nat -X ciao-floating-ip-pre
-EOF
-
+#Test for External IP Failures
+triggerIPTablesFailure 
 "$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test
-
 #Wait for the CNCI to report unsuccessful map
-retry=0
-ciao_events=0
-
-until [ $retry -ge 6 ]
-do
-	ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'`
-
-	if [ $ciao_events -eq 3 ]
-	then
-		break
-	fi
-
-	let retry=retry+1
-	sleep 1
-done
-
-if [ $ciao_events -ne 3 ]
-then
-	echo "FATAL ERROR: ciao external IP error not reported"
-	"$ciao_gobin"/ciao-cli event list
-	exit 1
-fi
+checkEventStatus 2 "Failed"
 
-mapped=`ciao-cli event list -f '{{if eq (len .) 3}}{{(index . 2).Message}}{{end}}' | cut -d " " -f 1,2,3`
-if [ "$mapped" != "Failed to map" ]
-then
-	echo "FATAL ERROR: Unknown event"
-	"$ciao_gobin"/ciao-cli event list
-	exit 1
-fi
-
-#Verify that we see the error
-"$ciao_gobin"/ciao-cli event list
-
-#Restore the iptables so that the cluster is usable
-ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" << EOF
-sudo iptables-restore  /tmp/rules.save
-EOF
-
-#Cleanup the pool
-export CIAO_USERNAME=$CIAO_ADMIN_USERNAME
-export CIAO_PASSWORD=$CIAO_ADMIN_PASSWORD
-"$ciao_gobin"/ciao-cli -tenant-name admin pool delete -name test
-if [ $? -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to delete pool"
-	exit 1
-fi
-export CIAO_USERNAME=$ciao_user
-export CIAO_PASSWORD=$ciao_passwd
+#Restore the state of the CNCI so that the cluster is usable
+#This also ensures that verify.sh can be run multiple times
+restoreIPTables
 
+#Cleanup pools
+deleteExternalIPPool
 
 #Now delete all instances
 "$ciao_gobin"/ciao-cli instance delete --all
-
-if [ $? -ne 0 ]
-then
-	echo "FATAL ERROR: Unable to delete instances"
-	exit 1
-fi
+exitOnError $?  "Unable to delete instances"
 
 "$ciao_gobin"/ciao-cli instance list
 
 #Wait for all the instance deletions to be reported back
-retry=0
-ciao_events=0
-
-until [ $retry -ge 6 ]
-do
-	ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'`
-
-	if [ $ciao_events -eq 6 ]
-	then
-		break
-	fi
-
-	let retry=retry+1
-	sleep 1
-done
+checkEventStatus 6 "Deleted"
 
-if [ $ciao_events -ne 6 ]
-then
-	echo "FATAL ERROR: ciao instances not deleted properly"
-	"$ciao_gobin"/ciao-cli event list
-	exit 1
-fi
-
-#Wait around a bit as instance delete is asynchronous
-retry=0
-ciao_networks=0
-until [ $retry -ge 6 ]
-do
-	#Verify that there are no ciao related artifacts left behind
-	ciao_networks=`sudo docker network ls --filter driver=ciao -q | wc -l`
-
-	if [ $ciao_networks -eq 0 ]
-	then
-		break
-	fi
-	let retry=retry+1
-	sleep 1
-done
-
-if [ $ciao_networks -ne 0 ]
-then
-	echo "FATAL ERROR: ciao docker networks not cleaned up"
-	sudo docker network ls --filter driver=ciao
-	exit 1
-fi
-
-
-#The only ciao interfaces left behind should be CNCI VNICs
-#Once we can delete tenants we should not even have them around
-cnci_vnics=`ip -d link | grep alias | grep cnci | wc -l`
-ciao_vnics=`ip -d link | grep alias | wc -l`
-
-if [ $cnci_vnics -ne $ciao_vnics ]
-then
-	echo "FATAL ERROR: ciao network interfaces not cleaned up"
-	ip -d link | grep alias
-	exit 1
-fi
+#Verify that there are no ciao related artifacts left behind
+checkForNetworkArtifacts

From 82eb7cb89bae1fb367fba58ad6245bdb9fa1b3f8 Mon Sep 17 00:00:00 2001
From: Manohar Castelino <manohar.r.castelino@intel.com>
Date: Tue, 29 Nov 2016 17:13:21 -0800
Subject: [PATCH 3/3] SingleVM: Add test for checking External IP Unmap failure

Add test for checking External IP Unmap failure.
Make the events handling a bit more generic by removing
all manifest constants.

Signed-off-by: Manohar Castelino <manohar.r.castelino@intel.com>
---
 testutil/singlevm/verify.sh | 42 ++++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 12 deletions(-)

diff --git a/testutil/singlevm/verify.sh b/testutil/singlevm/verify.sh
index ae5e90b24..49539606e 100755
--- a/testutil/singlevm/verify.sh
+++ b/testutil/singlevm/verify.sh
@@ -2,6 +2,7 @@
 
 ciao_bin="$HOME/local"
 ciao_gobin="$GOPATH"/bin
+event_counter=0
 
 #Utility functions
 function exitOnError {
@@ -42,19 +43,18 @@ function checkForNetworkArtifacts() {
 	fi
 }
 
-#Clear out a ciao chains to trigger a failure
+#There are too many failsafes in the CNCI. Hence just disable iptables utility to trigger failure
+#This also ensures that the CNCI is always left in a consistent state (sans the permission)
 function triggerIPTablesFailure {
 	ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" <<-EOF
-	sudo iptables-save > /tmp/rules.save
-	sudo iptables -t nat -D PREROUTING -j ciao-floating-ip-pre
-	sudo iptables -t nat -X ciao-floating-ip-pre
+	sudo chmod -x /usr/bin/iptables
 	EOF
 }
 
 #Restore the iptables so that the cluster is usable
 function restoreIPTables {
 	ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" <<-EOF
-	sudo iptables-restore  /tmp/rules.save
+	sudo chmod +x /usr/bin/iptables
 	EOF
 }
 
@@ -246,7 +246,7 @@ testinstance=`"$ciao_gobin"/ciao-cli instance list -f '{{with index . 0}}{{.ID}}
 "$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test
 
 #Wait for the CNCI to report successful map
-checkEventStatus 0 "Mapped"
+checkEventStatus $event_counter "Mapped"
 
 "$ciao_gobin"/ciao-cli event list
 "$ciao_gobin"/ciao-cli external-ip list
@@ -275,20 +275,37 @@ fi
 "$ciao_gobin"/ciao-cli external-ip unmap -address $testip
 
 #Wait for the CNCI to report successful unmap
-checkEventStatus 1 "Unmapped"
+event_counter=$((event_counter+1))
+checkEventStatus $event_counter "Unmapped"
 
 "$ciao_gobin"/ciao-cli external-ip list
 
 #Test for External IP Failures
-triggerIPTablesFailure 
+
+#Map failure
+triggerIPTablesFailure
 "$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test
 #Wait for the CNCI to report unsuccessful map
-checkEventStatus 2 "Failed"
+event_counter=$((event_counter+1))
+checkEventStatus $event_counter "Failed"
+restoreIPTables
+
+#Unmap failure
+"$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test
+event_counter=$((event_counter+1))
+checkEventStatus $event_counter "Mapped"
 
-#Restore the state of the CNCI so that the cluster is usable
-#This also ensures that verify.sh can be run multiple times
+triggerIPTablesFailure 
+"$ciao_gobin"/ciao-cli external-ip unmap -address $testip
+event_counter=$((event_counter+1))
+checkEventStatus $event_counter "Failed"
 restoreIPTables
 
+#Cleanup
+"$ciao_gobin"/ciao-cli external-ip unmap -address $testip
+event_counter=$((event_counter+1))
+checkEventStatus $event_counter "Unmapped"
+
 #Cleanup pools
 deleteExternalIPPool
 
@@ -299,7 +316,8 @@ exitOnError $?  "Unable to delete instances"
 "$ciao_gobin"/ciao-cli instance list
 
 #Wait for all the instance deletions to be reported back
-checkEventStatus 6 "Deleted"
+event_counter=$((event_counter+4))
+checkEventStatus $event_counter "Deleted"
 
 #Verify that there are no ciao related artifacts left behind
 checkForNetworkArtifacts