ciao-launcher: Send InstanceDeleted after instance deletion

Launcher was incorrectly sending the InstanceDeleted event before the
instance and its associated network assets had been deleted.  This
was causing a race condition in which controller was re-using some
of the resources assigned to the instance before it was actually deleted for
new instances.  When this happened the new instances typically failed to
start.  This commit delays the sending of the InstanceDelete event until
after the instance has been deleted.

Fixes #808

Signed-off-by: Mark Ryan <[email protected]>

ciao-launcher/instance.go

@@ -21,6 +21,8 @@ import (
 	"sync"
 	"time"
 
+	yaml "gopkg.in/yaml.v2"
+
 	storage "github.com/01org/ciao/ciao-storage"
 	"github.com/01org/ciao/payloads"
 	"github.com/01org/ciao/ssntp"
@@ -192,6 +194,24 @@ func (id *instanceData) stopCommand(cmd *insStopCmd) {
 	id.monitorCh <- virtualizerStopCmd{}
 }
 
+func (id *instanceData) sendInstanceDeletedEvent() {
+	var event payloads.EventInstanceDeleted
+
+	event.InstanceDeleted.InstanceUUID = id.instance
+
+	payload, err := yaml.Marshal(&event)
+	if err != nil {
+		glog.Errorf("Unable to Marshall STATS %v", err)
+		return
+	}
+
+	_, err = id.ac.conn.SendEvent(ssntp.InstanceDeleted, payload)
+	if err != nil {
+		glog.Errorf("Failed to send event command %v", err)
+		return
+	}
+}
+
 func (id *instanceData) deleteCommand(cmd *insDeleteCmd) bool {
 	if id.shuttingDown && !cmd.suicide {
 		deleteErr := &deleteError{nil, payloads.DeleteNoInstance}
@@ -211,6 +231,7 @@ func (id *instanceData) deleteCommand(cmd *insDeleteCmd) bool {
 	id.unmapVolumes()
 
 	if !cmd.suicide {
+		id.sendInstanceDeletedEvent()
 		id.ovsCh <- &ovsStatusCmd{}
 	}
 	return true

ciao-launcher/main.go

@@ -362,7 +362,6 @@ func processCommand(conn serverConn, cmd *cmdWrapper, ovsCh chan<- interface{})
 		errCh := make(chan error)
 		ovsCh <- &ovsRemoveCmd{
 			cmd.instance,
-			delCmd.suicide,
 			errCh}
 		<-errCh
 	}

ciao-launcher/overseer.go

@@ -58,7 +58,6 @@ type ovsGetCmd struct {
 
 type ovsRemoveCmd struct {
 	instance string
-	suicide  bool
 	errCh    chan<- error
 }
 
@@ -494,24 +493,6 @@ func getStats(instancesDir string) *cnStats {
 	return &s
 }
 
-func (ovs *overseer) sendInstanceDeletedEvent(instance string) {
-	var event payloads.EventInstanceDeleted
-
-	event.InstanceDeleted.InstanceUUID = instance
-
-	payload, err := yaml.Marshal(&event)
-	if err != nil {
-		glog.Errorf("Unable to Marshall STATS %v", err)
-		return
-	}
-
-	_, err = ovs.ac.conn.SendEvent(ssntp.InstanceDeleted, payload)
-	if err != nil {
-		glog.Errorf("Failed to send event command %v", err)
-		return
-	}
-}
-
 func (ovs *overseer) processGetCommand(cmd *ovsGetCmd) {
 	glog.Infof("Overseer: looking for instance %s", cmd.instance)
 	var insState ovsGetResult
@@ -579,9 +560,6 @@ func (ovs *overseer) processRemoveCommand(cmd *ovsRemoveCmd) {
 	}
 
 	delete(ovs.instances, cmd.instance)
-	if !cmd.suicide {
-		ovs.sendInstanceDeletedEvent(cmd.instance)
-	}
 	cmd.errCh <- nil
 }