From 42b27c65b3d3b9f92ad158246bc70a9db6bcc370 Mon Sep 17 00:00:00 2001 From: Tim Pepper Date: Mon, 20 Jun 2016 21:04:22 -0700 Subject: [PATCH 01/10] ciao-scheduler: forward delete failures to controller All controllers should get delete failure error frames forwarded by ssntp. This was set in the testutil forwards config, but not in scheduler for unknown reasons. Logically that forwarding must happen for correct cluster functionality. Signed-off-by: Tim Pepper --- ciao-scheduler/scheduler.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ciao-scheduler/scheduler.go b/ciao-scheduler/scheduler.go index e0ca3144a..de8637f8d 100644 --- a/ciao-scheduler/scheduler.go +++ b/ciao-scheduler/scheduler.go @@ -954,6 +954,10 @@ func setSSNTPForwardRules(sched *ssntpSchedulerServer) { Operand: ssntp.RestartFailure, Dest: ssntp.Controller, }, + { // all DeleteFailure events go to all Controllers + Operand: ssntp.DeleteFailure, + Dest: ssntp.Controller, + }, { // all START command are processed by the Command forwarder Operand: ssntp.START, CommandForward: sched, From dfb42cbf0ddc7bc8bd05a58eea930363c41af7ae Mon Sep 17 00:00:00 2001 From: Tim Pepper Date: Mon, 20 Jun 2016 21:13:48 -0700 Subject: [PATCH 02/10] testutil: clarify channel comments There were some copy paste propagated inconsistencies in the test event/error/command channel helpers. Set them all to be clear about which ssntp entity they hold. Signed-off-by: Tim Pepper --- testutil/agent.go | 18 +++++++++--------- testutil/controller.go | 18 +++++++++--------- testutil/server.go | 16 ++++++++-------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/testutil/agent.go b/testutil/agent.go index fcd8f86da..fcdfca280 100644 --- a/testutil/agent.go +++ b/testutil/agent.go @@ -94,7 +94,7 @@ func NewSsntpTestClientConnection(name string, role ssntp.Role, uuid string) (*S return client, nil } -// AddCmdChan adds a command to the SsntpTestClient command channel +// AddCmdChan adds an ssntp.Command to the SsntpTestClient command channel func (client *SsntpTestClient) AddCmdChan(cmd ssntp.Command) *chan Result { c := make(chan Result) @@ -105,7 +105,7 @@ func (client *SsntpTestClient) AddCmdChan(cmd ssntp.Command) *chan Result { return &c } -// GetCmdChanResult gets a CmdResult from the SsntpTestClient command channel +// GetCmdChanResult gets a Result from the SsntpTestClient command channel func (client *SsntpTestClient) GetCmdChanResult(c *chan Result, cmd ssntp.Command) (result Result, err error) { select { case result = <-*c: @@ -119,7 +119,7 @@ func (client *SsntpTestClient) GetCmdChanResult(c *chan Result, cmd ssntp.Comman return result, err } -// SendResultAndDelCmdChan deletes a command from the SsntpTestClient command channel +// SendResultAndDelCmdChan deletes an ssntp.Command from the SsntpTestClient command channel func (client *SsntpTestClient) SendResultAndDelCmdChan(cmd ssntp.Command, result Result) { client.CmdChansLock.Lock() defer client.CmdChansLock.Unlock() @@ -131,7 +131,7 @@ func (client *SsntpTestClient) SendResultAndDelCmdChan(cmd ssntp.Command, result } } -// AddEventChan adds a command to the SsntpTestClient event channel +// AddEventChan adds a ssntp.Event to the SsntpTestClient event channel func (client *SsntpTestClient) AddEventChan(evt ssntp.Event) *chan Result { c := make(chan Result) @@ -142,7 +142,7 @@ func (client *SsntpTestClient) AddEventChan(evt ssntp.Event) *chan Result { return &c } -// GetEventChanResult gets a CmdResult from the SsntpTestClient event channel +// GetEventChanResult gets a Result from the SsntpTestClient event channel func (client *SsntpTestClient) GetEventChanResult(c *chan Result, evt ssntp.Event) (result Result, err error) { select { case result = <-*c: @@ -156,7 +156,7 @@ func (client *SsntpTestClient) GetEventChanResult(c *chan Result, evt ssntp.Even return result, err } -// SendResultAndDelEventChan deletes an event from the SsntpTestClient event channel +// SendResultAndDelEventChan deletes an ssntp.Event from the SsntpTestClient event channel func (client *SsntpTestClient) SendResultAndDelEventChan(evt ssntp.Event, result Result) { client.EventChansLock.Lock() defer client.EventChansLock.Unlock() @@ -168,7 +168,7 @@ func (client *SsntpTestClient) SendResultAndDelEventChan(evt ssntp.Event, result } } -// AddErrorChan adds a command to the SsntpTestClient error channel +// AddErrorChan adds a ssntp.Error to the SsntpTestClient error channel func (client *SsntpTestClient) AddErrorChan(error ssntp.Error) *chan Result { c := make(chan Result) @@ -179,7 +179,7 @@ func (client *SsntpTestClient) AddErrorChan(error ssntp.Error) *chan Result { return &c } -// GetErrorChanResult gets a CmdResult from the SsntpTestClient error channel +// GetErrorChanResult gets a Result from the SsntpTestClient error channel func (client *SsntpTestClient) GetErrorChanResult(c *chan Result, error ssntp.Error) (result Result, err error) { select { case result = <-*c: @@ -193,7 +193,7 @@ func (client *SsntpTestClient) GetErrorChanResult(c *chan Result, error ssntp.Er return result, err } -// SendResultAndDelErrorChan deletes an error from the SsntpTestClient error channel +// SendResultAndDelErrorChan deletes an ssntp.Error from the SsntpTestClient error channel func (client *SsntpTestClient) SendResultAndDelErrorChan(error ssntp.Error, result Result) { client.ErrorChansLock.Lock() defer client.ErrorChansLock.Unlock() diff --git a/testutil/controller.go b/testutil/controller.go index 3aba227a7..22cfd4d57 100644 --- a/testutil/controller.go +++ b/testutil/controller.go @@ -75,7 +75,7 @@ func NewSsntpTestControllerConnection(name string, uuid string) (*SsntpTestContr return ctl, nil } -// AddCmdChan adds a command to the SsntpTestController command channel +// AddCmdChan adds an ssntp.Command to the SsntpTestController command channel func (ctl *SsntpTestController) AddCmdChan(cmd ssntp.Command) *chan Result { c := make(chan Result) @@ -86,7 +86,7 @@ func (ctl *SsntpTestController) AddCmdChan(cmd ssntp.Command) *chan Result { return &c } -// GetCmdChanResult gets a CmdResult from the SsntpTestController command channel +// GetCmdChanResult gets a Result from the SsntpTestController command channel func (ctl *SsntpTestController) GetCmdChanResult(c *chan Result, cmd ssntp.Command) (result Result, err error) { select { case result = <-*c: @@ -100,7 +100,7 @@ func (ctl *SsntpTestController) GetCmdChanResult(c *chan Result, cmd ssntp.Comma return result, err } -// SendResultAndDelCmdChan deletes a command from the SsntpTestController command channel +// SendResultAndDelCmdChan deletes an ssntp.Command from the SsntpTestController command channel func (ctl *SsntpTestController) SendResultAndDelCmdChan(cmd ssntp.Command, result Result) { ctl.CmdChansLock.Lock() defer ctl.CmdChansLock.Unlock() @@ -112,7 +112,7 @@ func (ctl *SsntpTestController) SendResultAndDelCmdChan(cmd ssntp.Command, resul } } -// AddEventChan adds a command to the SsntpTestController event channel +// AddEventChan adds an ssntp.Event to the SsntpTestController event channel func (ctl *SsntpTestController) AddEventChan(evt ssntp.Event) *chan Result { c := make(chan Result) @@ -123,7 +123,7 @@ func (ctl *SsntpTestController) AddEventChan(evt ssntp.Event) *chan Result { return &c } -// GetEventChanResult gets a CmdResult from the SsntpTestController event channel +// GetEventChanResult gets a Result from the SsntpTestController event channel func (ctl *SsntpTestController) GetEventChanResult(c *chan Result, evt ssntp.Event) (result Result, err error) { select { case result = <-*c: @@ -137,7 +137,7 @@ func (ctl *SsntpTestController) GetEventChanResult(c *chan Result, evt ssntp.Eve return result, err } -// SendResultAndDelEventChan deletes an event from the SsntpTestController event channel +// SendResultAndDelEventChan deletes an ssntpEvent from the SsntpTestController event channel func (ctl *SsntpTestController) SendResultAndDelEventChan(evt ssntp.Event, result Result) { ctl.EventChansLock.Lock() defer ctl.EventChansLock.Unlock() @@ -149,7 +149,7 @@ func (ctl *SsntpTestController) SendResultAndDelEventChan(evt ssntp.Event, resul } } -// AddErrorChan adds a command to the SsntpTestController error channel +// AddErrorChan adds an ssntp.Error to the SsntpTestController error channel func (ctl *SsntpTestController) AddErrorChan(error ssntp.Error) *chan Result { c := make(chan Result) @@ -160,7 +160,7 @@ func (ctl *SsntpTestController) AddErrorChan(error ssntp.Error) *chan Result { return &c } -// GetErrorChanResult gets a CmdResult from the SsntpTestController error channel +// GetErrorChanResult gets a Result from the SsntpTestController error channel func (ctl *SsntpTestController) GetErrorChanResult(c *chan Result, error ssntp.Error) (result Result, err error) { select { case result = <-*c: @@ -174,7 +174,7 @@ func (ctl *SsntpTestController) GetErrorChanResult(c *chan Result, error ssntp.E return result, err } -// SendResultAndDelErrorChan deletes an error from the SsntpTestController error channel +// SendResultAndDelErrorChan deletes an ssntp.Error from the SsntpTestController error channel func (ctl *SsntpTestController) SendResultAndDelErrorChan(error ssntp.Error, result Result) { ctl.ErrorChansLock.Lock() defer ctl.ErrorChansLock.Unlock() diff --git a/testutil/server.go b/testutil/server.go index ef890de5f..9fb44853f 100644 --- a/testutil/server.go +++ b/testutil/server.go @@ -43,7 +43,7 @@ type SsntpTestServer struct { NetClientsLock *sync.RWMutex } -// AddCmdChan adds a command to the SsntpTestServer command channel +// AddCmdChan adds an ssntp.Command to the SsntpTestServer command channel func (server *SsntpTestServer) AddCmdChan(cmd ssntp.Command) *chan Result { c := make(chan Result) @@ -54,7 +54,7 @@ func (server *SsntpTestServer) AddCmdChan(cmd ssntp.Command) *chan Result { return &c } -// GetCmdChanResult gets a CmdResult from the SsntpTestServer command channel +// GetCmdChanResult gets a Result from the SsntpTestServer command channel func (server *SsntpTestServer) GetCmdChanResult(c *chan Result, cmd ssntp.Command) (result Result, err error) { select { case result = <-*c: @@ -68,7 +68,7 @@ func (server *SsntpTestServer) GetCmdChanResult(c *chan Result, cmd ssntp.Comman return result, err } -// SendResultAndDelCmdChan deletes a command from the SsntpTestServer command channel +// SendResultAndDelCmdChan deletes an ssntp.Command from the SsntpTestServer command channel func (server *SsntpTestServer) SendResultAndDelCmdChan(cmd ssntp.Command, result Result) { server.CmdChansLock.Lock() defer server.CmdChansLock.Unlock() @@ -80,7 +80,7 @@ func (server *SsntpTestServer) SendResultAndDelCmdChan(cmd ssntp.Command, result } } -// AddEventChan adds a command to the SsntpTestServer event channel +// AddEventChan adds an ssntp.Event to the SsntpTestServer event channel func (server *SsntpTestServer) AddEventChan(evt ssntp.Event) *chan Result { c := make(chan Result) @@ -91,7 +91,7 @@ func (server *SsntpTestServer) AddEventChan(evt ssntp.Event) *chan Result { return &c } -// GetEventChanResult gets a CmdResult from the SsntpTestServer event channel +// GetEventChanResult gets a Result from the SsntpTestServer event channel func (server *SsntpTestServer) GetEventChanResult(c *chan Result, evt ssntp.Event) (result Result, err error) { select { case result = <-*c: @@ -105,7 +105,7 @@ func (server *SsntpTestServer) GetEventChanResult(c *chan Result, evt ssntp.Even return result, err } -// SendResultAndDelEventChan deletes an event from the SsntpTestServer event channel +// SendResultAndDelEventChan deletes an ssntp.Event from the SsntpTestServer event channel func (server *SsntpTestServer) SendResultAndDelEventChan(evt ssntp.Event, result Result) { server.EventChansLock.Lock() defer server.EventChansLock.Unlock() @@ -117,7 +117,7 @@ func (server *SsntpTestServer) SendResultAndDelEventChan(evt ssntp.Event, result } } -// AddErrorChan adds a command to the SsntpTestServer error channel +// AddErrorChan adds an ssntp.Error to the SsntpTestServer error channel func (server *SsntpTestServer) AddErrorChan(error ssntp.Error) *chan Result { c := make(chan Result) @@ -142,7 +142,7 @@ func (server *SsntpTestServer) GetErrorChanResult(c *chan Result, error ssntp.Er return result, err } -// SendResultAndDelErrorChan deletes an error from the SsntpTestServer error channel +// SendResultAndDelErrorChan deletes an ssntp.Error from the SsntpTestServer error channel func (server *SsntpTestServer) SendResultAndDelErrorChan(error ssntp.Error, result Result) { server.ErrorChansLock.Lock() defer server.ErrorChansLock.Unlock() From fe17043c163a76355f6ed0c573d3c5d41986f79c Mon Sep 17 00:00:00 2001 From: Tim Pepper Date: Mon, 20 Jun 2016 21:15:34 -0700 Subject: [PATCH 03/10] testutil: add ssntp.Status test Result channels Agents send ssntp.Status changes for server consumption. As with the event/error/command testutil.Result channels, we need one for status messages to be tracked in test cases to verify delivery. A next patch will introduce usage of the new status channel in a broader test case, but here at least is unit coverage of the helpers in isolation. Signed-off-by: Tim Pepper --- testutil/agent.go | 53 +++++++++++++++++++++++++++++++----- testutil/agent_test.go | 29 ++++++++++++++++++++ testutil/server.go | 60 ++++++++++++++++++++++++++++++++++------- testutil/server_test.go | 29 ++++++++++++++++++++ 4 files changed, 156 insertions(+), 15 deletions(-) diff --git a/testutil/agent.go b/testutil/agent.go index fcdfca280..9dfbacbe9 100644 --- a/testutil/agent.go +++ b/testutil/agent.go @@ -47,12 +47,14 @@ type SsntpTestClient struct { traces []*ssntp.Frame tracesLock *sync.Mutex - CmdChans map[ssntp.Command]chan Result - CmdChansLock *sync.Mutex - EventChans map[ssntp.Event]chan Result - EventChansLock *sync.Mutex - ErrorChans map[ssntp.Error]chan Result - ErrorChansLock *sync.Mutex + CmdChans map[ssntp.Command]chan Result + CmdChansLock *sync.Mutex + EventChans map[ssntp.Event]chan Result + EventChansLock *sync.Mutex + ErrorChans map[ssntp.Error]chan Result + ErrorChansLock *sync.Mutex + StatusChans map[ssntp.Status]chan Result + StatusChansLock *sync.Mutex } // NewSsntpTestClientConnection creates an SsntpTestClient and dials the server. @@ -78,6 +80,8 @@ func NewSsntpTestClientConnection(name string, role ssntp.Role, uuid string) (*S client.EventChansLock = &sync.Mutex{} client.ErrorChans = make(map[ssntp.Error]chan Result) client.ErrorChansLock = &sync.Mutex{} + client.StatusChans = make(map[ssntp.Status]chan Result) + client.StatusChansLock = &sync.Mutex{} client.instancesLock = &sync.Mutex{} client.tracesLock = &sync.Mutex{} @@ -205,6 +209,43 @@ func (client *SsntpTestClient) SendResultAndDelErrorChan(error ssntp.Error, resu } } +// AddStatusChan adds an ssntp.Status to the SsntpTestClient status channel +func (client *SsntpTestClient) AddStatusChan(status ssntp.Status) *chan Result { + c := make(chan Result) + + client.StatusChansLock.Lock() + client.StatusChans[status] = c + client.StatusChansLock.Unlock() + + return &c +} + +// GetStatusChanResult gets a Result from the SsntpTestClient status channel +func (client *SsntpTestClient) GetStatusChanResult(c *chan Result, status ssntp.Status) (result Result, err error) { + select { + case result = <-*c: + if result.Err != nil { + err = fmt.Errorf("Client error sending %s status: %s\n", status, result.Err) + } + case <-time.After(5 * time.Second): + err = fmt.Errorf("Timeout waiting for client %s status result\n", status) + } + + return result, err +} + +// SendResultAndDelStatusChan deletes an ssntp.Status from the SsntpTestClient status channel +func (client *SsntpTestClient) SendResultAndDelStatusChan(status ssntp.Status, result Result) { + client.StatusChansLock.Lock() + defer client.StatusChansLock.Unlock() + c, ok := client.StatusChans[status] + if ok { + delete(client.StatusChans, status) + c <- result + close(c) + } +} + // ConnectNotify implements the SSNTP client ConnectNotify callback for SsntpTestClient func (client *SsntpTestClient) ConnectNotify() { var result Result diff --git a/testutil/agent_test.go b/testutil/agent_test.go index fa857ee81..4556b7934 100644 --- a/testutil/agent_test.go +++ b/testutil/agent_test.go @@ -36,6 +36,35 @@ func TestNewSsntpTestClientonnectionArgs(t *testing.T) { } } +func TestAgentStatusChan(t *testing.T) { + agentCh := agent.AddStatusChan(ssntp.READY) + + var result Result + result.Err = errors.New("foo") + go agent.SendResultAndDelStatusChan(ssntp.READY, result) + + r, err := agent.GetStatusChanResult(agentCh, ssntp.READY) + if err == nil { + t.Fatal(err) + } + if r.Err != result.Err { + t.Fatalf("channel returned wrong result: expected \"%s\", got \"%s\"\n", result.Err, r.Err) + } +} + +func TestAgentStatusChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + + agentCh := agent.AddStatusChan(ssntp.READY) + + _, err := agent.GetStatusChanResult(agentCh, ssntp.READY) + if err == nil { + t.Fatal(err) + } +} + func TestAgentErrorChan(t *testing.T) { agentCh := agent.AddErrorChan(ssntp.StopFailure) diff --git a/testutil/server.go b/testutil/server.go index 9fb44853f..d7bf0808f 100644 --- a/testutil/server.go +++ b/testutil/server.go @@ -29,15 +29,17 @@ import ( // SsntpTestServer is global state for the testutil SSNTP server type SsntpTestServer struct { - Ssntp ssntp.Server - clients []string - clientsLock *sync.Mutex - CmdChans map[ssntp.Command]chan Result - CmdChansLock *sync.Mutex - EventChans map[ssntp.Event]chan Result - EventChansLock *sync.Mutex - ErrorChans map[ssntp.Error]chan Result - ErrorChansLock *sync.Mutex + Ssntp ssntp.Server + clients []string + clientsLock *sync.Mutex + CmdChans map[ssntp.Command]chan Result + CmdChansLock *sync.Mutex + EventChans map[ssntp.Event]chan Result + EventChansLock *sync.Mutex + ErrorChans map[ssntp.Error]chan Result + ErrorChansLock *sync.Mutex + StatusChans map[ssntp.Status]chan Result + StatusChansLock *sync.Mutex NetClients map[string]bool NetClientsLock *sync.RWMutex @@ -154,6 +156,43 @@ func (server *SsntpTestServer) SendResultAndDelErrorChan(error ssntp.Error, resu } } +// AddStatusChan adds an ssntp.Status to the SsntpTestServer status channel +func (server *SsntpTestServer) AddStatusChan(status ssntp.Status) *chan Result { + c := make(chan Result) + + server.StatusChansLock.Lock() + server.StatusChans[status] = c + server.StatusChansLock.Unlock() + + return &c +} + +// GetStatusChanResult gets a Result from the SsntpTestServer status channel +func (server *SsntpTestServer) GetStatusChanResult(c *chan Result, status ssntp.Status) (result Result, err error) { + select { + case result = <-*c: + if result.Err != nil { + err = fmt.Errorf("Server error handling %s status: %s\n", status, result.Err) + } + case <-time.After(5 * time.Second): + err = fmt.Errorf("Timeout waiting for server %s status result\n", status) + } + + return result, err +} + +// SendResultAndDelStatusChan deletes an ssntp.Status from the SsntpTestServer status channel +func (server *SsntpTestServer) SendResultAndDelStatusChan(error ssntp.Status, result Result) { + server.StatusChansLock.Lock() + defer server.StatusChansLock.Unlock() + c, ok := server.StatusChans[error] + if ok { + delete(server.StatusChans, error) + c <- result + close(c) + } +} + // ConnectNotify implements an SSNTP ConnectNotify callback for SsntpTestServer func (server *SsntpTestServer) ConnectNotify(uuid string, role ssntp.Role) { var result Result @@ -482,6 +521,9 @@ func StartTestServer(server *SsntpTestServer) { server.ErrorChans = make(map[ssntp.Error]chan Result) server.ErrorChansLock = &sync.Mutex{} + server.StatusChans = make(map[ssntp.Status]chan Result) + server.StatusChansLock = &sync.Mutex{} + server.NetClients = make(map[string]bool) server.NetClientsLock = &sync.RWMutex{} diff --git a/testutil/server_test.go b/testutil/server_test.go index c84523a37..92e1d3af1 100644 --- a/testutil/server_test.go +++ b/testutil/server_test.go @@ -24,6 +24,35 @@ import ( . "github.com/01org/ciao/testutil" ) +func TestServerStatusChan(t *testing.T) { + serverCh := server.AddStatusChan(ssntp.READY) + + var result Result + result.Err = errors.New("foo") + go server.SendResultAndDelStatusChan(ssntp.READY, result) + + r, err := server.GetStatusChanResult(serverCh, ssntp.READY) + if err == nil { + t.Fatal(err) + } + if r.Err != result.Err { + t.Fatalf("channel returned wrong result: expected \"%s\", got \"%s\"\n", result.Err, r.Err) + } +} + +func TestServerStatusChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + + serverCh := server.AddStatusChan(ssntp.READY) + + _, err := server.GetStatusChanResult(serverCh, ssntp.READY) + if err == nil { + t.Fatal(err) + } +} + func TestServerErrorChan(t *testing.T) { serverCh := server.AddErrorChan(ssntp.StopFailure) From 9e2488907e95ae9a8e086c4ffd92dc84e60f63cd Mon Sep 17 00:00:00 2001 From: Tim Pepper Date: Mon, 20 Jun 2016 21:20:44 -0700 Subject: [PATCH 04/10] testutil: add StatusNotify coverage Agents regularly send READY status indications after connecting to a server. These status frames are distinct from ssntp.STATS commands frames and need their own coverage. The client SendStats() helper is renamed to SendStatsCmd() for clarity versus the newly introduced SendStatus() helper. Signed-off-by: Tim Pepper --- ciao-controller/compute_test.go | 14 +++++++------- ciao-controller/controller_test.go | 16 ++++++++-------- testutil/agent.go | 24 ++++++++++++++++++++++-- testutil/client_server_test.go | 26 ++++++++++++++++++++++++-- testutil/server.go | 10 ++++++++++ 5 files changed, 71 insertions(+), 19 deletions(-) diff --git a/ciao-controller/compute_test.go b/ciao-controller/compute_test.go index 1529e1643..685957ab7 100644 --- a/ciao-controller/compute_test.go +++ b/ciao-controller/compute_test.go @@ -234,7 +234,7 @@ func TestDeleteServer(t *testing.T) { time.Sleep(2 * time.Second) c := client.AddCmdChan(ssntp.STATS) - go client.SendStats() + go client.SendStatsCmd() _, err = client.GetCmdChanResult(c, ssntp.STATS) if err != nil { t.Fatal(err) @@ -281,7 +281,7 @@ func TestServersActionStart(t *testing.T) { time.Sleep(2 * time.Second) c := client.AddCmdChan(ssntp.STATS) - go client.SendStats() + go client.SendStatsCmd() _, err = client.GetCmdChanResult(c, ssntp.STATS) if err != nil { t.Fatal(err) @@ -297,7 +297,7 @@ func TestServersActionStart(t *testing.T) { time.Sleep(1 * time.Second) c = client.AddCmdChan(ssntp.STATS) - go client.SendStats() + go client.SendStatsCmd() _, err = client.GetCmdChanResult(c, ssntp.STATS) if err != nil { t.Fatal(err) @@ -343,7 +343,7 @@ func TestServersActionStop(t *testing.T) { time.Sleep(2 * time.Second) c := client.AddCmdChan(ssntp.STATS) - go client.SendStats() + go client.SendStatsCmd() _, err = client.GetCmdChanResult(c, ssntp.STATS) if err != nil { t.Fatal(err) @@ -389,7 +389,7 @@ func TestServerActionStop(t *testing.T) { time.Sleep(2 * time.Second) c := client.AddCmdChan(ssntp.STATS) - go client.SendStats() + go client.SendStatsCmd() _, err = client.GetCmdChanResult(c, ssntp.STATS) if err != nil { t.Fatal(err) @@ -423,7 +423,7 @@ func TestServerActionStart(t *testing.T) { time.Sleep(1 * time.Second) c := client.AddCmdChan(ssntp.STATS) - go client.SendStats() + go client.SendStatsCmd() _, err = client.GetCmdChanResult(c, ssntp.STATS) if err != nil { t.Fatal(err) @@ -446,7 +446,7 @@ func TestServerActionStart(t *testing.T) { time.Sleep(1 * time.Second) c = client.AddCmdChan(ssntp.STATS) - go client.SendStats() + go client.SendStatsCmd() _, err = client.GetCmdChanResult(c, ssntp.STATS) if err != nil { t.Fatal(err) diff --git a/ciao-controller/controller_test.go b/ciao-controller/controller_test.go index 9218a652a..3bd1fa745 100644 --- a/ciao-controller/controller_test.go +++ b/ciao-controller/controller_test.go @@ -269,7 +269,7 @@ func TestDeleteInstance(t *testing.T) { time.Sleep(1 * time.Second) - client.SendStats() + client.SendStatsCmd() c := server.AddCmdChan(ssntp.DELETE) @@ -297,7 +297,7 @@ func TestStopInstance(t *testing.T) { time.Sleep(1 * time.Second) - client.SendStats() + client.SendStatsCmd() c := server.AddCmdChan(ssntp.STOP) @@ -325,7 +325,7 @@ func TestRestartInstance(t *testing.T) { time.Sleep(1 * time.Second) - client.SendStats() + client.SendStatsCmd() c := server.AddCmdChan(ssntp.STOP) @@ -347,7 +347,7 @@ func TestRestartInstance(t *testing.T) { // now attempt to restart time.Sleep(1 * time.Second) - client.SendStats() + client.SendStatsCmd() c = server.AddCmdChan(ssntp.RESTART) @@ -400,7 +400,7 @@ func TestInstanceDeletedEvent(t *testing.T) { time.Sleep(1 * time.Second) - client.SendStats() + client.SendStatsCmd() time.Sleep(1 * time.Second) @@ -482,7 +482,7 @@ func TestStopFailure(t *testing.T) { time.Sleep(1 * time.Second) - client.SendStats() + client.SendStatsCmd() time.Sleep(1 * time.Second) @@ -534,7 +534,7 @@ func TestRestartFailure(t *testing.T) { time.Sleep(1 * time.Second) - client.SendStats() + client.SendStatsCmd() time.Sleep(1 * time.Second) @@ -555,7 +555,7 @@ func TestRestartFailure(t *testing.T) { time.Sleep(1 * time.Second) - client.SendStats() + client.SendStatsCmd() time.Sleep(1 * time.Second) diff --git a/testutil/agent.go b/testutil/agent.go index 9dfbacbe9..bf14e4063 100644 --- a/testutil/agent.go +++ b/testutil/agent.go @@ -444,8 +444,8 @@ func (client *SsntpTestClient) EventNotify(event ssntp.Event, frame *ssntp.Frame func (client *SsntpTestClient) ErrorNotify(error ssntp.Error, frame *ssntp.Frame) { } -// SendStats pushes an ssntp.STATS command frame from the SsntpTestClient -func (client *SsntpTestClient) SendStats() { +// SendStatsCmd pushes an ssntp.STATS command frame from the SsntpTestClient +func (client *SsntpTestClient) SendStatsCmd() { var result Result payload := StatsPayload(client.UUID, client.Name, client.instances, nil) @@ -463,6 +463,26 @@ func (client *SsntpTestClient) SendStats() { client.SendResultAndDelCmdChan(ssntp.STATS, result) } +// SendStatus pushes an ssntp status frame from the SsntpTestClient with +// the indicated total and available memory statistics +func (client *SsntpTestClient) SendStatus(memTotal int, memAvail int) { + var result Result + + payload := ReadyPayload(client.UUID, memTotal, memAvail) + + y, err := yaml.Marshal(payload) + if err != nil { + result.Err = err + } else { + _, err = client.Ssntp.SendStatus(ssntp.READY, y) + if err != nil { + result.Err = err + } + } + + client.SendResultAndDelCmdChan(ssntp.STATS, result) +} + // SendTrace allows an SsntpTestClient to push an ssntp.TraceReport event frame func (client *SsntpTestClient) SendTrace() { var result Result diff --git a/testutil/client_server_test.go b/testutil/client_server_test.go index 9e48f6a14..31645982f 100644 --- a/testutil/client_server_test.go +++ b/testutil/client_server_test.go @@ -33,6 +33,28 @@ var controller *SsntpTestController var agent *SsntpTestClient var netAgent *SsntpTestClient +func TestSendAgentStatus(t *testing.T) { + serverCh := server.AddStatusChan(ssntp.READY) + + go agent.SendStatus(16384, 16384) + + _, err := server.GetStatusChanResult(serverCh, ssntp.READY) + if err != nil { + t.Fatal(err) + } +} + +func TestSendNetAgentStatus(t *testing.T) { + serverCh := server.AddStatusChan(ssntp.READY) + + go netAgent.SendStatus(16384, 16384) + + _, err := server.GetStatusChanResult(serverCh, ssntp.READY) + if err != nil { + t.Fatal(err) + } +} + func TestStart(t *testing.T) { serverCh := server.AddCmdChan(ssntp.START) agentCh := agent.AddCmdChan(ssntp.START) @@ -85,12 +107,12 @@ func TestStartFailure(t *testing.T) { } } -func TestSendStatus(t *testing.T) { +func TestSendStats(t *testing.T) { agentCh := agent.AddCmdChan(ssntp.STATS) serverCh := server.AddCmdChan(ssntp.STATS) controllerCh := controller.AddCmdChan(ssntp.STATS) - go agent.SendStats() + go agent.SendStatsCmd() _, err := agent.GetCmdChanResult(agentCh, ssntp.STATS) if err != nil { diff --git a/testutil/server.go b/testutil/server.go index d7bf0808f..7f21f51b3 100644 --- a/testutil/server.go +++ b/testutil/server.go @@ -236,6 +236,16 @@ func (server *SsntpTestServer) DisconnectNotify(uuid string, role ssntp.Role) { // StatusNotify is an SSNTP callback stub for SsntpTestServer func (server *SsntpTestServer) StatusNotify(uuid string, status ssntp.Status, frame *ssntp.Frame) { + var result Result + + switch status { + case ssntp.READY: + fmt.Printf("server received READY from node %s\n", uuid) + default: + fmt.Printf("server unhandled status frame from node %s\n", uuid) + } + + server.SendResultAndDelStatusChan(status, result) } // CommandNotify implements an SSNTP CommandNotify callback for SsntpTestServer From 461b0c2c57b2202f9e34bcbd7eec99cec312b00a Mon Sep 17 00:00:00 2001 From: Tim Pepper Date: Mon, 20 Jun 2016 21:25:27 -0700 Subject: [PATCH 05/10] testutil: use testing's Short() and Skip() in timeout tests The test channel helpers have long timeouts. As the number of channels grew from command only to also include event, error and status frames, these unit tests are adding minutes to the over test. They don't add a whole lot of benefit in normal runs and come a considerable cost in time. Given they're written tests, I'm not going to delete them, rather use Short() and Skip() to not run them in our normal -short test invocations. Signed-off-by: Tim Pepper --- testutil/agent_test.go | 12 ++++++++++++ testutil/controller_test.go | 12 ++++++++++++ testutil/server_test.go | 12 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/testutil/agent_test.go b/testutil/agent_test.go index 4556b7934..9b6957e69 100644 --- a/testutil/agent_test.go +++ b/testutil/agent_test.go @@ -82,6 +82,10 @@ func TestAgentErrorChan(t *testing.T) { } func TestAgentErrorChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + agentCh := agent.AddErrorChan(ssntp.StopFailure) _, err := agent.GetErrorChanResult(agentCh, ssntp.StopFailure) @@ -107,6 +111,10 @@ func TestAgentEventChan(t *testing.T) { } func TestAgentEventChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + agentCh := agent.AddEventChan(ssntp.TraceReport) _, err := agent.GetEventChanResult(agentCh, ssntp.TraceReport) @@ -132,6 +140,10 @@ func TestAgentCmdChan(t *testing.T) { } func TestAgentCmdChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + agentCh := agent.AddCmdChan(ssntp.START) _, err := agent.GetCmdChanResult(agentCh, ssntp.START) diff --git a/testutil/controller_test.go b/testutil/controller_test.go index bea4bd3bb..49fa45cac 100644 --- a/testutil/controller_test.go +++ b/testutil/controller_test.go @@ -48,6 +48,10 @@ func TestControllerErrorChan(t *testing.T) { } func TestControllerErrorChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + controllerCh := controller.AddErrorChan(ssntp.StopFailure) _, err := controller.GetErrorChanResult(controllerCh, ssntp.StopFailure) @@ -73,6 +77,10 @@ func TestControllerEventChan(t *testing.T) { } func TestControllerEventChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + controllerCh := controller.AddEventChan(ssntp.TraceReport) _, err := controller.GetEventChanResult(controllerCh, ssntp.TraceReport) @@ -98,6 +106,10 @@ func TestControllerCmdChan(t *testing.T) { } func TestControllerCmdChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + controllerCh := controller.AddCmdChan(ssntp.START) _, err := controller.GetCmdChanResult(controllerCh, ssntp.START) diff --git a/testutil/server_test.go b/testutil/server_test.go index 92e1d3af1..d3f1e1460 100644 --- a/testutil/server_test.go +++ b/testutil/server_test.go @@ -70,6 +70,10 @@ func TestServerErrorChan(t *testing.T) { } func TestServerErrorChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + serverCh := server.AddErrorChan(ssntp.StopFailure) _, err := server.GetErrorChanResult(serverCh, ssntp.StopFailure) @@ -95,6 +99,10 @@ func TestServerEventChan(t *testing.T) { } func TestServerEventChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + serverCh := server.AddEventChan(ssntp.TraceReport) _, err := server.GetEventChanResult(serverCh, ssntp.TraceReport) @@ -120,6 +128,10 @@ func TestServerCmdChan(t *testing.T) { } func TestServerCmdChanTimeout(t *testing.T) { + if testing.Short() { + t.Skip() + } + serverCh := server.AddCmdChan(ssntp.START) _, err := server.GetCmdChanResult(serverCh, ssntp.START) From a2e4c946f6eb1b780f34ddb454ca3b56a63a41b7 Mon Sep 17 00:00:00 2001 From: Tim Pepper Date: Mon, 20 Jun 2016 21:27:43 -0700 Subject: [PATCH 06/10] testutil: bugfixing in client server test The client server test simulates typical SSNTP command/event/error/status flows. In many cases I wrote test cases to start the flow from the agent when those flows really must start at the controller. Using the client server test as a basis for increased ciao-scheduler coverage immediately had a number of test cases fail. The events were not flowing through a real ssntp server as expected, because they were not originating from a proper source. This is easily fixed by calling controller.Ssntp.*() instead of the agent and netagent client functions. Signed-off-by: Tim Pepper --- testutil/client_server_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testutil/client_server_test.go b/testutil/client_server_test.go index 31645982f..f86e43094 100644 --- a/testutil/client_server_test.go +++ b/testutil/client_server_test.go @@ -138,7 +138,7 @@ func TestStartTraced(t *testing.T) { Label: []byte("testutilTracedSTART"), } - _, err := agent.Ssntp.SendTracedCommand(ssntp.START, []byte(StartYaml), traceConfig) + _, err := controller.Ssntp.SendTracedCommand(ssntp.START, []byte(StartYaml), traceConfig) if err != nil { t.Fatal(err) } @@ -173,7 +173,7 @@ func TestStartCNCI(t *testing.T) { netAgentCh := netAgent.AddCmdChan(ssntp.START) serverCh := server.AddCmdChan(ssntp.START) - _, err := netAgent.Ssntp.SendCommand(ssntp.START, []byte(CNCIStartYaml)) + _, err := controller.Ssntp.SendCommand(ssntp.START, []byte(CNCIStartYaml)) if err != nil { t.Fatal(err) } @@ -192,7 +192,7 @@ func TestStop(t *testing.T) { agentCh := agent.AddCmdChan(ssntp.STOP) serverCh := server.AddCmdChan(ssntp.STOP) - _, err := agent.Ssntp.SendCommand(ssntp.STOP, []byte(StopYaml)) + _, err := controller.Ssntp.SendCommand(ssntp.STOP, []byte(StopYaml)) if err != nil { t.Fatal(err) } @@ -247,7 +247,7 @@ func TestRestart(t *testing.T) { agentCh := agent.AddCmdChan(ssntp.RESTART) serverCh := server.AddCmdChan(ssntp.RESTART) - _, err := agent.Ssntp.SendCommand(ssntp.RESTART, []byte(RestartYaml)) + _, err := controller.Ssntp.SendCommand(ssntp.RESTART, []byte(RestartYaml)) if err != nil { t.Fatal(err) } From a4209d9a3a4253a8e2abab5aa57615e7a7c62ccd Mon Sep 17 00:00:00 2001 From: Tim Pepper Date: Mon, 20 Jun 2016 21:31:31 -0700 Subject: [PATCH 07/10] testutil: add a Ready payload generator Detailed testing of the scheduler and controller will want more than the static ready payload that is in testutil. Here I start a helper function which takes memory parameters to generate a payload for test callers. Signed-off-by: Tim Pepper --- testutil/payloads.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/testutil/payloads.go b/testutil/payloads.go index 9bd7626ee..208f96442 100644 --- a/testutil/payloads.go +++ b/testutil/payloads.go @@ -352,6 +352,21 @@ const NodeConnectedYaml = `node_connected: node_type: ` + payloads.NetworkNode + ` ` +// ReadyPayload is a helper to craft a mostly fixed ssntp.READY status +// payload, with parameters to specify the source node uuid and memory metrics +func ReadyPayload(uuid string, memTotal int, memAvail int) payloads.Ready { + p := payloads.Ready{ + NodeUUID: uuid, + MemTotalMB: memTotal, + MemAvailableMB: memAvail, + DiskTotalMB: 500000, + DiskAvailableMB: 256000, + Load: 0, + CpusOnline: 4, + } + return p +} + // ReadyYaml is a sample node READY ssntp.Status payload for test cases const ReadyYaml = `node_uuid: ` + AgentUUID + ` mem_total_mb: 3896 From 7c458feebc7f1b4ab959d2653e67d8bb1a465d35 Mon Sep 17 00:00:00 2001 From: Tim Pepper Date: Tue, 21 Jun 2016 16:14:22 -0700 Subject: [PATCH 08/10] ciao-scheduler: increase coverage with ssntp flows from testutil I've used the testutil/client_server_test.go as the basis for increased coverage of scheduler. I had reasonable unit testing of the initial scheduler internal functions, but needed something to better simulate the ssntp cluster in order to get more meaningful coverage. Thus my past two weeks of work in testutil. This patch roughly copies testutil/client_server_test.go into scheduler_internal_test.go to extend the fully isolated scheduler tests to ssntp cluster interactive test flows. The primary difference versus what is in the testutil/ is that the synthetic server and its Result channels are not used, but rather the real scheduler implementation. Signed-off-by: Tim Pepper --- ciao-scheduler/scheduler_internal_test.go | 501 +++++++++++++++++++++- 1 file changed, 500 insertions(+), 1 deletion(-) diff --git a/ciao-scheduler/scheduler_internal_test.go b/ciao-scheduler/scheduler_internal_test.go index 87d633260..f9bfa9f07 100644 --- a/ciao-scheduler/scheduler_internal_test.go +++ b/ciao-scheduler/scheduler_internal_test.go @@ -17,17 +17,21 @@ package main import ( + "errors" "flag" "fmt" "os" "sync" "testing" + "time" "github.com/01org/ciao/payloads" "github.com/01org/ciao/ssntp" "github.com/01org/ciao/testutil" + "github.com/docker/distribution/uuid" ) +// an ssntpSchedulerServer instance for non-SSNTP unit tests var sched *ssntpSchedulerServer /****************************************************************************/ @@ -159,7 +163,15 @@ func createStartWorkload(vCpus int, memMB int, diskMB int) *payloads.Start { func TestMain(m *testing.M) { flag.Parse() - os.Exit(m.Run()) + err := ssntpTestsSetup() + if err != nil { + os.Exit(1) + } + + ret := m.Run() + + ssntpTestsTeardown() + os.Exit(ret) } func TestPickComputeNode(t *testing.T) { @@ -639,3 +651,490 @@ func TestGetWorkloadAgentUUID(t *testing.T) { } } } + +/****************************************************************************/ +// pulled from testutil and modified slightly + +// SSNTP entities for integrated test cases +var server *ssntpSchedulerServer +var controller *testutil.SsntpTestController +var agent *testutil.SsntpTestClient +var netAgent *testutil.SsntpTestClient + +// these status sends need to come early so the agents are marked online +// for later ssntp.START's +func TestSendAgentStatus(t *testing.T) { + var wg sync.WaitGroup + + server.cnMutex.Lock() + cn := server.cnMap[testutil.AgentUUID] + if cn == nil { + t.Fatalf("agent node not connected (uuid: %s)", testutil.AgentUUID) + } + server.cnMutex.Unlock() + + wg.Add(1) + go func() { + agent.SendStatus(163840, 163840) + wg.Done() + }() + + wg.Wait() + tgtStatus := ssntp.READY + waitForAgent(testutil.AgentUUID, &tgtStatus) + + server.cnMutex.Lock() + cn = server.cnMap[testutil.AgentUUID] + if cn != nil && cn.status != tgtStatus { + t.Fatalf("agent node incorrect status: expected %s, got %s", tgtStatus.String(), cn.status.String()) + } + server.cnMutex.Unlock() +} +func TestSendNetAgentStatus(t *testing.T) { + var wg sync.WaitGroup + + server.nnMutex.Lock() + nn := server.nnMap[testutil.NetAgentUUID] + if nn == nil { + t.Fatalf("netagent node not connected (uuid: %s)", testutil.NetAgentUUID) + } + server.nnMutex.Unlock() + + wg.Add(1) + go func() { + netAgent.SendStatus(163840, 163840) + wg.Done() + }() + + wg.Wait() + tgtStatus := ssntp.READY + waitForNetAgent(testutil.NetAgentUUID, &tgtStatus) + + server.nnMutex.Lock() + nn = server.nnMap[testutil.NetAgentUUID] + if nn != nil && nn.status != tgtStatus { + t.Fatalf("netagent node incorrect status: expected %s, got %s", tgtStatus.String(), nn.status.String()) + } + server.nnMutex.Unlock() +} + +func TestStart(t *testing.T) { + agentCh := agent.AddCmdChan(ssntp.START) + + go controller.Ssntp.SendCommand(ssntp.START, []byte(testutil.StartYaml)) + + _, err := agent.GetCmdChanResult(agentCh, ssntp.START) + if err != nil { + t.Fatal(err) + } +} + +func TestStartFailure(t *testing.T) { + agentCh := agent.AddCmdChan(ssntp.START) + + controllerErrorCh := controller.AddErrorChan(ssntp.StartFailure) + fmt.Printf("Expecting controller to note: \"%s\"\n", ssntp.StartFailure) + + agent.StartFail = true + agent.StartFailReason = payloads.FullCloud + defer func() { + agent.StartFail = false + agent.StartFailReason = "" + }() + + go controller.Ssntp.SendCommand(ssntp.START, []byte(testutil.StartYaml)) + + _, err := agent.GetCmdChanResult(agentCh, ssntp.START) + if err == nil { // agent will process the START and does error + t.Fatal(err) + } + + _, err = controller.GetErrorChanResult(controllerErrorCh, ssntp.StartFailure) + if err != nil { + t.Fatal(err) + } +} + +func TestSendStats(t *testing.T) { + agentCh := agent.AddCmdChan(ssntp.STATS) + controllerCh := controller.AddCmdChan(ssntp.STATS) + + go agent.SendStatsCmd() + + _, err := agent.GetCmdChanResult(agentCh, ssntp.STATS) + if err != nil { + t.Fatal(err) + } + _, err = controller.GetCmdChanResult(controllerCh, ssntp.STATS) + if err != nil { + t.Fatal(err) + } +} + +func TestStartTraced(t *testing.T) { + agentCh := agent.AddCmdChan(ssntp.START) + + traceConfig := &ssntp.TraceConfig{ + PathTrace: true, + Start: time.Now(), + Label: []byte("testutilTracedSTART"), + } + + _, err := controller.Ssntp.SendTracedCommand(ssntp.START, []byte(testutil.StartYaml), traceConfig) + if err != nil { + t.Fatal(err) + } + + _, err = agent.GetCmdChanResult(agentCh, ssntp.START) + if err != nil { + t.Fatal(err) + } +} + +func TestSendTrace(t *testing.T) { + agentCh := agent.AddEventChan(ssntp.TraceReport) + + go agent.SendTrace() + + _, err := agent.GetEventChanResult(agentCh, ssntp.TraceReport) + if err != nil { + t.Fatal(err) + } +} + +func TestStartCNCI(t *testing.T) { + netAgentCh := netAgent.AddCmdChan(ssntp.START) + + _, err := controller.Ssntp.SendCommand(ssntp.START, []byte(testutil.CNCIStartYaml)) + if err != nil { + t.Fatal(err) + } + + _, err = netAgent.GetCmdChanResult(netAgentCh, ssntp.START) + if err != nil { + t.Fatal(err) + } +} + +func TestStop(t *testing.T) { + agentCh := agent.AddCmdChan(ssntp.STOP) + + _, err := controller.Ssntp.SendCommand(ssntp.STOP, []byte(testutil.StopYaml)) + if err != nil { + t.Fatal(err) + } + + _, err = agent.GetCmdChanResult(agentCh, ssntp.STOP) + if err != nil { + t.Fatal(err) + } +} + +func TestStopFailure(t *testing.T) { + agentCh := agent.AddCmdChan(ssntp.STOP) + + controllerErrorCh := controller.AddErrorChan(ssntp.StopFailure) + fmt.Printf("Expecting controller to note: \"%s\"\n", ssntp.StopFailure) + + agent.StopFail = true + agent.StopFailReason = payloads.StopNoInstance + defer func() { + agent.StopFail = false + agent.StopFailReason = "" + }() + + go controller.Ssntp.SendCommand(ssntp.STOP, []byte(testutil.StopYaml)) + + _, err := agent.GetCmdChanResult(agentCh, ssntp.STOP) + if err == nil { // agent will process the STOP and does error + t.Fatal(err) + } + + _, err = controller.GetErrorChanResult(controllerErrorCh, ssntp.StopFailure) + if err != nil { + t.Fatal(err) + } +} + +func TestRestart(t *testing.T) { + agentCh := agent.AddCmdChan(ssntp.RESTART) + + _, err := controller.Ssntp.SendCommand(ssntp.RESTART, []byte(testutil.RestartYaml)) + if err != nil { + t.Fatal(err) + } + + _, err = agent.GetCmdChanResult(agentCh, ssntp.RESTART) + if err != nil { + t.Fatal(err) + } +} + +func TestRestartFailure(t *testing.T) { + agentCh := agent.AddCmdChan(ssntp.RESTART) + + controllerErrorCh := controller.AddErrorChan(ssntp.RestartFailure) + fmt.Printf("Expecting controller to note: \"%s\"\n", ssntp.RestartFailure) + + agent.RestartFail = true + agent.RestartFailReason = payloads.RestartNoInstance + defer func() { + agent.RestartFail = false + agent.RestartFailReason = "" + }() + + go controller.Ssntp.SendCommand(ssntp.RESTART, []byte(testutil.RestartYaml)) + + _, err := agent.GetCmdChanResult(agentCh, ssntp.RESTART) + if err == nil { // agent will process the RESTART and does error + t.Fatal(err) + } + + _, err = controller.GetErrorChanResult(controllerErrorCh, ssntp.RestartFailure) + if err != nil { + t.Fatal(err) + } +} + +func doDelete(fail bool) error { + agentCh := agent.AddCmdChan(ssntp.DELETE) + + var controllerErrorCh *chan testutil.Result + + if fail == true { + controllerErrorCh = controller.AddErrorChan(ssntp.DeleteFailure) + fmt.Printf("Expecting controller to note: \"%s\"\n", ssntp.DeleteFailure) + + agent.DeleteFail = true + agent.DeleteFailReason = payloads.DeleteNoInstance + + defer func() { + agent.DeleteFail = false + agent.DeleteFailReason = "" + }() + } + + go controller.Ssntp.SendCommand(ssntp.DELETE, []byte(testutil.DeleteYaml)) + + _, err := agent.GetCmdChanResult(agentCh, ssntp.DELETE) + if fail == false && err != nil { // agent unexpected fail + return err + } + + if fail == true { + if err == nil { // agent unexpected success + return err + } + _, err = controller.GetErrorChanResult(controllerErrorCh, ssntp.DeleteFailure) + if err != nil { + return err + } + } + + return nil +} + +func propagateInstanceDeleted() error { + agentCh := agent.AddEventChan(ssntp.InstanceDeleted) + controllerCh := controller.AddEventChan(ssntp.InstanceDeleted) + + go agent.SendDeleteEvent(testutil.InstanceUUID) + + _, err := agent.GetEventChanResult(agentCh, ssntp.InstanceDeleted) + if err != nil { + return err + } + _, err = controller.GetEventChanResult(controllerCh, ssntp.InstanceDeleted) + if err != nil { + return err + } + return nil +} + +func TestDelete(t *testing.T) { + fail := false + + err := doDelete(fail) + if err != nil { + t.Fatal(err) + } + + err = propagateInstanceDeleted() + if err != nil { + t.Fatal(err) + } +} + +func TestDeleteFailure(t *testing.T) { + fail := true + + err := doDelete(fail) + if err != nil { + t.Fatal(err) + } +} + +func stopServer() error { + controllerCh := controller.AddEventChan(ssntp.NodeDisconnected) + netAgentCh := netAgent.AddEventChan(ssntp.NodeDisconnected) + agentCh := agent.AddEventChan(ssntp.NodeDisconnected) + + server.ssntp.Stop() + + _, err := controller.GetEventChanResult(controllerCh, ssntp.NodeDisconnected) + if err != nil { + return err + } + _, err = netAgent.GetEventChanResult(netAgentCh, ssntp.NodeDisconnected) + if err != nil { + return err + } + _, err = agent.GetEventChanResult(agentCh, ssntp.NodeDisconnected) + if err != nil { + return err + } + return nil +} + +func restartServer() error { + controllerCh := controller.AddEventChan(ssntp.NodeConnected) + netAgentCh := netAgent.AddEventChan(ssntp.NodeConnected) + agentCh := agent.AddEventChan(ssntp.NodeConnected) + + server = configSchedulerServer() + if server == nil { + return errors.New("unable to configure scheduler") + } + go server.ssntp.Serve(server.config, server) + //go heartBeatLoop(server) ...handy for debugging + + _, err := controller.GetEventChanResult(controllerCh, ssntp.NodeConnected) + if err != nil { + return err + } + _, err = netAgent.GetEventChanResult(netAgentCh, ssntp.NodeConnected) + if err != nil { + return err + } + _, err = agent.GetEventChanResult(agentCh, ssntp.NodeConnected) + if err != nil { + return err + } + return nil +} + +func TestReconnects(t *testing.T) { + err := stopServer() + if err != nil { + t.Fatal(err) + } + + time.Sleep(1 * time.Second) + + err = restartServer() + if err != nil { + t.Fatal(err) + } +} + +func waitForController(uuid string) { + for { + server.controllerMutex.Lock() + c := server.controllerMap[uuid] + server.controllerMutex.Unlock() + + if c == nil { + fmt.Printf("awaiting controller %s\n", uuid) + time.Sleep(50 * time.Millisecond) + } else { + return + } + } +} +func waitForAgent(uuid string, status *ssntp.Status) { + for { + server.cnMutex.Lock() + cn := server.cnMap[uuid] + server.cnMutex.Unlock() + + if cn == nil { + fmt.Printf("awaiting agent %s\n", uuid) + time.Sleep(50 * time.Millisecond) + } else if status != nil && *status != cn.status { + fmt.Printf("awaiting agent %s in state %s\n", uuid, status.String()) + time.Sleep(50 * time.Millisecond) + } else { + return + } + } +} +func waitForNetAgent(uuid string, status *ssntp.Status) { + for { + server.nnMutex.Lock() + nn := server.nnMap[uuid] + server.nnMutex.Unlock() + + if nn == nil { + fmt.Printf("awaiting netagent %s\n", uuid) + time.Sleep(50 * time.Millisecond) + } else if status != nil && *status != nn.status { + fmt.Printf("awaiting netagent %s in state %s\n", uuid, status.String()) + time.Sleep(50 * time.Millisecond) + } else { + return + } + } +} + +func ssntpTestsSetup() error { + var err error + + // start server + server = configSchedulerServer() + if server == nil { + return errors.New("unable to configure scheduler") + } + go server.ssntp.Serve(server.config, server) + //go heartBeatLoop(server) ...handy for debugging + + // start controller + controllerUUID := uuid.Generate().String() + controller, err = testutil.NewSsntpTestControllerConnection("Controller Client", controllerUUID) + if err != nil { + return err + } + + // start agent + agent, err = testutil.NewSsntpTestClientConnection("AGENT Client", ssntp.AGENT, testutil.AgentUUID) + if err != nil { + return err + } + + // start netagent + netAgent, err = testutil.NewSsntpTestClientConnection("NETAGENT Client", ssntp.NETAGENT, testutil.NetAgentUUID) + if err != nil { + return err + } + + // insure the three clients are connected: + waitForController(controllerUUID) + waitForAgent(testutil.AgentUUID, nil) + waitForNetAgent(testutil.NetAgentUUID, nil) + + return nil +} + +func ssntpTestsTeardown() { + // stop everybody + time.Sleep(1 * time.Second) + controller.Ssntp.Close() + + time.Sleep(1 * time.Second) + netAgent.Ssntp.Close() + + time.Sleep(1 * time.Second) + agent.Ssntp.Close() + + time.Sleep(1 * time.Second) + server.ssntp.Stop() +} From a2c16ce7a699c62c4ed01ccd079a41b4e1ac34bc Mon Sep 17 00:00:00 2001 From: Tim Pepper Date: Tue, 21 Jun 2016 15:26:27 -0700 Subject: [PATCH 09/10] testutil: use standard NetAgentUUID in client-server test The example test is about showing how to get reproducible flows through the cluster. For that the NetAgent should be at a known UUID, not a random one. Signed-off-by: Tim Pepper --- testutil/client_server_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testutil/client_server_test.go b/testutil/client_server_test.go index f86e43094..6d50af4e5 100644 --- a/testutil/client_server_test.go +++ b/testutil/client_server_test.go @@ -477,8 +477,7 @@ func TestMain(m *testing.M) { } // start netagent - netAgentUUID := uuid.Generate().String() - netAgent, err = NewSsntpTestClientConnection("NETAGENT Client", ssntp.NETAGENT, netAgentUUID) + netAgent, err = NewSsntpTestClientConnection("NETAGENT Client", ssntp.NETAGENT, NetAgentUUID) if err != nil { os.Exit(1) } From 64c508ceae275f994ad402e81ef8704638c56bb6 Mon Sep 17 00:00:00 2001 From: Tim Pepper Date: Tue, 21 Jun 2016 21:12:34 -0700 Subject: [PATCH 10/10] testutil: add README.md I've neglected to include a readme for the testutil. This is the start of one which hopefully explains why testutil exists distinct from other test options and how to use it. Signed-off-by: Tim Pepper --- testutil/README.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 testutil/README.md diff --git a/testutil/README.md b/testutil/README.md new file mode 100644 index 000000000..edf1c44e7 --- /dev/null +++ b/testutil/README.md @@ -0,0 +1,55 @@ +Ciao Testutil +============= + +A given ciao component will have unit tests which cover its internals in a +component specific way. But this is limited in scope and cannot cover all +of the component's functionality, because that functionality is ultimately +about creating and servicing SSNTP command/event/error/status flows. + +To enable testing of those SSNTP flows, the testutil package provides +a set of common test tools for ciao. Included are: + +* shared payload constants +* shared test certificates +* test agent implementation for ssntp.AGENT and ssntp.NETAGENT roles +* test controller implementation for ssntp.Controller role +* test server implementation for ssntp.SERVER role +* example client-server test spanning the above SSNTP actors +* channels for tracking command/event/error/status flows across the SSNTP + test actors + +This allows a ciao component to be tested more meaninfully, but in partial +isolation. The ciao component under test would be a real implementation, +but its SSNTP peers are the shared synthetic implementations from the +testutil package. + +For an example of how to enable basic SSNTP test flows +and track the results through the channel helpers +review the internals of the [example client-server +test](https://github.com/01org/ciao/blob/master/testutil/client_server_test.go). + +Additional test options +======================= + +Virtual +------- + +The next level of test breadth comes from actually running +a test cluster with real implementations of each ciao +component. This is provided in a controlled fashion via the +[singlevm](https://github.com/01org/ciao/tree/master/testutil/singlevm) +CI script, which has detailed documentation +on the wiki at [Single VM Development +Environment](https://github.com/01org/ciao/wiki/HOWTO:-Single-VM-Development-Environment). + +Physical +-------- + +Testing is of course also possible on a real hardware cluster which has +been set up according to the [cluster setup +guide](https://clearlinux.org/documentation/ciao-cluster-setup.html). +A minimal Build Acceptance Test (BAT) framework outputting TAP +(Test Anything Protocol) results is published [in our release +tools](https://github.com/01org/ciao/tree/master/_release/bat). +The python script drives ciao-cli to query and manipulate the state of +a cluster.