diff --git a/ciao-cli/README.md b/ciao-cli/README.md index afe804a1b..553703173 100644 --- a/ciao-cli/README.md +++ b/ciao-cli/README.md @@ -52,10 +52,14 @@ The options are: The commands are: event + external-ip + image instance node + pool tenant trace + volume workload Use "ciao-cli command -help" for more information about that command. diff --git a/ciao-cli/external-ips.go b/ciao-cli/external-ips.go new file mode 100644 index 000000000..35f491a15 --- /dev/null +++ b/ciao-cli/external-ips.go @@ -0,0 +1,877 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "net" + "net/http" + "os" + "text/tabwriter" + + "github.com/01org/ciao/ciao-controller/api" + "github.com/01org/ciao/ciao-controller/types" +) + +const ( + externalSubnetDesc = `struct { + ID string // ID of the external subnet + CIDR string // CIDR representation of the subnet +}` + externalIPDesc = `struct { + ID string // ID of the external IP + Address string // the IPv4 Address +}` + poolTemplateDesc = `struct { + ID string // ID of the pool (Admin only) + Name string // Name of the pool + TotalIPs int // Total IPs in pool (Admin only) + Free int // Total Free IPs in pool (Admin only) +}` + poolShowTemplateDesc = `struct { + ID string // ID of the pool + Name string // name of the pool + Free int // Total free IPs in pool + TotalIPs int // Total IPs in pool + Subnets []ExternalSubnet // Subnets in this pool + IPs []ExternalIP // Individual IPs in this pool +}` + externalIPTemplate = `struct { + ID string // ID of the mapped IP + ExternalIP string // External IP address + InternalIP string // Internal IP address + InstanceID string // ID of the instance that is mapped (Admin only) + TenantID string // ID of the tenant (Admin only) + PoolID string // ID of the allocation pool (Admin only) + PoolName string // Name of the allocation pool (Admin only) +}` +) + +func getCiaoExternalIPsResource() (string, string, error) { + url, err := getCiaoResource("external-ips", api.ExternalIPsV1) + return url, api.ExternalIPsV1, err +} + +// TBD: in an ideal world, we'd modify the GET to take a query. +func getExternalIPRef(address string) (string, error) { + var IPs []types.MappedIP + + url, ver, err := getCiaoExternalIPsResource() + if err != nil { + return "", err + } + + resp, err := sendCiaoRequest("GET", url, nil, nil, &ver) + if err != nil { + return "", err + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("External IP list failed: %s", resp.Status) + } + + err = unmarshalHTTPResponse(resp, &IPs) + if err != nil { + return "", err + } + + for _, IP := range IPs { + if IP.ExternalIP == address { + url := getRef("self", IP.Links) + if url != "" { + return url, nil + } + } + } + + return "", types.ErrAddressNotFound +} + +var externalIPCommand = &command{ + SubCommands: map[string]subCommand{ + "map": new(externalIPMapCommand), + "list": new(externalIPListCommand), + "unmap": new(externalIPUnMapCommand), + }, +} + +type externalIPMapCommand struct { + Flag flag.FlagSet + instanceID string + poolName string +} + +func (cmd *externalIPMapCommand) usage(...string) { + fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] external-ip map [flags] + +Map an external IP from a given pool to an instance. + +The map flags are: + +`) + cmd.Flag.PrintDefaults() + os.Exit(2) +} + +func (cmd *externalIPMapCommand) parseArgs(args []string) []string { + cmd.Flag.StringVar(&cmd.instanceID, "instance", "", "ID of the instance to map IP to.") + cmd.Flag.StringVar(&cmd.poolName, "pool", "", "Name of the pool to map from.") + cmd.Flag.Usage = func() { cmd.usage() } + cmd.Flag.Parse(args) + return cmd.Flag.Args() +} + +func (cmd *externalIPMapCommand) run(args []string) error { + if cmd.instanceID == "" { + errorf("Missing required -instance parameter") + cmd.usage() + } + + req := types.MapIPRequest{ + InstanceID: cmd.instanceID, + } + + if cmd.poolName != "" { + req.PoolName = &cmd.poolName + } + + b, err := json.Marshal(req) + if err != nil { + fatalf(err.Error()) + } + + body := bytes.NewReader(b) + + url, ver, err := getCiaoExternalIPsResource() + if err != nil { + fatalf(err.Error()) + } + + resp, err := sendCiaoRequest("POST", url, nil, body, &ver) + if err != nil { + fatalf(err.Error()) + } + + if resp.StatusCode != http.StatusNoContent { + fatalf("External IP map failed: %s", resp.Status) + } + + fmt.Printf("Requested external IP for: %s\n", cmd.instanceID) + + return nil +} + +type externalIPListCommand struct { + Flag flag.FlagSet + template string +} + +func (cmd *externalIPListCommand) usage(...string) { + fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] external-ip list [flags] + +List all mapped external IPs. + +The list flags are: + +`) + cmd.Flag.PrintDefaults() + + fmt.Fprintf(os.Stderr, ` +The template passed to the -f option operates on a + +[]%s +`, externalIPTemplate) + fmt.Fprintln(os.Stderr, templateFunctionHelp) + os.Exit(2) +} + +func (cmd *externalIPListCommand) parseArgs(args []string) []string { + cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output") + cmd.Flag.Usage = func() { cmd.usage() } + cmd.Flag.Parse(args) + return cmd.Flag.Args() +} + +func (cmd *externalIPListCommand) run(args []string) error { + var IPs []types.MappedIP + + url, ver, err := getCiaoExternalIPsResource() + if err != nil { + fatalf(err.Error()) + } + + resp, err := sendCiaoRequest("GET", url, nil, nil, &ver) + if err != nil { + fatalf(err.Error()) + } + + if resp.StatusCode != http.StatusOK { + fatalf("External IP list failed: %s", resp.Status) + } + + err = unmarshalHTTPResponse(resp, &IPs) + if err != nil { + fatalf(err.Error()) + } + + if cmd.template != "" { + return outputToTemplate("external-ip-list", cmd.template, + &IPs) + } + + w := new(tabwriter.Writer) + w.Init(os.Stdout, 0, 1, 1, ' ', 0) + fmt.Fprintf(w, "#\tExternalIP\tInternalIP") + if checkPrivilege() { + fmt.Fprintf(w, "\tInstanceID\tTenantID\tPoolName\n") + } else { + fmt.Fprintf(w, "\n") + } + + for i, IP := range IPs { + fmt.Fprintf(w, "%d", i+1) + fmt.Fprintf(w, "\t%s", IP.ExternalIP) + fmt.Fprintf(w, "\t%s", IP.InternalIP) + if IP.InstanceID != "" { + fmt.Fprintf(w, "\t%s", IP.InstanceID) + } + + if IP.TenantID != "" { + fmt.Fprintf(w, "\t%s", IP.TenantID) + } + + if IP.PoolName != "" { + fmt.Fprintf(w, "\t%s", IP.PoolName) + } + + fmt.Fprintf(w, "\n") + } + + w.Flush() + + return nil +} + +type externalIPUnMapCommand struct { + address string + Flag flag.FlagSet +} + +func (cmd *externalIPUnMapCommand) usage(...string) { + fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] external-ip unmap [flags] + +Unmap a given external IP. + +The unmap flags are: + +`) + cmd.Flag.PrintDefaults() + os.Exit(2) +} + +func (cmd *externalIPUnMapCommand) parseArgs(args []string) []string { + cmd.Flag.StringVar(&cmd.address, "address", "", "External IP to unmap.") + cmd.Flag.Usage = func() { cmd.usage() } + cmd.Flag.Parse(args) + return cmd.Flag.Args() +} + +func (cmd *externalIPUnMapCommand) run(args []string) error { + if cmd.address == "" { + errorf("Missing required -instance parameter") + cmd.usage() + } + + url, err := getExternalIPRef(cmd.address) + if err != nil { + fatalf(err.Error()) + } + + ver := api.ExternalIPsV1 + + resp, err := sendCiaoRequest("DELETE", url, nil, nil, &ver) + if err != nil { + fatalf(err.Error()) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusAccepted { + fatalf("Unmap of address failed: %s", resp.Status) + } + + fmt.Printf("Requested unmap of: %s\n", cmd.address) + + return nil +} + +var poolCommand = &command{ + SubCommands: map[string]subCommand{ + "create": new(poolCreateCommand), + "list": new(poolListCommand), + "show": new(poolShowCommand), + "delete": new(poolDeleteCommand), + "add": new(poolAddCommand), + "remove": new(poolRemoveCommand), + }, +} + +type poolCreateCommand struct { + Flag flag.FlagSet + name string +} + +func getCiaoPoolsResource() (string, error) { + return getCiaoResource("pools", api.PoolsV1) +} + +func getCiaoPoolRef(name string) (string, error) { + var pools types.ListPoolsResponse + + query := queryValue{ + name: "name", + value: name, + } + + url, err := getCiaoPoolsResource() + if err != nil { + return "", err + } + + ver := api.PoolsV1 + + resp, err := sendCiaoRequest("GET", url, []queryValue{query}, nil, &ver) + if err != nil { + return "", err + } + + err = unmarshalHTTPResponse(resp, &pools) + if err != nil { + return "", err + } + + // we have now the pool ID + if len(pools.Pools) != 1 { + return "", errors.New("No pool by that name found") + } + + links := pools.Pools[0].Links + url = getRef("self", links) + if url == "" { + return url, errors.New("Invalid Link returned from controller") + } + + return url, nil +} + +func getCiaoPool(name string) (types.Pool, error) { + var pool types.Pool + + url, err := getCiaoPoolRef(name) + if err != nil { + return pool, nil + } + + ver := api.PoolsV1 + + resp, err := sendCiaoRequest("GET", url, nil, nil, &ver) + if err != nil { + return pool, err + } + + err = unmarshalHTTPResponse(resp, &pool) + if err != nil { + return pool, err + } + + return pool, nil +} + +func (cmd *poolCreateCommand) usage(...string) { + fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] pool create [flags] + +Creates a new external IP pool. + +The create flags are: + +`) + cmd.Flag.PrintDefaults() + os.Exit(2) +} + +// TBD: add support for specifying a subnet or []ip addresses. +func (cmd *poolCreateCommand) parseArgs(args []string) []string { + cmd.Flag.StringVar(&cmd.name, "name", "", "Name of pool") + cmd.Flag.Usage = func() { cmd.usage() } + cmd.Flag.Parse(args) + return cmd.Flag.Args() +} + +func (cmd *poolCreateCommand) run(args []string) error { + if cmd.name == "" { + errorf("Missing required -name parameter") + cmd.usage() + } + + req := types.NewPoolRequest{ + Name: cmd.name, + } + + b, err := json.Marshal(req) + if err != nil { + fatalf(err.Error()) + } + + body := bytes.NewReader(b) + + url, err := getCiaoPoolsResource() + if err != nil { + fatalf(err.Error()) + } + + ver := api.PoolsV1 + + resp, err := sendCiaoRequest("POST", url, nil, body, &ver) + if err != nil { + fatalf(err.Error()) + } + + if resp.StatusCode != http.StatusNoContent { + fatalf("Pool creation failed: %s", resp.Status) + } + + fmt.Printf("Created new pool: %s\n", cmd.name) + + return nil +} + +type poolListCommand struct { + Flag flag.FlagSet + template string +} + +func (cmd *poolListCommand) usage(...string) { + fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] pool list [flags] + +List all ciao external IP pools. + +The list flags are: + +`) + cmd.Flag.PrintDefaults() + fmt.Fprintf(os.Stderr, ` +The template passed to the -f option operates on a + +[]%s +`, poolTemplateDesc) + fmt.Fprintln(os.Stderr, templateFunctionHelp) + + os.Exit(2) +} + +func (cmd *poolListCommand) parseArgs(args []string) []string { + cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output") + cmd.Flag.Usage = func() { cmd.usage() } + cmd.Flag.Parse(args) + return cmd.Flag.Args() +} + +// change this command to return different output depending +// on the privilege level of user. Check privilege, then +// if not privileged, build non-priviledged URL. +func (cmd *poolListCommand) run(args []string) error { + var pools types.ListPoolsResponse + + url, err := getCiaoPoolsResource() + if err != nil { + fatalf(err.Error()) + } + + ver := api.PoolsV1 + + resp, err := sendCiaoRequest("GET", url, nil, nil, &ver) + if err != nil { + fatalf(err.Error()) + } + + err = unmarshalHTTPResponse(resp, &pools) + if err != nil { + fatalf(err.Error()) + } + + if cmd.template != "" { + return outputToTemplate("pool-list", cmd.template, + &pools.Pools) + } + + w := new(tabwriter.Writer) + w.Init(os.Stdout, 0, 1, 1, ' ', 0) + fmt.Fprintf(w, "#\tName") + if checkPrivilege() { + fmt.Fprintf(w, "\tTotalIPs\tFreeIPs\n") + } else { + fmt.Fprintf(w, "\n") + } + + for i, pool := range pools.Pools { + fmt.Fprintf(w, "%d", i+1) + fmt.Fprintf(w, "\t%s", pool.Name) + + if pool.TotalIPs != nil { + fmt.Fprintf(w, "\t%d", *pool.TotalIPs) + } + + if pool.Free != nil { + fmt.Fprintf(w, "\t%d", *pool.Free) + } + + fmt.Fprintf(w, "\n") + } + + w.Flush() + + return nil +} + +type poolShowCommand struct { + Flag flag.FlagSet + name string + template string +} + +func (cmd *poolShowCommand) usage(...string) { + fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] pool show [flags] + +Show ciao external IP pool details. + +The show flags are: + +`) + cmd.Flag.PrintDefaults() + fmt.Fprintf(os.Stderr, ` +The template passed to the -f option operates on a + +%s +`, poolShowTemplateDesc) + fmt.Fprintf(os.Stderr, ` +The externalSubnets are described by + +%s +`, externalSubnetDesc) + + fmt.Fprintf(os.Stderr, ` +The externalIPs are described by + +%s +`, externalIPDesc) + fmt.Fprintln(os.Stderr, templateFunctionHelp) + + os.Exit(2) +} + +func (cmd *poolShowCommand) parseArgs(args []string) []string { + cmd.Flag.StringVar(&cmd.name, "name", "", "Name of pool") + cmd.Flag.StringVar(&cmd.template, "f", "", "Template used to format output") + cmd.Flag.Usage = func() { cmd.usage() } + cmd.Flag.Parse(args) + return cmd.Flag.Args() +} + +func dumpPool(pool types.Pool) { + fmt.Printf("\tUUID: %s\n", pool.ID) + fmt.Printf("\tName: %s\n", pool.Name) + fmt.Printf("\tFree IPs: %d\n", pool.Free) + fmt.Printf("\tTotal IPs: %d\n", pool.TotalIPs) + + for _, sub := range pool.Subnets { + fmt.Printf("\tSubnet: %s\n", sub.CIDR) + } + + for _, ip := range pool.IPs { + fmt.Printf("\tIP Address: %s\n", ip.Address) + } +} + +func (cmd *poolShowCommand) run(args []string) error { + var pool types.Pool + + if cmd.name == "" { + errorf("Missing required -name parameter") + cmd.usage() + } + + pool, err := getCiaoPool(cmd.name) + if err != nil { + fatalf(err.Error()) + } + + if cmd.template != "" { + return outputToTemplate("pool-show", cmd.template, + &pool) + } + + dumpPool(pool) + + return nil +} + +type poolDeleteCommand struct { + Flag flag.FlagSet + name string +} + +func (cmd *poolDeleteCommand) usage(...string) { + fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] pool delete [flags] + +Delete an unused ciao external IP pool. + +The delete flags are: + +`) + cmd.Flag.PrintDefaults() + os.Exit(2) +} + +func (cmd *poolDeleteCommand) parseArgs(args []string) []string { + cmd.Flag.StringVar(&cmd.name, "name", "", "Name of pool") + cmd.Flag.Usage = func() { cmd.usage() } + cmd.Flag.Parse(args) + return cmd.Flag.Args() +} + +func (cmd *poolDeleteCommand) run(args []string) error { + if cmd.name == "" { + errorf("Missing required -name parameter") + cmd.usage() + } + + url, err := getCiaoPoolRef(cmd.name) + if err != nil { + fatalf(err.Error()) + } + + ver := api.PoolsV1 + + resp, err := sendCiaoRequest("DELETE", url, nil, nil, &ver) + if err != nil { + fatalf(err.Error()) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent { + fatalf("Pool deletion failed: %s", resp.Status) + } + + fmt.Printf("Deleted pool: %s\n", cmd.name) + + return nil +} + +type poolAddCommand struct { + Flag flag.FlagSet + name string + subnet string + ips []string +} + +func (cmd *poolAddCommand) usage(...string) { + fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] pool add [flags] [ip1 ip2...] + +Add external IPs to a pool. + +The add flags are: + +`) + cmd.Flag.PrintDefaults() + os.Exit(2) +} + +func (cmd *poolAddCommand) parseArgs(args []string) []string { + cmd.Flag.StringVar(&cmd.name, "name", "", "Name of pool") + cmd.Flag.StringVar(&cmd.subnet, "subnet", "", "Subnet in CIDR format") + cmd.Flag.Usage = func() { cmd.usage() } + cmd.Flag.Parse(args) + return cmd.Flag.Args() +} + +func (cmd *poolAddCommand) run(args []string) error { + var req types.NewAddressRequest + + if cmd.name == "" { + errorf("Missing required -name parameter") + cmd.usage() + } + + url, err := getCiaoPoolRef(cmd.name) + if err != nil { + fatalf(err.Error()) + } + + if cmd.subnet != "" { + // verify it's a good address. + _, _, err := net.ParseCIDR(cmd.subnet) + if err != nil { + fatalf(err.Error()) + } + + req.Subnet = &cmd.subnet + } else if len(args) < 1 { + errorf("Missing any addresses to add") + cmd.usage() + } else { + for _, addr := range args { + // verify it's a good address + IP := net.ParseIP(addr) + if IP == nil { + fatalf("Invalid IP address") + } + + newAddr := types.NewIPAddressRequest{ + IP: addr, + } + + req.IPs = append(req.IPs, newAddr) + } + } + + b, err := json.Marshal(req) + if err != nil { + fatalf(err.Error()) + } + + body := bytes.NewReader(b) + + ver := api.PoolsV1 + + resp, err := sendCiaoRequest("POST", url, nil, body, &ver) + if err != nil { + fatalf(err.Error()) + } + + if resp.StatusCode != http.StatusNoContent { + fatalf("Adding address failed: %s", resp.Status) + } + + fmt.Printf("Added new address to: %s\n", cmd.name) + + return nil +} + +type poolRemoveCommand struct { + Flag flag.FlagSet + name string + subnet string + ip string +} + +func (cmd *poolRemoveCommand) usage(...string) { + fmt.Fprintf(os.Stderr, `usage: ciao-cli [options] pool remove [flags] + +Remove unmapped external IPs from a pool. + +The remove flags are: + +`) + cmd.Flag.PrintDefaults() + os.Exit(2) +} + +func (cmd *poolRemoveCommand) parseArgs(args []string) []string { + cmd.Flag.StringVar(&cmd.name, "name", "", "Name of pool") + cmd.Flag.StringVar(&cmd.subnet, "subnet", "", "Subnet in CIDR format") + cmd.Flag.StringVar(&cmd.ip, "ip", "", "IPv4 Address") + cmd.Flag.Usage = func() { cmd.usage() } + cmd.Flag.Parse(args) + return cmd.Flag.Args() +} + +func getSubnetRef(pool types.Pool, cidr string) string { + for _, sub := range pool.Subnets { + if sub.CIDR == cidr { + return getRef("self", sub.Links) + } + } + + return "" +} + +func getIPRef(pool types.Pool, address string) string { + for _, ip := range pool.IPs { + if ip.Address == address { + return getRef("self", ip.Links) + } + } + + return "" +} + +func (cmd *poolRemoveCommand) run(args []string) error { + if cmd.name == "" { + errorf("Missing required -name parameter") + cmd.usage() + } + + if cmd.subnet == "" && cmd.ip == "" { + errorf("You must specify subnet or ip address to remove") + cmd.usage() + } + + if cmd.subnet != "" && cmd.ip != "" { + errorf("You can only remove one item at a time") + cmd.usage() + } + + pool, err := getCiaoPool(cmd.name) + if err != nil { + fatalf(err.Error()) + } + + var url string + + if cmd.subnet != "" { + url = getSubnetRef(pool, cmd.subnet) + } + + if cmd.ip != "" { + url = getIPRef(pool, cmd.ip) + } + + if url == "" { + fatalf("Address not present") + } + + ver := api.PoolsV1 + + resp, err := sendCiaoRequest("DELETE", url, nil, nil, &ver) + if err != nil { + fatalf(err.Error()) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent { + fatalf("Address removal failed: %s", resp.Status) + } + + fmt.Printf("Removed address from pool: %s\n", cmd.name) + return nil +} diff --git a/ciao-cli/identity.go b/ciao-cli/identity.go index bc4af7102..2284cb724 100644 --- a/ciao-cli/identity.go +++ b/ciao-cli/identity.go @@ -181,7 +181,7 @@ func getUserProjects(username string, password string) ([]Project, error) { identity := fmt.Sprintf("%s/v3/users/%s/projects", *identityURL, user) - resp, err := sendHTTPRequestToken("GET", identity, nil, token, nil) + resp, err := sendHTTPRequestToken("GET", identity, nil, token, nil, nil) if err != nil { return nil, err } @@ -233,7 +233,7 @@ func getAllProjects(username string, password string) (*IdentityProjects, error) identity := fmt.Sprintf("%s/v3/auth/projects", *identityURL) - resp, err := sendHTTPRequestToken("GET", identity, nil, token, nil) + resp, err := sendHTTPRequestToken("GET", identity, nil, token, nil, nil) if err != nil { return nil, err } diff --git a/ciao-cli/main.go b/ciao-cli/main.go index 6c37ea975..74b0eb30d 100644 --- a/ciao-cli/main.go +++ b/ciao-cli/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" + "errors" "flag" "fmt" "io" @@ -30,6 +31,8 @@ import ( "strconv" "text/template" + "github.com/01org/ciao/ciao-controller/api" + "github.com/01org/ciao/ciao-controller/types" "github.com/golang/glog" ) @@ -81,14 +84,16 @@ type subCommand interface { } var commands = map[string]subCommand{ - "instance": instanceCommand, - "workload": workloadCommand, - "tenant": tenantCommand, - "event": eventCommand, - "node": nodeCommand, - "trace": traceCommand, - "image": imageCommand, - "volume": volumeCommand, + "instance": instanceCommand, + "workload": workloadCommand, + "tenant": tenantCommand, + "event": eventCommand, + "node": nodeCommand, + "trace": traceCommand, + "image": imageCommand, + "volume": volumeCommand, + "pool": poolCommand, + "external-ip": externalIPCommand, } var scopedToken string @@ -135,6 +140,7 @@ var ( tenantID = flag.String("tenant-id", "", "Tenant UUID") tenantName = flag.String("tenant-name", "", "Tenant name") computePort = flag.Int("computeport", openstackComputePort, "Openstack Compute API port") + ciaoPort = flag.Int("ciaoport", api.Port, "ciao API port") caCertFile = flag.String("ca-file", "", "CA Certificate") ) @@ -176,7 +182,12 @@ func buildComputeURL(format string, args ...interface{}) string { return fmt.Sprintf(prefix+format, args...) } -func sendHTTPRequestToken(method string, url string, values []queryValue, token string, body io.Reader) (*http.Response, error) { +func buildCiaoURL(format string, args ...interface{}) string { + prefix := fmt.Sprintf("https://%s:%d/", *controllerURL, *ciaoPort) + return fmt.Sprintf(prefix+format, args...) +} + +func sendHTTPRequestToken(method string, url string, values []queryValue, token string, body io.Reader, content *string) (*http.Response, error) { req, err := http.NewRequest(method, os.ExpandEnv(url), body) if err != nil { return nil, err @@ -199,7 +210,11 @@ func sendHTTPRequestToken(method string, url string, values []queryValue, token req.Header.Add("X-Auth-Token", token) } - if body != nil { + if content != nil { + contentType := fmt.Sprintf("application/%s", *content) + req.Header.Set("Content-Type", contentType) + req.Header.Set("Accept", contentType) + } else if body != nil { req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") } @@ -238,7 +253,7 @@ func sendHTTPRequestToken(method string, url string, values []queryValue, token } func sendHTTPRequest(method string, url string, values []queryValue, body io.Reader) (*http.Response, error) { - return sendHTTPRequestToken(method, url, values, scopedToken, body) + return sendHTTPRequestToken(method, url, values, scopedToken, body, nil) } func unmarshalHTTPResponse(resp *http.Response, v interface{}) error { @@ -263,6 +278,56 @@ func unmarshalHTTPResponse(resp *http.Response, v interface{}) error { return nil } +func sendCiaoRequest(method string, url string, values []queryValue, body io.Reader, content *string) (*http.Response, error) { + return sendHTTPRequestToken(method, url, values, scopedToken, body, content) +} + +func getRef(rel string, links []types.Link) string { + for _, link := range links { + if link.Rel == rel { + return link.Href + } + } + return "" +} + +func getCiaoResource(name string, minVersion string) (string, error) { + var resources []types.APILink + var url string + + if checkPrivilege() { + url = buildCiaoURL("") + } else { + url = buildCiaoURL(fmt.Sprintf("%s", *tenantID)) + } + + resp, err := sendCiaoRequest("GET", url, nil, nil, nil) + if err != nil { + return "", err + } + + err = unmarshalHTTPResponse(resp, &resources) + if err != nil { + return "", err + } + + for _, l := range resources { + if l.Rel == name && l.MinVersion == minVersion { + return l.Href, nil + } + } + + return "", errors.New("Supported version of resource not found") +} + +func checkPrivilege() bool { + if *tenantName == "admin" { + return true + } + + return false +} + func limitToString(limit int) string { if limit == -1 { return "Unlimited" diff --git a/ciao-controller/api/api.go b/ciao-controller/api/api.go new file mode 100644 index 000000000..8d3a3d55a --- /dev/null +++ b/ciao-controller/api/api.go @@ -0,0 +1,516 @@ +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/01org/ciao/ciao-controller/types" + "github.com/golang/glog" + "github.com/gorilla/mux" +) + +// Port is the default port number for the ciao API. +const Port = 8889 + +const ( + // PoolsV1 is the content-type string for v1 of our pools resource + PoolsV1 = "x.ciao.pools.v1" + + // ExternalIPsV1 is the content-type string for v1 of our external-ips resource + ExternalIPsV1 = "x.ciao.external-ips.v1" +) + +// HTTPErrorData represents the HTTP response body for +// a compute API request error. +type HTTPErrorData struct { + Code int `json:"code"` + Name string `json:"name"` + Message string `json:"message"` +} + +// HTTPReturnErrorCode represents the unmarshalled version for Return codes +// when a API call is made and you need to return explicit data of +// the call as OpenStack format +// http://developer.openstack.org/api-guide/compute/faults.html +type HTTPReturnErrorCode struct { + Error HTTPErrorData `json:"error"` +} + +// Response contains the http status and any response struct to be marshalled. +type Response struct { + status int + response interface{} +} + +func errorResponse(err error) Response { + switch err { + case types.ErrPoolNotFound, + types.ErrTenantNotFound, + types.ErrAddressNotFound, + types.ErrInstanceNotFound: + return Response{http.StatusNotFound, nil} + + case types.ErrQuota, + types.ErrInstanceNotAssigned, + types.ErrDuplicateSubnet, + types.ErrDuplicateIP, + types.ErrInvalidIP, + types.ErrPoolNotEmpty, + types.ErrInvalidPoolAddress, + types.ErrBadRequest, + types.ErrPoolEmpty, + types.ErrDuplicatePoolName: + return Response{http.StatusForbidden, nil} + + default: + return Response{http.StatusInternalServerError, nil} + } +} + +// Handler is a custom handler for the compute APIs. +// This custom handler allows us to more cleanly return an error and response, +// and pass some package level context into the handler. +type Handler struct { + *Context + Handler func(*Context, http.ResponseWriter, *http.Request) (Response, error) +} + +func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // set the content type to whatever was requested. + contentType := r.Header.Get("Content-Type") + + resp, err := h.Handler(h.Context, w, r) + if err != nil { + data := HTTPErrorData{ + Code: resp.status, + Name: http.StatusText(resp.status), + Message: err.Error(), + } + + code := HTTPReturnErrorCode{ + Error: data, + } + + b, err := json.Marshal(code) + if err != nil { + http.Error(w, http.StatusText(resp.status), resp.status) + } + + http.Error(w, string(b), resp.status) + } + + b, err := json.Marshal(resp.response) + if err != nil { + http.Error(w, http.StatusText(http.StatusInternalServerError), + http.StatusInternalServerError) + } + + w.Header().Set("Content-Type", contentType) + w.WriteHeader(resp.status) + w.Write(b) +} + +func listResources(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + var links []types.APILink + vars := mux.Vars(r) + tenantID, ok := vars["tenant"] + + // we support the "pools" resource. + link := types.APILink{ + Rel: "pools", + Version: PoolsV1, + MinVersion: PoolsV1, + } + + if !ok { + link.Href = fmt.Sprintf("%s/pools", c.URL) + } else { + link.Href = fmt.Sprintf("%s/%s/pools", c.URL, tenantID) + } + + links = append(links, link) + + // we support the "external-ips" resource + link = types.APILink{ + Rel: "external-ips", + Version: ExternalIPsV1, + MinVersion: ExternalIPsV1, + } + + if !ok { + link.Href = fmt.Sprintf("%s/external-ips", c.URL) + } else { + link.Href = fmt.Sprintf("%s/%s/external-ips", c.URL, tenantID) + } + + links = append(links, link) + + return Response{http.StatusOK, links}, nil +} + +func dumpPool(pool types.Pool) { + glog.V(2).Info("Pool") + glog.V(2).Info("-----------------------") + glog.V(2).Infof("ID: %s\n", pool.ID) + glog.V(2).Infof("Name: %s\n", pool.Name) + glog.V(2).Infof("TotalIPs: %d\n", pool.TotalIPs) + glog.V(2).Infof("Free: %d\n", pool.Free) + glog.V(2).Infof("Links: %v\n", pool.Links) + glog.V(2).Infof("Subnets:\n") + + for _, sub := range pool.Subnets { + glog.V(2).Infof("%v", sub) + } +} + +func showPool(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + vars := mux.Vars(r) + ID := vars["pool"] + + pool, err := c.ShowPool(ID) + if err != nil { + return errorResponse(err), err + } + + dumpPool(pool) + + return Response{http.StatusOK, pool}, nil +} + +func listPools(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + var resp types.ListPoolsResponse + vars := mux.Vars(r) + _, ok := vars["tenant"] + + pools, err := c.ListPools() + if err != nil { + return errorResponse(err), err + } + + queries := r.URL.Query() + + names, returnNamedPool := queries["name"] + + for i, p := range pools { + dumpPool(p) + + var match bool + + if returnNamedPool == true { + for _, name := range names { + if name == p.Name { + match = true + } + } + } else { + match = true + } + + if match { + summary := types.PoolSummary{ + ID: p.ID, + Name: p.Name, + } + + if !ok { + summary.TotalIPs = &pools[i].TotalIPs + summary.Free = &pools[i].Free + summary.Links = pools[i].Links + } + + resp.Pools = append(resp.Pools, summary) + } + } + + return Response{http.StatusOK, resp}, err +} + +func addPool(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + var req types.NewPoolRequest + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return errorResponse(err), err + } + + err = json.Unmarshal(body, &req) + if err != nil { + return errorResponse(err), err + } + + var ips []string + + for _, ip := range req.IPs { + ips = append(ips, ip.IP) + } + + _, err = c.AddPool(req.Name, req.Subnet, ips) + if err != nil { + return errorResponse(err), err + } + + return Response{http.StatusNoContent, nil}, nil +} + +func deletePool(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + vars := mux.Vars(r) + ID := vars["pool"] + + err := c.DeletePool(ID) + if err != nil { + return errorResponse(err), err + } + + return Response{http.StatusNoContent, nil}, nil +} + +func addToPool(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + vars := mux.Vars(r) + ID := vars["pool"] + + var req types.NewAddressRequest + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return errorResponse(err), err + } + + err = json.Unmarshal(body, &req) + if err != nil { + return errorResponse(err), err + } + + var ips []string + + for _, ip := range req.IPs { + ips = append(ips, ip.IP) + } + + err = c.AddAddress(ID, req.Subnet, ips) + if err != nil { + return errorResponse(err), err + } + + return Response{http.StatusNoContent, nil}, nil +} + +func deleteSubnet(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + vars := mux.Vars(r) + poolID := vars["pool"] + subnetID := vars["subnet"] + + err := c.RemoveAddress(poolID, &subnetID, nil) + if err != nil { + return errorResponse(err), err + } + + return Response{http.StatusNoContent, nil}, nil +} + +func deleteExternalIP(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + vars := mux.Vars(r) + poolID := vars["pool"] + IPID := vars["ip_id"] + + err := c.RemoveAddress(poolID, nil, &IPID) + if err != nil { + return errorResponse(err), err + } + + return Response{http.StatusNoContent, nil}, nil +} + +func listMappedIPs(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + vars := mux.Vars(r) + tenantID, ok := vars["tenant"] + var IPs []types.MappedIP + var short []types.MappedIPShort + + if !ok { + IPs = c.ListMappedAddresses(nil) + return Response{http.StatusOK, IPs}, nil + } + + IPs = c.ListMappedAddresses(&tenantID) + for _, IP := range IPs { + s := types.MappedIPShort{ + ID: IP.ID, + ExternalIP: IP.ExternalIP, + InternalIP: IP.InternalIP, + Links: IP.Links, + } + short = append(short, s) + } + + return Response{http.StatusOK, short}, nil +} + +func mapExternalIP(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + var req types.MapIPRequest + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return errorResponse(err), err + } + + err = json.Unmarshal(body, &req) + if err != nil { + return errorResponse(err), err + } + + err = c.MapAddress(req.PoolName, req.InstanceID) + if err != nil { + return errorResponse(err), err + } + + return Response{http.StatusNoContent, nil}, nil +} + +func unmapExternalIP(c *Context, w http.ResponseWriter, r *http.Request) (Response, error) { + vars := mux.Vars(r) + tenantID, ok := vars["tenant"] + mappingID := vars["mapping_id"] + + var IPs []types.MappedIP + + if !ok { + IPs = c.ListMappedAddresses(nil) + } else { + IPs = c.ListMappedAddresses(&tenantID) + } + + for _, m := range IPs { + if m.ID == mappingID { + err := c.UnMapAddress(m.ExternalIP) + if err != nil { + return errorResponse(err), err + } + + return Response{http.StatusAccepted, nil}, nil + } + } + + return errorResponse(types.ErrAddressNotFound), types.ErrAddressNotFound +} + +// Service is an interface which must be implemented by the ciao API context. +type Service interface { + AddPool(name string, subnet *string, ips []string) (types.Pool, error) + ListPools() ([]types.Pool, error) + ShowPool(id string) (types.Pool, error) + DeletePool(id string) error + AddAddress(poolID string, subnet *string, IPs []string) error + RemoveAddress(poolID string, subnetID *string, IPID *string) error + ListMappedAddresses(tenantID *string) []types.MappedIP + MapAddress(poolName *string, instanceID string) error + UnMapAddress(ID string) error +} + +// Context is used to provide the services and current URL to the handlers. +type Context struct { + URL string + Service +} + +// Config is used to setup the Context for the ciao API. +type Config struct { + URL string + CiaoService Service +} + +// Routes returns the supported ciao API endpoints. +// A plain application/json request will return v1 of the resource +// since we only have one version of this api so far, that means +// most routes will match both json as well as our custom +// content type. +func Routes(config Config) *mux.Router { + // make new Context + context := &Context{config.URL, config.CiaoService} + + r := mux.NewRouter() + + // external IP pools + route := r.Handle("/", Handler{context, listResources}) + route.Methods("GET") + + route = r.Handle("/{tenant:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}", Handler{context, listResources}) + route.Methods("GET") + + matchContent := fmt.Sprintf("application/(%s|json)", PoolsV1) + + route = r.Handle("/pools", Handler{context, listPools}) + route.Methods("GET") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/{tenant:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}/pools", Handler{context, listPools}) + route.Methods("GET") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/pools", Handler{context, addPool}) + route.Methods("POST") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/pools/{pool:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}", Handler{context, showPool}) + route.Methods("GET") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/pools/{pool:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}", Handler{context, deletePool}) + route.Methods("DELETE") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/pools/{pool:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}", Handler{context, addToPool}) + route.Methods("POST") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/pools/{pool:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}/subnets/{subnet:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}", Handler{context, deleteSubnet}) + route.Methods("DELETE") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/pools/{pool:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}/external-ips/{ip_id:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}", Handler{context, deleteExternalIP}) + route.Methods("DELETE") + route.HeadersRegexp("Content-Type", matchContent) + + // mapped external IPs + matchContent = fmt.Sprintf("application/(%s|json)", ExternalIPsV1) + + route = r.Handle("/external-ips", Handler{context, listMappedIPs}) + route.Methods("GET") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/{tenant:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}/external-ips", Handler{context, listMappedIPs}) + route.Methods("GET") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/external-ips", Handler{context, mapExternalIP}) + route.Methods("POST") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/{tenant:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}/external-ips", Handler{context, mapExternalIP}) + route.Methods("POST") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/external-ips/{mapping_id:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}", Handler{context, unmapExternalIP}) + route.Methods("DELETE") + route.HeadersRegexp("Content-Type", matchContent) + + route = r.Handle("/{tenant:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}/external-ips/{mapping_id:[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[8|9|aA|bB][a-f0-9]{3}-?[a-f0-9]{12}}", Handler{context, unmapExternalIP}) + route.Methods("DELETE") + route.HeadersRegexp("Content-Type", matchContent) + return r +} diff --git a/ciao-controller/api/api_test.go b/ciao-controller/api/api_test.go new file mode 100644 index 000000000..6eaae8773 --- /dev/null +++ b/ciao-controller/api/api_test.go @@ -0,0 +1,285 @@ +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/01org/ciao/ciao-controller/types" +) + +type test struct { + method string + pattern string + handler func(*Context, http.ResponseWriter, *http.Request) (Response, error) + request string + media string + expectedStatus int + expectedResponse string +} + +func myHostname() string { + host, _ := os.Hostname() + return host +} + +var tests = []test{ + { + "GET", + "/", + listResources, + "", + "application/text", + http.StatusOK, + `[{"rel":"pools","href":"/pools","version":"x.ciao.pools.v1","minimum_version":"x.ciao.pools.v1"},{"rel":"external-ips","href":"/external-ips","version":"x.ciao.external-ips.v1","minimum_version":"x.ciao.external-ips.v1"}]`, + }, + { + "GET", + "/pools", + listPools, + "", + "application/x.ciao.v1.pools", + http.StatusOK, + `{"pools":[{"id":"validID","name":"testpool","free":0,"total_ips":0,"links":[{"rel":"self","href":"/pools/validID"}]}]}`, + }, + { + "GET", + "/pools?name=testpool", + listPools, + "", + "application/x.ciao.v1.pools", + http.StatusOK, + `{"pools":[{"id":"validID","name":"testpool","free":0,"total_ips":0,"links":[{"rel":"self","href":"/pools/validID"}]}]}`, + }, + { + "POST", + "/pools", + addPool, + `{"name":"testpool"}`, + "application/x.ciao.v1.pools", + http.StatusNoContent, + "null", + }, + { + "GET", + "/pools/validID", + showPool, + "", + "application/x.ciao.v1.pools", + http.StatusOK, + `{"id":"validID","name":"testpool","free":0,"total_ips":0,"links":[{"rel":"self","href":"/pools/validID"}],"subnets":[],"ips":[]}`, + }, + { + "DELETE", + "/pools/validID", + deletePool, + "", + "application/x.ciao.v1.pools", + http.StatusNoContent, + "null", + }, + { + "POST", + "/pools/validID", + addToPool, + `{"subnet":"192.168.0.0/24"}`, + "application/x.ciao.v1.pools", + http.StatusNoContent, + "null", + }, + { + "DELETE", + "/pools/validID/subnets/validID", + deleteSubnet, + "", + "application/x.ciao.v1.pools", + http.StatusNoContent, + "null", + }, + { + "DELETE", + "/pools/validID/external-ips/validID", + deleteExternalIP, + "", + "application/x.ciao.v1.pools", + http.StatusNoContent, + "null", + }, + { + "GET", + "/external-ips", + listMappedIPs, + "", + ExternalIPsV1, + http.StatusOK, + `[{"mapping_id":"validID","external_ip":"192.168.0.1","internal_ip":"172.16.0.1","instance_id":"","tenant_id":"validtenant","pool_id":"validpool","pool_name":"mypool","links":[{"rel":"self","href":"/external-ips/validID"},{"rel":"pool","href":"/pools/validpool"}]}]`, + }, + { + "POST", + "/validID/external-ips", + mapExternalIP, + `{"pool_name":"apool","instance_id":"validinstanceID"}`, + "application/x.ciao.v1.pools", + http.StatusNoContent, + "null", + }, +} + +type testCiaoService struct{} + +func (ts testCiaoService) ListPools() ([]types.Pool, error) { + self := types.Link{ + Rel: "self", + Href: "/pools/validID", + } + + resp := types.Pool{ + ID: "validID", + Name: "testpool", + Free: 0, + TotalIPs: 0, + Subnets: []types.ExternalSubnet{}, + IPs: []types.ExternalIP{}, + Links: []types.Link{self}, + } + + return []types.Pool{resp}, nil +} + +func (ts testCiaoService) AddPool(name string, subnet *string, ips []string) (types.Pool, error) { + return types.Pool{}, nil +} + +func (ts testCiaoService) ShowPool(id string) (types.Pool, error) { + self := types.Link{ + Rel: "self", + Href: "/pools/validID", + } + + resp := types.Pool{ + ID: "validID", + Name: "testpool", + Free: 0, + TotalIPs: 0, + Subnets: []types.ExternalSubnet{}, + IPs: []types.ExternalIP{}, + Links: []types.Link{self}, + } + + return resp, nil +} + +func (ts testCiaoService) DeletePool(id string) error { + return nil +} + +func (ts testCiaoService) AddAddress(poolID string, subnet *string, ips []string) error { + return nil +} + +func (ts testCiaoService) RemoveAddress(poolID string, subnet *string, extIP *string) error { + return nil +} + +func (ts testCiaoService) ListMappedAddresses(tenant *string) []types.MappedIP { + var ref string + + m := types.MappedIP{ + ID: "validID", + ExternalIP: "192.168.0.1", + InternalIP: "172.16.0.1", + TenantID: "validtenant", + PoolID: "validpool", + PoolName: "mypool", + } + + if tenant != nil { + ref = fmt.Sprintf("%s/external-ips/%s", *tenant, m.ID) + } else { + ref = fmt.Sprintf("/external-ips/%s", m.ID) + } + + link := types.Link{ + Rel: "self", + Href: ref, + } + + m.Links = []types.Link{link} + + if tenant == nil { + ref := fmt.Sprintf("/pools/%s", m.PoolID) + + link := types.Link{ + Rel: "pool", + Href: ref, + } + + m.Links = append(m.Links, link) + } + + return []types.MappedIP{m} +} + +func (ts testCiaoService) MapAddress(name *string, instanceID string) error { + return nil +} + +func (ts testCiaoService) UnMapAddress(string) error { + return nil +} + +func TestResponse(t *testing.T) { + var ts testCiaoService + + context := &Context{"", ts} + + for _, tt := range tests { + req, err := http.NewRequest(tt.method, tt.pattern, bytes.NewBuffer([]byte(tt.request))) + if err != nil { + t.Fatal(err) + } + + req.Header.Set("Content-Type", tt.media) + + rr := httptest.NewRecorder() + handler := Handler{context, tt.handler} + + handler.ServeHTTP(rr, req) + + status := rr.Code + if status != tt.expectedStatus { + t.Errorf("got %v, expected %v", status, tt.expectedStatus) + } + + if rr.Body.String() != tt.expectedResponse { + t.Errorf("%s: failed\ngot: %v\nexp: %v", tt.pattern, rr.Body.String(), tt.expectedResponse) + } + } +} + +func TestRoutes(t *testing.T) { + var ts testCiaoService + config := Config{"", ts} + + r := Routes(config) + if r == nil { + t.Fatalf("No routes returned") + } +} diff --git a/ciao-controller/client.go b/ciao-controller/client.go index 1fd243cef..13412fc8e 100644 --- a/ciao-controller/client.go +++ b/ciao-controller/client.go @@ -17,8 +17,10 @@ package main import ( + "fmt" "time" + "github.com/01org/ciao/ciao-controller/types" "github.com/01org/ciao/payloads" "github.com/01org/ciao/ssntp" "github.com/golang/glog" @@ -82,10 +84,68 @@ func (client *ssntpClient) deleteEphemeralStorage(instanceID string) { } } +func (client *ssntpClient) unassignEvent(payload []byte) { + var event payloads.EventPublicIPUnassigned + err := yaml.Unmarshal(payload, &event) + if err != nil { + glog.Warning(err) + return + } + + i, err := client.ctl.ds.GetInstance(event.UnassignedIP.InstanceUUID) + if err != nil { + glog.Warning(err) + return + } + + err = client.ctl.ds.UnMapExternalIP(event.UnassignedIP.PublicIP) + if err != nil { + glog.Warning(err) + return + } + + msg := fmt.Sprintf("Unmapped %s from %s", event.UnassignedIP.PublicIP, event.UnassignedIP.PrivateIP) + client.ctl.ds.LogEvent(i.TenantID, msg) +} + +func (client *ssntpClient) assignEvent(payload []byte) { + var event payloads.EventPublicIPAssigned + err := yaml.Unmarshal(payload, &event) + if err != nil { + glog.Warning(err) + return + } + + i, err := client.ctl.ds.GetInstance(event.AssignedIP.InstanceUUID) + if err != nil { + glog.Warning(err) + return + } + + msg := fmt.Sprintf("Mapped %s to %s", event.AssignedIP.PublicIP, event.AssignedIP.PrivateIP) + client.ctl.ds.LogEvent(i.TenantID, msg) +} + +func (client *ssntpClient) unassignError(payload []byte) { + var failure payloads.ErrorPublicIPFailure + err := yaml.Unmarshal(payload, &failure) + if err != nil { + glog.Warning("Error unmarshalling ErrorPublicIPFailure") + return + } + + // we can't unmap the IP - all we can do is log. + msg := fmt.Sprintf("Failed to unmap %s from %s: %s", failure.PublicIP, failure.InstanceUUID, failure.Reason.String()) + client.ctl.ds.LogEvent(failure.TenantUUID, msg) +} + func (client *ssntpClient) EventNotify(event ssntp.Event, frame *ssntp.Frame) { payload := frame.Payload glog.Info("EVENT ", event, " for ", client.name) + + glog.V(1).Info(string(payload)) + switch event { case ssntp.InstanceDeleted: var event payloads.EventInstanceDeleted @@ -134,14 +194,21 @@ func (client *ssntpClient) EventNotify(event ssntp.Event, frame *ssntp.Frame) { glog.Infof("Node %s disconnected", nodeDisconnected.Disconnected.NodeUUID) client.ctl.ds.DeleteNode(nodeDisconnected.Disconnected.NodeUUID) + case ssntp.PublicIPAssigned: + client.assignEvent(payload) + + case ssntp.PublicIPUnassigned: + client.unassignEvent(payload) + } - glog.V(1).Info(string(payload)) } func (client *ssntpClient) ErrorNotify(err ssntp.Error, frame *ssntp.Frame) { payload := frame.Payload glog.Info("ERROR (", err, ") for ", client.name) + glog.V(1).Info(string(payload)) + switch err { case ssntp.StartFailure: var failure payloads.ErrorStartFailure @@ -185,8 +252,26 @@ func (client *ssntpClient) ErrorNotify(err ssntp.Error, frame *ssntp.Frame) { } client.ctl.ds.DetachVolumeFailure(failure.InstanceUUID, failure.VolumeUUID, failure.Reason) + case ssntp.AssignPublicIPFailure: + var failure payloads.ErrorPublicIPFailure + err := yaml.Unmarshal(payload, &failure) + if err != nil { + glog.Warning("Error unmarshalling ErrorPublicIPFailure") + return + } + + err = client.ctl.ds.UnMapExternalIP(failure.PublicIP) + if err != nil { + glog.Warning(err) + } + + msg := fmt.Sprintf("Failed to map %s to %s: %s", failure.PublicIP, failure.InstanceUUID, failure.Reason.String()) + client.ctl.ds.LogEvent(failure.TenantUUID, msg) + + case ssntp.UnassignPublicIPFailure: + client.unassignError(payload) + } - glog.V(1).Info(string(payload)) } func newSSNTPClient(ctl *controller, config *ssntp.Config) (*ssntpClient, error) { @@ -358,3 +443,51 @@ func (client *ssntpClient) detachVolume(volID string, instanceID string, nodeID func (client *ssntpClient) Disconnect() { client.ssntp.Close() } + +func (client *ssntpClient) mapExternalIP(t types.Tenant, m types.MappedIP) error { + payload := payloads.CommandAssignPublicIP{ + AssignIP: payloads.PublicIPCommand{ + ConcentratorUUID: t.CNCIID, + TenantUUID: m.TenantID, + InstanceUUID: m.InstanceID, + PublicIP: m.ExternalIP, + PrivateIP: m.InternalIP, + VnicMAC: t.CNCIMAC, + }, + } + + y, err := yaml.Marshal(payload) + if err != nil { + return err + } + + glog.Infof("Request Map of %s to %s\n", m.ExternalIP, m.InternalIP) + glog.V(1).Info(string(y)) + + _, err = client.ssntp.SendCommand(ssntp.AssignPublicIP, y) + return err +} + +func (client *ssntpClient) unMapExternalIP(t types.Tenant, m types.MappedIP) error { + payload := payloads.CommandReleasePublicIP{ + ReleaseIP: payloads.PublicIPCommand{ + ConcentratorUUID: t.CNCIID, + TenantUUID: m.TenantID, + InstanceUUID: m.InstanceID, + PublicIP: m.ExternalIP, + PrivateIP: m.InternalIP, + VnicMAC: t.CNCIMAC, + }, + } + + y, err := yaml.Marshal(payload) + if err != nil { + return err + } + + glog.Infof("Request unmap of %s from %s\n", m.ExternalIP, m.InternalIP) + glog.V(1).Info(string(y)) + + _, err = client.ssntp.SendCommand(ssntp.ReleasePublicIP, y) + return err +} diff --git a/ciao-controller/controller_test.go b/ciao-controller/controller_test.go index 962f2805d..2bd0c42c3 100644 --- a/ciao-controller/controller_test.go +++ b/ciao-controller/controller_test.go @@ -23,6 +23,7 @@ import ( "net" "os" "path/filepath" + "reflect" "testing" "time" @@ -1235,6 +1236,8 @@ func TestStorageConfig(t *testing.T) { if err != nil { t.Fatal(err) } + + wls[0].Storage = nil } func createTestVolume(tenantID string, size int, t *testing.T) string { @@ -1370,6 +1373,412 @@ func TestListVolumesDetail(t *testing.T) { } } +func testAddPool(t *testing.T, name string, subnet *string, ips []string) { + pool, err := ctl.AddPool(name, subnet, ips) + if err != nil { + t.Fatal(err) + } + + if pool.ID == "" { + t.Fatal("id not set") + } + + expected := types.Pool{ + ID: pool.ID, + Name: name, + } + + if subnet != nil { + if pool.Subnets[0].ID == "" { + t.Fatal("subnet id not created") + } + + sub := types.ExternalSubnet{ + ID: pool.Subnets[0].ID, + CIDR: *subnet, + } + + expected.Subnets = []types.ExternalSubnet{sub} + + _, ipNet, err := net.ParseCIDR(*subnet) + if err != nil { + t.Fatal(err) + } + + ones, bits := ipNet.Mask.Size() + expected.TotalIPs = (1 << uint32(bits-ones)) - 2 + expected.Free = expected.TotalIPs + } else if len(ips) > 0 { + // not an easy way to check this, so we're going to + // do some manual tests + if pool.TotalIPs != len(ips) || + pool.Free != len(ips) || + len(pool.IPs) != len(ips) { + t.Fatal("External IPs not handled correctly") + } + return + } + + if reflect.DeepEqual(expected, pool) == false { + t.Fatalf("expected %v, got %v\n", expected, pool) + } +} + +func deletePool(name string) error { + pools, err := ctl.ListPools() + if err != nil { + return err + } + + if len(pools) < 1 { + return types.ErrPoolNotFound + } + + for _, pool := range pools { + if pool.Name == name { + return ctl.DeletePool(pool.ID) + } + } + + return types.ErrPoolNotFound +} + +func TestAddPoolWithSubnet(t *testing.T) { + subnet := "192.168.0.0/16" + testAddPool(t, "test1", &subnet, []string{}) + deletePool("test1") +} + +func TestAddPoolWithIPs(t *testing.T) { + ips := []string{"10.10.0.1", "10.10.0.2"} + testAddPool(t, "test2", nil, ips) + deletePool("test2") +} + +func TestAddPool(t *testing.T) { + testAddPool(t, "test3", nil, []string{}) + deletePool("test3") +} + +func TestListPools(t *testing.T) { + testAddPool(t, "listPoolTest", nil, []string{}) + + pools, err := ctl.ListPools() + if err != nil { + t.Fatal(err) + } + + if len(pools) < 1 { + t.Fatal("Unable to retrieve pools") + } + + for _, pool := range pools { + if pool.Name == "listPoolTest" { + err := ctl.DeletePool(pool.ID) + if err != nil { + t.Fatal(err) + } + return + } + } + + t.Fatal("Could not list pools") +} + +func TestShowPool(t *testing.T) { + testAddPool(t, "showPoolTest", nil, []string{}) + + pools, err := ctl.ListPools() + if err != nil { + t.Fatal(err) + } + + if len(pools) < 1 { + t.Fatal("Unable to retrieve pools") + } + + for _, pool := range pools { + if pool.Name == "showPoolTest" { + _, err := ctl.ShowPool(pool.ID) + if err != nil { + t.Fatal(err) + } + + err = ctl.DeletePool(pool.ID) + if err != nil { + t.Fatal(err) + } + + return + } + } + + t.Fatal("Could not show pool") +} + +func TestDeletePool(t *testing.T) { + testAddPool(t, "deletePoolTest", nil, []string{}) + + pools, err := ctl.ListPools() + if err != nil { + t.Fatal(err) + } + + if len(pools) < 1 { + t.Fatal("Unable to retrieve pools") + } + + for _, pool := range pools { + if pool.Name == "deletePoolTest" { + err := ctl.DeletePool(pool.ID) + if err != nil { + t.Fatal(err) + } + + _, err = ctl.ShowPool(pool.ID) + if err != types.ErrPoolNotFound { + t.Fatal("Pool not deleted") + } + return + } + } + + t.Fatal("Could not delete pool") +} + +func TestAddPoolSubnet(t *testing.T) { + subnet := "192.168.0.0/24" + + testAddPool(t, "addsubnet", nil, []string{}) + + pools, err := ctl.ListPools() + if err != nil { + t.Fatal(err) + } + + if len(pools) < 1 { + t.Fatal("Unable to retrieve pools") + } + + for _, pool := range pools { + if pool.Name == "addsubnet" { + err := ctl.AddAddress(pool.ID, &subnet, []string{}) + if err != nil { + t.Fatal(err) + } + + p1, err := ctl.ShowPool(pool.ID) + if err != nil { + t.Fatal(err) + } + + // we should have a our subnet. + if p1.Subnets[0].CIDR != subnet { + t.Fatalf("expectd %s subnet got %s", subnet, p1.Subnets[0].CIDR) + } + + err = ctl.DeletePool(pool.ID) + if err != nil { + t.Fatal(err) + } + return + } + } + +} + +func TestAddPoolAddress(t *testing.T) { + address := "192.168.1.1" + + testAddPool(t, "addaddress", nil, []string{}) + + pools, err := ctl.ListPools() + if err != nil { + t.Fatal(err) + } + + if len(pools) < 1 { + t.Fatal("Unable to retrieve pools") + } + + for _, pool := range pools { + if pool.Name == "addaddress" { + err := ctl.AddAddress(pool.ID, nil, []string{address}) + if err != nil { + t.Fatal(err) + } + + p1, err := ctl.ShowPool(pool.ID) + if err != nil { + t.Fatal(err) + } + + // we should have a our subnet. + if p1.IPs[0].Address != address { + t.Fatalf("expected %s address got %s", address, p1.IPs[0].Address) + } + + err = ctl.DeletePool(pool.ID) + if err != nil { + t.Fatal(err) + } + return + } + } + +} + +func TestRemovePoolSubnet(t *testing.T) { + subnet := "192.168.0.0/24" + address := "192.168.1.1" + + testAddPool(t, "addsubnet", &subnet, []string{}) + + pools, err := ctl.ListPools() + if err != nil { + t.Fatal(err) + } + + var pool types.Pool + + for _, pool = range pools { + if pool.Name == "addsubnet" { + // make sure the subnet is there. + for _, sub := range pool.Subnets { + if sub.CIDR == subnet { + err := ctl.RemoveAddress(pool.ID, &sub.ID, nil) + if err != nil { + t.Fatalf("%s: %v\n", err, pool.Subnets) + } + } + } + } + break + } + + p1, err := ctl.ShowPool(pool.ID) + if err != nil { + t.Fatal(err) + } + + if len(p1.Subnets) != 0 || p1.TotalIPs != 0 || p1.Free != 0 { + fmt.Printf("pool %v\n", p1) + t.Fatal("subnet not deleted") + } + + err = ctl.AddAddress(pool.ID, nil, []string{address}) + if err != nil { + t.Fatal(err) + } + + p1, err = ctl.ShowPool(pool.ID) + if err != nil { + t.Fatal(err) + } + + err = ctl.RemoveAddress(pool.ID, nil, &p1.IPs[0].ID) + if err != nil { + t.Fatalf("%s: %v\n", err, pool.IPs) + } + + err = ctl.RemoveAddress(pool.ID, nil, nil) + if err != types.ErrBadRequest { + t.Fatal("invalid remove address request allowed") + } +} + +func TestMapAddress(t *testing.T) { + var reason payloads.StartFailureReason + + client, instances := testStartWorkload(t, 1, false, reason) + defer client.Shutdown() + + ips := []string{"10.10.0.1"} + poolName := "testmap" + + testAddPool(t, poolName, nil, ips) + + pools, err := ctl.ListPools() + if err != nil { + t.Fatal(err) + } + + if len(pools) < 1 { + t.Fatal("Unable to retrieve pools") + } + + for _, pool := range pools { + if pool.Name == "testmap" { + if pool.Free != 1 { + t.Fatal("Pool Free not correct") + } + } + } + + err = ctl.MapAddress(&poolName, instances[0].ID) + if err != nil { + t.Fatal(err) + } + + pools, err = ctl.ListPools() + if err != nil { + t.Fatal(err) + } + + if len(pools) < 1 { + t.Fatal("Unable to retrieve pools") + } + + for _, pool := range pools { + if pool.Name == "testmap" { + if pool.Free != 0 { + fmt.Printf("%v", pool) + t.Fatal("Pool Free not decremented") + } + } + } +} + +func TestMapAddressNoPool(t *testing.T) { + var reason payloads.StartFailureReason + + client, instances := testStartWorkload(t, 1, false, reason) + defer client.Shutdown() + + ips := []string{"10.10.0.2"} + poolName := "testmapnopool" + + testAddPool(t, poolName, nil, ips) + + err := ctl.MapAddress(nil, instances[0].ID) + if err != nil { + t.Fatal(err) + } + + pools, err := ctl.ListPools() + if err != nil { + t.Fatal(err) + } + + if len(pools) < 1 { + t.Fatal("Unable to retrieve pools") + } + + for _, pool := range pools { + if pool.Name == "testmapnopool" { + if pool.Free != 0 { + fmt.Printf("%v", pool) + t.Fatal("Pool Free not decremented") + } + } + } + + mappedIPs := ctl.ListMappedAddresses(&instances[0].TenantID) + if len(mappedIPs) != 1 { + t.Fatal("mapped IP not in list") + } +} + var testClients []*testutil.SsntpTestClient var ctl *controller var server *testutil.SsntpTestServer diff --git a/ciao-controller/external_ip.go b/ciao-controller/external_ip.go new file mode 100644 index 000000000..9319ae055 --- /dev/null +++ b/ciao-controller/external_ip.go @@ -0,0 +1,267 @@ +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "net" + + "github.com/01org/ciao/ciao-controller/types" + "github.com/01org/ciao/ssntp/uuid" +) + +func (c *controller) makePoolLinks(pool *types.Pool) { + for i := range pool.Subnets { + subnet := &pool.Subnets[i] + + ref := fmt.Sprintf("%s/pools/%s/subnets/%s", + c.apiURL, pool.ID, subnet.ID) + + link := types.Link{ + Rel: "self", + Href: ref, + } + + subnet.Links = []types.Link{link} + } + + for i := range pool.IPs { + IP := &pool.IPs[i] + + ref := fmt.Sprintf("%s/pools/%s/external-ips/%s", + c.apiURL, pool.ID, IP.ID) + + link := types.Link{ + Rel: "self", + Href: ref, + } + + IP.Links = []types.Link{link} + } + + selfRef := fmt.Sprintf("%s/pools/%s", c.apiURL, pool.ID) + link := types.Link{ + Rel: "self", + Href: selfRef, + } + + pool.Links = []types.Link{link} +} + +func (c *controller) makeMappedIPLinks(IP *types.MappedIP, tenant *string) { + var ref string + + if tenant != nil { + ref = fmt.Sprintf("%s/%s/external-ips/%s", + c.apiURL, *tenant, IP.ID) + } else { + ref = fmt.Sprintf("%s/external-ips/%s", + c.apiURL, IP.ID) + } + + selfLink := types.Link{ + Rel: "self", + Href: ref, + } + + IP.Links = []types.Link{selfLink} + + if tenant == nil { + poolRef := fmt.Sprintf("%s/pools/%s", c.apiURL, IP.PoolID) + link := types.Link{ + Rel: "pool", + Href: poolRef, + } + IP.Links = append(IP.Links, link) + } +} + +func (c *controller) AddPool(name string, subnet *string, ips []string) (types.Pool, error) { + pools, err := c.ds.GetPools() + if err != nil { + return types.Pool{}, err + } + + for _, p := range pools { + if p.Name == name { + return types.Pool{}, types.ErrDuplicatePoolName + } + } + + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: name, + } + + if subnet != nil { + sub := types.ExternalSubnet{ + ID: uuid.Generate().String(), + CIDR: *subnet, + } + + _, ipNet, err := net.ParseCIDR(*subnet) + if err != nil { + return pool, err + } + + ones, bits := ipNet.Mask.Size() + + // subtract out gateway and broadcast + TotalIPs := (1 << uint32(bits-ones)) - 2 + pool.TotalIPs = TotalIPs + pool.Free = pool.TotalIPs + pool.Subnets = append(pool.Subnets, sub) + } else if len(ips) > 0 { + for _, i := range ips { + addr := net.ParseIP(i) + if addr == nil { + return pool, types.ErrInvalidIP + } + + IP := types.ExternalIP{ + ID: uuid.Generate().String(), + Address: i, + } + + pool.IPs = append(pool.IPs, IP) + } + pool.TotalIPs = len(ips) + pool.Free = pool.TotalIPs + } + + err = c.ds.AddPool(pool) + + return pool, err +} + +func (c *controller) ListPools() ([]types.Pool, error) { + pools, err := c.ds.GetPools() + if err != nil { + return pools, err + } + + // update the links. we do this here because we get the + // current hostname:port. + for i := range pools { + pool := &pools[i] + c.makePoolLinks(pool) + } + + return pools, nil +} + +func (c *controller) ShowPool(ID string) (types.Pool, error) { + pool, err := c.ds.GetPool(ID) + if err != nil { + return pool, err + } + + c.makePoolLinks(&pool) + + return pool, nil +} + +func (c *controller) AddAddress(poolID string, subnet *string, ips []string) error { + if subnet != nil { + return c.ds.AddExternalSubnet(poolID, *subnet) + } + + return c.ds.AddExternalIPs(poolID, ips) +} + +func (c *controller) DeletePool(ID string) error { + return c.ds.DeletePool(ID) +} + +func (c *controller) RemoveAddress(poolID string, subnetID *string, IPID *string) error { + if subnetID != nil { + return c.ds.DeleteSubnet(poolID, *subnetID) + } + + if IPID != nil { + return c.ds.DeleteExternalIP(poolID, *IPID) + } + + return types.ErrBadRequest +} + +func (c *controller) ListMappedAddresses(tenant *string) []types.MappedIP { + IPs := c.ds.GetMappedIPs(tenant) + + for i := range IPs { + IP := &IPs[i] + c.makeMappedIPLinks(IP, tenant) + } + + return IPs +} + +func (c *controller) MapAddress(poolName *string, instanceID string) error { + var m types.MappedIP + + pools, err := c.ds.GetPools() + if err != nil { + return err + } + + err = types.ErrPoolEmpty + + for _, pool := range pools { + if poolName != nil { + if pool.Name == *poolName { + m, err = c.ds.MapExternalIP(pool.ID, instanceID) + break + } + } else if pool.Free > 0 { + m, err = c.ds.MapExternalIP(pool.ID, instanceID) + break + } + } + + if err != nil { + return err + } + + // get tenant CNCI info + t, err := c.ds.GetTenant(m.TenantID) + if err != nil { + _ = c.UnMapAddress(m.ExternalIP) + return err + } + + err = c.client.mapExternalIP(*t, m) + if err != nil { + // can never fail at this point. + _ = c.UnMapAddress(m.ExternalIP) + } + + return err +} + +func (c *controller) UnMapAddress(address string) error { + // get mapping + m, err := c.ds.GetMappedIP(address) + if err != nil { + return err + } + + // get tenant CNCI info + t, err := c.ds.GetTenant(m.TenantID) + if err != nil { + return err + } + + return c.client.unMapExternalIP(*t, m) +} diff --git a/ciao-controller/internal/datastore/datastore.go b/ciao-controller/internal/datastore/datastore.go index 6567c5545..d2e242907 100644 --- a/ciao-controller/internal/datastore/datastore.go +++ b/ciao-controller/internal/datastore/datastore.go @@ -126,6 +126,16 @@ type persistentStore interface { createStorageAttachment(a types.StorageAttachment) error getAllStorageAttachments() (map[string]types.StorageAttachment, error) deleteStorageAttachment(ID string) error + + // external IP interfaces + createPool(pool types.Pool) error + updatePool(pool types.Pool) error + getAllPools() map[string]types.Pool + deletePool(ID string) error + + createMappedIP(m types.MappedIP) error + deleteMappedIP(ID string) error + getMappedIPs() map[string]types.MappedIP } // Datastore provides context for the datastore package. @@ -166,6 +176,32 @@ type Datastore struct { attachLock *sync.RWMutex // maybe add a map[instanceid][]types.StorageAttachment // to make retrieval of volumes faster. + + pools map[string]types.Pool + externalSubnets map[string]bool + externalIPs map[string]bool + mappedIPs map[string]types.MappedIP + poolsLock *sync.RWMutex +} + +func (ds *Datastore) initExternalIPs() { + ds.poolsLock = &sync.RWMutex{} + ds.externalSubnets = make(map[string]bool) + ds.externalIPs = make(map[string]bool) + + ds.pools = ds.db.getAllPools() + + for _, pool := range ds.pools { + for _, subnet := range pool.Subnets { + ds.externalSubnets[subnet.CIDR] = true + } + + for _, IP := range pool.IPs { + ds.externalIPs[IP.Address] = true + } + } + + ds.mappedIPs = ds.db.getMappedIPs() } // Init initializes the private data for the Datastore object. @@ -283,6 +319,8 @@ func (ds *Datastore) Init(config Config) error { ds.attachLock = &sync.RWMutex{} + ds.initExternalIPs() + return err } @@ -1388,6 +1426,11 @@ func (ds *Datastore) ClearLog() error { return ds.db.clearLog() } +// LogEvent will add a message to the persistent event log. +func (ds *Datastore) LogEvent(tenant string, msg string) { + ds.db.logEvent(tenant, string(userInfo), msg) +} + // AddBlockDevice will store information about new BlockData into // the datastore. func (ds *Datastore) AddBlockDevice(device types.BlockData) error { @@ -1700,3 +1743,517 @@ func (ds *Datastore) GetVolumeAttachments(volume string) ([]types.StorageAttachm return attachments, nil } + +// GetPool will return an external IP Pool +func (ds *Datastore) GetPool(ID string) (types.Pool, error) { + ds.poolsLock.RLock() + p, ok := ds.pools[ID] + ds.poolsLock.RUnlock() + + if !ok { + return p, types.ErrPoolNotFound + } + + return p, nil +} + +// GetPools will return a list of external IP Pools +func (ds *Datastore) GetPools() ([]types.Pool, error) { + var pools []types.Pool + + ds.poolsLock.RLock() + + for _, p := range ds.pools { + pools = append(pools, p) + } + + ds.poolsLock.RUnlock() + + return pools, nil +} + +// lock for the map must be held by caller. +func (ds *Datastore) isDuplicateSubnet(new *net.IPNet) bool { + for s, exists := range ds.externalSubnets { + if exists == true { + // this will always succeed + _, subnet, _ := net.ParseCIDR(s) + + if subnet.Contains(new.IP) || new.Contains(subnet.IP) { + return true + } + } + } + + return false +} + +// lock for the map must be held by the caller +func (ds *Datastore) isDuplicateIP(new net.IP) bool { + // first make sure the IP isn't covered by a subnet + for s, exists := range ds.externalSubnets { + // this will always succeed + _, subnet, _ := net.ParseCIDR(s) + + if exists == true { + if subnet.Contains(new) { + return true + } + } + } + + // next make sure that the IP isn't already in a + // different pool + return ds.externalIPs[new.String()] +} + +// AddPool will add a brand new pool to our datastore. +func (ds *Datastore) AddPool(pool types.Pool) error { + ds.poolsLock.Lock() + + if len(pool.Subnets) > 0 { + // check each one to make sure it's not in use. + for _, subnet := range pool.Subnets { + _, newSubnet, err := net.ParseCIDR(subnet.CIDR) + if err != nil { + ds.poolsLock.Unlock() + return err + } + + if ds.isDuplicateSubnet(newSubnet) { + ds.poolsLock.Unlock() + return types.ErrDuplicateSubnet + } + + // update our list of used subnets + ds.externalSubnets[subnet.CIDR] = true + } + } else if len(pool.IPs) > 0 { + var newIPs []net.IP + + // make sure valid and not duplicate + for _, newIP := range pool.IPs { + IP := net.ParseIP(newIP.Address) + if IP == nil { + ds.poolsLock.Unlock() + return types.ErrInvalidIP + } + + if ds.isDuplicateIP(IP) { + ds.poolsLock.Unlock() + return types.ErrDuplicateIP + } + + newIPs = append(newIPs, IP) + } + + // now that the whole list is confirmed, we can update + for _, IP := range newIPs { + ds.externalIPs[IP.String()] = true + } + } + + ds.pools[pool.ID] = pool + err := ds.db.createPool(pool) + + ds.poolsLock.Unlock() + + if err != nil { + // lock must not be held when calling. + ds.DeletePool(pool.ID) + } + + return err +} + +// DeletePool will delete an unused pool from our datastore. +func (ds *Datastore) DeletePool(ID string) error { + ds.poolsLock.Lock() + defer ds.poolsLock.Unlock() + + p, ok := ds.pools[ID] + if !ok { + return types.ErrPoolNotFound + } + + // make sure all ips in this pool are not used. + if p.Free != p.TotalIPs { + return types.ErrPoolNotEmpty + } + + // delete from persistent store + err := ds.db.deletePool(ID) + if err != nil { + return err + } + + // delete all subnets + for _, subnet := range p.Subnets { + delete(ds.externalSubnets, subnet.CIDR) + } + + // delete any individual IPs + for _, IP := range p.IPs { + delete(ds.externalIPs, IP.Address) + } + + // delete the whole pool + delete(ds.pools, ID) + + return err +} + +// AddExternalSubnet will add a new subnet to an existing pool. +func (ds *Datastore) AddExternalSubnet(poolID string, subnet string) error { + sub := types.ExternalSubnet{ + ID: uuid.Generate().String(), + CIDR: subnet, + } + + _, ipNet, err := net.ParseCIDR(subnet) + if err != nil { + return err + } + + ds.poolsLock.Lock() + defer ds.poolsLock.Unlock() + + p, ok := ds.pools[poolID] + if !ok { + return types.ErrPoolNotFound + } + + if ds.isDuplicateSubnet(ipNet) { + return types.ErrDuplicateSubnet + } + + ones, bits := ipNet.Mask.Size() + + // deduct gateway and broadcast + newIPs := (1 << uint32(bits-ones)) - 2 + p.TotalIPs += newIPs + p.Free += newIPs + p.Subnets = append(p.Subnets, sub) + + err = ds.db.updatePool(p) + if err != nil { + return err + } + + // we are committed now. + ds.pools[poolID] = p + ds.externalSubnets[sub.CIDR] = true + + return nil +} + +// AddExternalIPs will add a list of individual IPs to an existing pool. +func (ds *Datastore) AddExternalIPs(poolID string, IPs []string) error { + ds.poolsLock.Lock() + defer ds.poolsLock.Unlock() + + p, ok := ds.pools[poolID] + if !ok { + return types.ErrPoolNotFound + } + + // make sure valid and not duplicate + for _, newIP := range IPs { + IP := net.ParseIP(newIP) + if IP == nil { + return types.ErrInvalidIP + } + + if ds.isDuplicateIP(IP) { + return types.ErrDuplicateIP + } + + ExtIP := types.ExternalIP{ + ID: uuid.Generate().String(), + Address: IP.String(), + } + + p.TotalIPs++ + p.Free++ + p.IPs = append(p.IPs, ExtIP) + } + + // update persistent store. + err := ds.db.updatePool(p) + if err != nil { + return err + } + + // update cache. + for _, IP := range p.IPs { + ds.externalIPs[IP.Address] = true + } + ds.pools[poolID] = p + + return nil +} + +// DeleteSubnet will remove an unused subnet from an existing pool. +func (ds *Datastore) DeleteSubnet(poolID string, subnetID string) error { + ds.poolsLock.Lock() + defer ds.poolsLock.Unlock() + + p, ok := ds.pools[poolID] + if !ok { + return types.ErrPoolNotFound + } + + for i, sub := range p.Subnets { + if sub.ID != subnetID { + continue + } + + // this path will be taken only once. + IP, ipNet, err := net.ParseCIDR(sub.CIDR) + if err != nil { + return err + } + + // check each address in this subnet is not mapped. + for IP := IP.Mask(ipNet.Mask); ipNet.Contains(IP); incrementIP(IP) { + _, ok := ds.mappedIPs[IP.String()] + if ok { + return types.ErrPoolNotEmpty + } + } + + ones, bits := ipNet.Mask.Size() + numIPs := (1 << uint32(bits-ones)) - 2 + p.TotalIPs -= numIPs + p.Free -= numIPs + p.Subnets = append(p.Subnets[:i], p.Subnets[i+1:]...) + + err = ds.db.updatePool(p) + if err != nil { + return err + } + + delete(ds.externalSubnets, sub.CIDR) + ds.pools[poolID] = p + + return nil + } + + return types.ErrInvalidPoolAddress +} + +// DeleteExternalIP will remove an individual IP address from a pool. +func (ds *Datastore) DeleteExternalIP(poolID string, addrID string) error { + ds.poolsLock.Lock() + defer ds.poolsLock.Unlock() + + p, ok := ds.pools[poolID] + if !ok { + return types.ErrPoolNotFound + } + + for i, extIP := range p.IPs { + if extIP.ID != addrID { + continue + } + + // this path will be taken only once. + // check address is not mapped. + _, ok := ds.mappedIPs[extIP.Address] + if ok { + return types.ErrPoolNotEmpty + } + + p.TotalIPs-- + p.Free-- + p.IPs = append(p.IPs[:i], p.IPs[i+1:]...) + + err := ds.db.updatePool(p) + if err != nil { + return err + } + + delete(ds.externalIPs, extIP.Address) + ds.pools[poolID] = p + + return nil + } + + return types.ErrInvalidPoolAddress +} + +func incrementIP(IP net.IP) { + for i := len(IP) - 1; i >= 0; i-- { + IP[i]++ + if IP[i] > 0 { + break + } + } +} + +// GetMappedIPs will return a list of mapped external IPs by tenant. +func (ds *Datastore) GetMappedIPs(tenant *string) []types.MappedIP { + var mappedIPs []types.MappedIP + + ds.poolsLock.RLock() + defer ds.poolsLock.RUnlock() + + for _, m := range ds.mappedIPs { + if tenant != nil { + if m.TenantID != *tenant { + continue + } + } + mappedIPs = append(mappedIPs, m) + } + + return mappedIPs +} + +// GetMappedIP will return a MappedIP struct for the given address. +func (ds *Datastore) GetMappedIP(address string) (types.MappedIP, error) { + ds.poolsLock.RLock() + defer ds.poolsLock.RUnlock() + + m, ok := ds.mappedIPs[address] + if !ok { + return types.MappedIP{}, types.ErrAddressNotFound + } + + return m, nil +} + +// MapExternalIP will allocate an external IP to an instance from a given pool. +func (ds *Datastore) MapExternalIP(poolID string, instanceID string) (types.MappedIP, error) { + var m types.MappedIP + + instance, err := ds.GetInstance(instanceID) + if err != nil { + return m, err + } + + ds.poolsLock.Lock() + defer ds.poolsLock.Unlock() + + pool, ok := ds.pools[poolID] + if !ok { + return m, types.ErrPoolNotFound + } + + if pool.Free == 0 { + return m, types.ErrPoolEmpty + } + + // find a free IP address in any subnet. + for _, sub := range pool.Subnets { + IP, ipNet, err := net.ParseCIDR(sub.CIDR) + if err != nil { + return m, err + } + + initIP := IP.Mask(ipNet.Mask) + + // skip gateway + incrementIP(initIP) + + // check each address in this subnet + for IP := initIP; ipNet.Contains(IP); incrementIP(IP) { + _, ok := ds.mappedIPs[IP.String()] + if !ok { + m.ID = uuid.Generate().String() + m.ExternalIP = IP.String() + m.InternalIP = instance.IPAddress + m.InstanceID = instanceID + m.TenantID = instance.TenantID + m.PoolID = pool.ID + m.PoolName = pool.Name + + pool.Free-- + + err = ds.db.createMappedIP(m) + if err != nil { + return types.MappedIP{}, err + } + ds.mappedIPs[IP.String()] = m + + err = ds.db.updatePool(pool) + if err != nil { + return types.MappedIP{}, err + } + + ds.pools[poolID] = pool + + return m, nil + } + } + } + + // we are still looking. Check our individual IPs + for _, IP := range pool.IPs { + _, ok := ds.mappedIPs[IP.Address] + if !ok { + m.ID = uuid.Generate().String() + m.ExternalIP = IP.Address + m.InternalIP = instance.IPAddress + m.InstanceID = instanceID + m.TenantID = instance.TenantID + m.PoolID = pool.ID + m.PoolName = pool.Name + + pool.Free-- + + err = ds.db.createMappedIP(m) + if err != nil { + return types.MappedIP{}, err + } + ds.mappedIPs[IP.Address] = m + + err = ds.db.updatePool(pool) + if err != nil { + return types.MappedIP{}, err + } + + ds.pools[poolID] = pool + + return m, nil + } + } + + // if you got here you are out of luck. But you never should. + glog.Errorf("Pool reports %d free addresses but none found\n", pool.Free) + return m, types.ErrPoolEmpty +} + +// UnMapExternalIP will stop associating a given address with an instance. +func (ds *Datastore) UnMapExternalIP(address string) error { + ds.poolsLock.Lock() + defer ds.poolsLock.Unlock() + + m, ok := ds.mappedIPs[address] + if !ok { + return types.ErrAddressNotFound + } + + // get pool and update Free + pool, ok := ds.pools[m.PoolID] + if !ok { + return types.ErrPoolNotFound + } + + pool.Free++ + + err := ds.db.deleteMappedIP(m.ID) + if err != nil { + return err + } + delete(ds.mappedIPs, address) + + err = ds.db.updatePool(pool) + if err != nil { + return err + } + + ds.pools[pool.ID] = pool + + return nil +} diff --git a/ciao-controller/internal/datastore/datastore_test.go b/ciao-controller/internal/datastore/datastore_test.go index 29b13d58a..bb1f964a9 100644 --- a/ciao-controller/internal/datastore/datastore_test.go +++ b/ciao-controller/internal/datastore/datastore_test.go @@ -24,6 +24,7 @@ import ( "fmt" "net" "os" + "reflect" "testing" "time" @@ -2277,6 +2278,574 @@ func TestGetVolumeAttachments(t *testing.T) { } } +func TestAddPool(t *testing.T) { + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err := ds.AddPool(pool) + if err != nil { + t.Fatal(err) + } + + // add one with a subnet. + subnet := types.ExternalSubnet{ + ID: uuid.Generate().String(), + CIDR: "192.168.0.0/24", + } + + pool2 := types.Pool{ + ID: uuid.Generate().String(), + Name: "test2", + Subnets: []types.ExternalSubnet{subnet}, + } + + err = ds.AddPool(pool2) + if err != nil { + t.Fatal(err) + } + + // add one with a duplicate subnet - should fail. + subnet = types.ExternalSubnet{ + ID: uuid.Generate().String(), + CIDR: "192.168.0.0/24", + } + + pool3 := types.Pool{ + ID: uuid.Generate().String(), + Name: "test3", + Subnets: []types.ExternalSubnet{subnet}, + } + + err = ds.AddPool(pool3) + if err != types.ErrDuplicateSubnet { + t.Fatal("Duplicate subnet allowed") + } + + // add one with ip addresses + addr := types.ExternalIP{ + ID: uuid.Generate().String(), + Address: "192.168.1.1", + } + + pool4 := types.Pool{ + ID: uuid.Generate().String(), + Name: "test4", + IPs: []types.ExternalIP{addr}, + } + + err = ds.AddPool(pool4) + if err != nil { + t.Fatal(err) + } + + // add one with a duplicate IP - should fail. + addr = types.ExternalIP{ + ID: uuid.Generate().String(), + Address: "192.168.1.1", + } + + pool5 := types.Pool{ + ID: uuid.Generate().String(), + Name: "test5", + IPs: []types.ExternalIP{addr}, + } + + err = ds.AddPool(pool5) + if err != types.ErrDuplicateIP { + t.Fatal("Duplicate IP allowed") + } + + // add one that overlaps an existing subnet + addr.Address = "192.168.0.1" + pool5.IPs = []types.ExternalIP{addr} + err = ds.AddPool(pool5) + if err != types.ErrDuplicateIP { + t.Fatal("Duplicate IP allowed") + } + + // delete all the pools + pools, err := ds.GetPools() + if err != nil { + t.Fatal(err) + } + + for _, p := range pools { + err := ds.DeletePool(p.ID) + if err != nil { + t.Fatal(err) + } + } +} + +func TestGetPool(t *testing.T) { + orig := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err := ds.AddPool(orig) + if err != nil { + t.Fatal(err) + } + + pool, err := ds.GetPool(orig.ID) + if err != nil { + t.Fatal(err) + } + + if reflect.DeepEqual(orig, pool) == false { + t.Fatalf("expected %v, got %v\n", orig, pool) + } + + // try to get an invalid pool + _, err = ds.GetPool(uuid.Generate().String()) + if err != types.ErrPoolNotFound { + t.Fatal("Found non existent pool") + } + + err = ds.DeletePool(orig.ID) + if err != nil { + t.Fatal(err) + } +} + +func TestAddExternalSubnet(t *testing.T) { + orig := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err := ds.AddPool(orig) + if err != nil { + t.Fatal(err) + } + + subnet := "192.168.2.0/24" + err = ds.AddExternalSubnet(orig.ID, subnet) + if err != nil { + t.Fatal(err) + } + + pool, err := ds.GetPool(orig.ID) + if err != nil { + t.Fatal(err) + } + + if len(pool.Subnets) != 1 || pool.Subnets[0].CIDR != subnet { + t.Fatal("subnet not added correctly") + } + + // try to add to a not existing pool + err = ds.AddExternalSubnet(uuid.Generate().String(), subnet) + if err != types.ErrPoolNotFound { + t.Fatal("Unknown pool allowed") + } + + // try to add an overlapping subnet + overlap := "192.168.0.0/8" + err = ds.AddExternalSubnet(orig.ID, overlap) + if err != types.ErrDuplicateSubnet { + t.Fatal("overlapping subnet allowed") + } + + // try an invalid subnet + invalid := "not.a.subnet/10" + err = ds.AddExternalSubnet(orig.ID, invalid) + if err == nil { + t.Fatal("invalid subnet allowed") + } + + // cleanup. + err = ds.DeletePool(orig.ID) + if err != nil { + t.Fatal(err) + } +} + +func TestAddExternalIPs(t *testing.T) { + orig := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err := ds.AddPool(orig) + if err != nil { + t.Fatal(err) + } + + IPs := []string{"192.168.0.1"} + err = ds.AddExternalIPs(orig.ID, IPs) + if err != nil { + t.Fatal(err) + } + + pool, err := ds.GetPool(orig.ID) + if err != nil { + t.Fatal(err) + } + + if len(pool.IPs) != 1 || pool.IPs[0].Address != IPs[0] { + t.Fatal("address not added correctly") + } + + // add an invalid IP + IPs = []string{"not.a.IP"} + err = ds.AddExternalIPs(orig.ID, IPs) + if err != types.ErrInvalidIP { + t.Fatal("invalid IP allowed") + } + + // add a duplicate IP + IPs = []string{"192.168.0.1"} + err = ds.AddExternalIPs(orig.ID, IPs) + if err != types.ErrDuplicateIP { + t.Fatal("duplicate IP allowed") + } + + // add to an invalid pool + IPs = []string{"192.168.0.2"} + err = ds.AddExternalIPs(uuid.Generate().String(), IPs) + if err != types.ErrPoolNotFound { + t.Fatal("duplicate IP allowed") + } + + // cleanup. + err = ds.DeletePool(orig.ID) + if err != nil { + t.Fatal(err) + } +} + +func TestDeleteExternalSubnet(t *testing.T) { + orig := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err := ds.AddPool(orig) + if err != nil { + t.Fatal(err) + } + + subnet := "192.168.2.0/24" + err = ds.AddExternalSubnet(orig.ID, subnet) + if err != nil { + t.Fatal(err) + } + + pool, err := ds.GetPool(orig.ID) + if err != nil { + t.Fatal(err) + } + + // delete from the wrong pool + err = ds.DeleteSubnet(uuid.Generate().String(), pool.Subnets[0].CIDR) + if err != types.ErrPoolNotFound { + t.Fatal("delete from invalid pool allowed") + } + + // delete the wrong address + err = ds.DeleteSubnet(pool.ID, "192.168.0.0/24") + if err != types.ErrInvalidPoolAddress { + t.Fatal("delete of wrong subnet") + } + + // delete an invalid address + err = ds.DeleteSubnet(pool.ID, "192.not.a.subnet/24") + if err == nil { + t.Fatal("delete of invalid subnet") + } + + // try to delete a mapped subnet + tenant, err := addTestTenant() + if err != nil { + t.Fatal(err) + } + + wls, err := ds.GetWorkloads() + if err != nil { + t.Fatal(err) + } + + instance, err := addTestInstance(tenant, wls[0]) + if err != nil { + t.Fatal(err) + } + + m, err := ds.MapExternalIP(pool.ID, instance.ID) + if err != nil { + t.Fatal(err) + } + + err = ds.DeleteSubnet(pool.ID, pool.Subnets[0].ID) + if err != types.ErrPoolNotEmpty { + t.Fatal("delete with mapped IP in subnet allowed") + } + + // unmap + err = ds.UnMapExternalIP(m.ExternalIP) + if err != nil { + t.Fatal(err) + } + + // delete an existing subnet + err = ds.DeleteSubnet(pool.ID, pool.Subnets[0].ID) + if err != nil { + t.Fatal(err) + } + + // cleanup. + err = ds.DeletePool(orig.ID) + if err != nil { + t.Fatal(err) + } +} + +func TestDeleteExternalIPs(t *testing.T) { + orig := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err := ds.AddPool(orig) + if err != nil { + t.Fatal(err) + } + + IPs := []string{"192.168.0.1"} + err = ds.AddExternalIPs(orig.ID, IPs) + if err != nil { + t.Fatal(err) + } + + pool, err := ds.GetPool(orig.ID) + if err != nil { + t.Fatal(err) + } + + // try to delete from invalid pool + err = ds.DeleteExternalIP(uuid.Generate().String(), pool.IPs[0].ID) + if err != types.ErrPoolNotFound { + t.Fatal("delete from invalid pool") + } + + // try to delete an invalid address + err = ds.DeleteExternalIP(pool.ID, uuid.Generate().String()) + if err != types.ErrInvalidPoolAddress { + t.Fatal("delete invalid address") + } + + // try to delete a mapped address + tenant, err := addTestTenant() + if err != nil { + t.Fatal(err) + } + + wls, err := ds.GetWorkloads() + if err != nil { + t.Fatal(err) + } + + instance, err := addTestInstance(tenant, wls[0]) + if err != nil { + t.Fatal(err) + } + + m, err := ds.MapExternalIP(pool.ID, instance.ID) + if err != nil { + t.Fatal(err) + } + + err = ds.DeleteExternalIP(pool.ID, pool.IPs[0].ID) + if err != types.ErrPoolNotEmpty { + t.Fatal("delete mapped address") + } + + // unmap + err = ds.UnMapExternalIP(m.ExternalIP) + if err != nil { + t.Fatal(err) + } + + err = ds.DeleteExternalIP(pool.ID, pool.IPs[0].ID) + if err != nil { + t.Fatal(err) + } + + // cleanup. + err = ds.DeletePool(pool.ID) + if err != nil { + t.Fatal(err) + } +} + +func TestMapIPs(t *testing.T) { + orig := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err := ds.AddPool(orig) + if err != nil { + t.Fatal(err) + } + + IPs := []string{"192.168.0.1"} + err = ds.AddExternalIPs(orig.ID, IPs) + if err != nil { + t.Fatal(err) + } + + pool, err := ds.GetPool(orig.ID) + if err != nil { + t.Fatal(err) + } + + // prepare for map + tenant, err := addTestTenant() + if err != nil { + t.Fatal(err) + } + + wls, err := ds.GetWorkloads() + if err != nil { + t.Fatal(err) + } + + instance, err := addTestInstance(tenant, wls[0]) + if err != nil { + t.Fatal(err) + } + + // try to map to an invalid instance. + _, err = ds.MapExternalIP(pool.ID, uuid.Generate().String()) + if err == nil { + t.Fatal("map to invalid instance allowed") + } + + // try to map to an invalid pool + _, err = ds.MapExternalIP(uuid.Generate().String(), instance.ID) + if err != types.ErrPoolNotFound { + t.Fatal("map to invalid pool allowed") + } + + // try to map to an empty pool + err = ds.DeleteExternalIP(pool.ID, pool.IPs[0].ID) + if err != nil { + t.Fatal(err) + } + + _, err = ds.MapExternalIP(pool.ID, instance.ID) + if err != types.ErrPoolEmpty { + t.Fatal(err) + } + + // try to map to a valid instance. + err = ds.AddExternalIPs(orig.ID, IPs) + if err != nil { + t.Fatal(err) + } + + m, err := ds.MapExternalIP(pool.ID, instance.ID) + if err != nil { + t.Fatal(err) + } + + // unmap + err = ds.UnMapExternalIP(m.ExternalIP) + if err != nil { + t.Fatal(err) + } + + // cleanup. + err = ds.DeletePool(pool.ID) + if err != nil { + t.Fatal(err) + } +} + +func TestGetMappedIPs(t *testing.T) { + orig := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err := ds.AddPool(orig) + if err != nil { + t.Fatal(err) + } + + IPs := []string{"192.168.0.1"} + err = ds.AddExternalIPs(orig.ID, IPs) + if err != nil { + t.Fatal(err) + } + + pool, err := ds.GetPool(orig.ID) + if err != nil { + t.Fatal(err) + } + + // prepare for map + tenant, err := addTestTenant() + if err != nil { + t.Fatal(err) + } + + wls, err := ds.GetWorkloads() + if err != nil { + t.Fatal(err) + } + + instance, err := addTestInstance(tenant, wls[0]) + if err != nil { + t.Fatal(err) + } + + m, err := ds.MapExternalIP(pool.ID, instance.ID) + if err != nil { + t.Fatal(err) + } + + // get mapped ips with tenant + ips := ds.GetMappedIPs(&instance.TenantID) + if len(ips) != 1 { + t.Fatal("GetMappedIPs failed") + } + + // get without tenant. + ips = ds.GetMappedIPs(nil) + if len(ips) != 1 { + t.Fatal("GetMappedIPs failed") + } + + // get specific mapped IP + _, err = ds.GetMappedIP(m.ExternalIP) + if err != nil { + t.Fatal(err) + } + + // get invalid mapped IP + _, err = ds.GetMappedIP("192.168.0.2") + if err != types.ErrAddressNotFound { + t.Fatal("found invalid address") + } + + // unmap + err = ds.UnMapExternalIP(m.ExternalIP) + if err != nil { + t.Fatal(err) + } + + // cleanup. + err = ds.DeletePool(pool.ID) + if err != nil { + t.Fatal(err) + } +} + var ds *Datastore var tablesInitPath = flag.String("tables_init_path", "../../tables", "path to csv files") diff --git a/ciao-controller/internal/datastore/sqlite3db.go b/ciao-controller/internal/datastore/sqlite3db.go index 556e785ec..0785432dd 100644 --- a/ciao-controller/internal/datastore/sqlite3db.go +++ b/ciao-controller/internal/datastore/sqlite3db.go @@ -538,6 +538,147 @@ func (d traceData) Init() error { return d.ds.exec(d.db, cmd) } +type poolData struct { + namedData +} + +func (d poolData) Populate() error { + lines, err := d.ReadCsv() + if err != nil { + return err + } + + for _, line := range lines { + poolID := line[0] + poolName := line[1] + free, _ := strconv.Atoi(line[2]) + total, _ := strconv.Atoi(line[3]) + err = d.ds.create(d.name, poolID, poolName, free, total) + if err != nil { + glog.V(2).Info("could not add pool: ", err) + } + } + + return err +} + +func (d poolData) Init() error { + cmd := `CREATE TABLE IF NOT EXISTS pools + ( + id varchar(32), + name string, + free int, + total int, + PRIMARY KEY(id, name) + );` + + return d.ds.exec(d.db, cmd) +} + +type subnetPoolData struct { + namedData +} + +func (d subnetPoolData) Populate() error { + lines, err := d.ReadCsv() + if err != nil { + return err + } + + for _, line := range lines { + subnetID := line[0] + poolID := line[1] + cidr := line[2] + err = d.ds.create(d.name, subnetID, poolID, cidr) + if err != nil { + glog.V(2).Info("could not add subnet: ", err) + } + } + + return err +} + +func (d subnetPoolData) Init() error { + cmd := `CREATE TABLE IF NOT EXISTS subnet_pool + ( + id varchar(32) primary key, + pool_id varchar(32), + cidr string + );` + + return d.ds.exec(d.db, cmd) +} + +type addressData struct { + namedData +} + +func (d addressData) Populate() error { + lines, err := d.ReadCsv() + if err != nil { + return err + } + + for _, line := range lines { + addressID := line[0] + poolID := line[1] + address := line[2] + err = d.ds.create(d.name, addressID, poolID, address) + if err != nil { + glog.V(2).Info("could not add address: ", err) + } + } + + return err +} + +func (d addressData) Init() error { + cmd := `CREATE TABLE IF NOT EXISTS address_pool + ( + id varchar(32) primary key, + pool_id varchar(32), + address string + );` + + return d.ds.exec(d.db, cmd) +} + +type mappedIPData struct { + namedData +} + +func (d mappedIPData) Populate() error { + lines, err := d.ReadCsv() + if err != nil { + return err + } + + for _, line := range lines { + mappingID := line[0] + externalIP := line[1] + instanceID := line[2] + poolID := line[3] + err = d.ds.create(d.name, mappingID, externalIP, instanceID, poolID) + if err != nil { + glog.V(2).Info("could not add mapping: ", err) + } + } + + return err +} + +func (d mappedIPData) Init() error { + cmd := `CREATE TABLE IF NOT EXISTS mapped_ips + ( + id varchar(32) primary key, + external_ip string, + instance_id varchar(32), + pool_id varchar(32) + );` + + return d.ds.exec(d.db, cmd) +} + func (ds *sqliteDB) exec(db *sql.DB, cmd string) error { glog.V(2).Info("exec: ", cmd) @@ -631,6 +772,10 @@ func getPersistentStore(config Config) (persistentStore, error) { blockData{namedData{ds: ds, name: "block_data", db: ds.db}}, attachments{namedData{ds: ds, name: "attachments", db: ds.db}}, workloadStorage{namedData{ds: ds, name: "workload_storage", db: ds.db}}, + poolData{namedData{ds: ds, name: "pools", db: ds.db}}, + subnetPoolData{namedData{ds: ds, name: "subnet_pool", db: ds.db}}, + addressData{namedData{ds: ds, name: "address_pool", db: ds.db}}, + mappedIPData{namedData{ds: ds, name: "mapped_ips", db: ds.db}}, } ds.tableInitPath = config.InitTablesPath @@ -2388,3 +2533,388 @@ func (ds *sqliteDB) deleteStorageAttachment(ID string) error { return err } + +// this is here just for readability. +func (ds *sqliteDB) createPool(pool types.Pool) error { + return ds.updatePool(pool) +} + +// lock must be held by caller. Any rollbacks will need to be handled +// by caller. +func (ds *sqliteDB) updateSubnets(tx *sql.Tx, pool types.Pool) error { + // get currently known subnets. + subnets, err := ds.getPoolSubnets(pool.ID) + if err != nil { + // TBD: what about row not found? + return err + } + + // make a map of pool subnets by ID + subMap := make(map[string]bool) + for _, sub := range pool.Subnets { + subMap[sub.ID] = true + } + + // do we have any subnets that need deleting? + for _, sub := range subnets { + _, ok := subMap[sub.ID] + if !ok { + _, err = tx.Exec("DELETE FROM subnet_pool WHERE id = ?", sub.ID) + if err != nil { + return err + } + } + } + + // any subnets that already exist in the table will be ignored, + // new ones will be added. + for _, subnet := range pool.Subnets { + _, err = tx.Exec("INSERT OR IGNORE INTO subnet_pool (id, pool_id, cidr) VALUES (?, ?, ?)", subnet.ID, pool.ID, subnet.CIDR) + if err != nil { + return err + } + } + + return nil +} + +// lock must be held by caller. Any rollbacks will need to be handled +// by caller. +func (ds *sqliteDB) updateAddresses(tx *sql.Tx, pool types.Pool) error { + // get currently known individual addresses. + addresses, err := ds.getPoolAddresses(pool.ID) + if err != nil { + // TBD: what about row not found? + return err + } + + // make a map of pool addresses by ID + addrMap := make(map[string]bool) + for _, addr := range pool.IPs { + addrMap[addr.ID] = true + } + + // do we have any individual IPs that need deleting? + for _, addr := range addresses { + _, ok := addrMap[addr.ID] + if !ok { + _, err = tx.Exec("DELETE FROM address_pool WHERE id = ?", addr.ID) + if err != nil { + tx.Rollback() + return err + } + } + } + + // any addresses that already exist in the table will be ignored, + // new ones will be added. + for _, IP := range pool.IPs { + _, err = tx.Exec("INSERT OR IGNORE INTO address_pool (id, pool_id, address) VALUES (?, ?, ?)", IP.ID, pool.ID, IP.Address) + if err != nil { + tx.Rollback() + return err + } + } + + return nil +} + +// updatePool is used to update all pool related fields even if they +// are in different tables. +func (ds *sqliteDB) updatePool(pool types.Pool) error { + datastore := ds.getTableDB("pools") + + ds.dbLock.Lock() + defer ds.dbLock.Unlock() + + pools := ds.getAllPools() + + // do the below as a single transaction. + tx, err := datastore.Begin() + if err != nil { + return err + } + + err = ds.updateSubnets(tx, pool) + if err != nil { + tx.Rollback() + return err + } + + err = ds.updateAddresses(tx, pool) + if err != nil { + tx.Rollback() + return err + } + + // if this is a new pool, put it in, otherwise just update. + _, ok := pools[pool.ID] + if !ok { + _, err = tx.Exec("INSERT INTO pools (id, name, free, total) VALUES (?, ?, ?, ?)", pool.ID, pool.Name, pool.Free, pool.TotalIPs) + if err != nil { + tx.Rollback() + return err + } + } else { + // update free and total counts. + _, err = tx.Exec("UPDATE pools SET free = ?, total = ? WHERE id = ?", pool.Free, pool.TotalIPs, pool.ID) + if err != nil { + tx.Rollback() + return err + } + } + + tx.Commit() + + return nil +} + +func (ds *sqliteDB) getAllPools() map[string]types.Pool { + pools := make(map[string]types.Pool) + + datastore := ds.getTableDB("pools") + + query := `SELECT id, + name, + free, + total + FROM pools` + + rows, err := datastore.Query(query) + if err != nil { + return nil + } + defer rows.Close() + + for rows.Next() { + var pool types.Pool + + err = rows.Scan(&pool.ID, &pool.Name, &pool.Free, &pool.TotalIPs) + if err != nil { + continue + } + + pool.Subnets, err = ds.getPoolSubnets(pool.ID) + if err != nil { + continue + } + + pool.IPs, err = ds.getPoolAddresses(pool.ID) + if err != nil { + continue + } + + pools[pool.ID] = pool + } + + if err = rows.Err(); err != nil { + return nil + } + + return pools +} + +func (ds *sqliteDB) deletePool(ID string) error { + datastore := ds.getTableDB("pools") + + ds.dbLock.Lock() + defer ds.dbLock.Unlock() + + tx, err := datastore.Begin() + if err != nil { + return err + } + + // lock is held here and ok because the + // get functions don't hold a lock. + subnets, err := ds.getPoolSubnets(ID) + if err != nil { + return err + } + + IPs, err := ds.getPoolAddresses(ID) + if err != nil { + return err + } + + for _, subnet := range subnets { + _, err = tx.Exec("DELETE FROM subnet_pool WHERE id = ?", subnet.ID) + if err != nil { + tx.Rollback() + return err + } + } + + for _, addr := range IPs { + _, err = tx.Exec("DELETE FROM address_pool WHERE id = ?", addr.ID) + if err != nil { + tx.Rollback() + return err + } + } + + _, err = tx.Exec("DELETE FROM pools WHERE id = ?", ID) + if err != nil { + tx.Rollback() + return err + } + + tx.Commit() + + return err +} + +func (ds *sqliteDB) getPoolSubnets(poolID string) ([]types.ExternalSubnet, error) { + var subnets []types.ExternalSubnet + + datastore := ds.getTableDB("subnet_pool") + + query := `SELECT id, + cidr + FROM subnet_pool + WHERE pool_id = ?` + + rows, err := datastore.Query(query, poolID) + if err != nil { + return subnets, err + } + defer rows.Close() + + for rows.Next() { + var subnet types.ExternalSubnet + + err = rows.Scan(&subnet.ID, &subnet.CIDR) + if err != nil { + continue + } + + subnets = append(subnets, subnet) + } + + if err = rows.Err(); err != nil { + return subnets, err + } + + return subnets, nil +} + +func (ds *sqliteDB) getPoolAddresses(poolID string) ([]types.ExternalIP, error) { + var IPs []types.ExternalIP + + datastore := ds.getTableDB("address_pool") + + query := `SELECT id, + address + FROM address_pool + WHERE pool_id = ?` + + rows, err := datastore.Query(query, poolID) + if err != nil { + return IPs, err + } + defer rows.Close() + + for rows.Next() { + var IP types.ExternalIP + + err = rows.Scan(&IP.ID, &IP.Address) + if err != nil { + continue + } + + IPs = append(IPs, IP) + } + + if err = rows.Err(); err != nil { + return IPs, err + } + + return IPs, nil +} + +func (ds *sqliteDB) createMappedIP(m types.MappedIP) error { + datastore := ds.getTableDB("mapped_ips") + + ds.dbLock.Lock() + defer ds.dbLock.Unlock() + + tx, err := datastore.Begin() + if err != nil { + return err + } + + _, err = tx.Exec("INSERT INTO mapped_ips (id, pool_id, external_ip, instance_id) VALUES (?, ?, ?, ?)", m.ID, m.PoolID, m.ExternalIP, m.InstanceID) + if err != nil { + tx.Rollback() + return err + } + + tx.Commit() + + return nil +} + +func (ds *sqliteDB) deleteMappedIP(ID string) error { + datastore := ds.getTableDB("mapped_ips") + + ds.dbLock.Lock() + defer ds.dbLock.Unlock() + + tx, err := datastore.Begin() + if err != nil { + return err + } + + _, err = tx.Exec("DELETE FROM mapped_ips WHERE id = ?", ID) + if err != nil { + tx.Rollback() + return err + } + + tx.Commit() + + return err +} + +func (ds *sqliteDB) getMappedIPs() map[string]types.MappedIP { + IPs := make(map[string]types.MappedIP) + + datastore := ds.getTableDB("mapped_ips") + + query := `SELECT mapped_ips.id, + mapped_ips.pool_id, + mapped_ips.external_ip, + mapped_ips.instance_id, + instances.ip, + instances.tenant_id, + pools.name + FROM mapped_ips + JOIN instances + ON instances.id = mapped_ips.instance_id + JOIN pools + ON pools.id = mapped_ips.pool_id` + + rows, err := datastore.Query(query) + if err != nil { + fmt.Println(err) + return IPs + } + defer rows.Close() + + for rows.Next() { + var IP types.MappedIP + + err = rows.Scan(&IP.ID, &IP.PoolID, &IP.ExternalIP, &IP.InstanceID, &IP.InternalIP, &IP.TenantID, &IP.PoolName) + if err != nil { + continue + } + + IPs[IP.ExternalIP] = IP + } + + if err = rows.Err(); err != nil { + fmt.Println(err) + } + + return IPs +} diff --git a/ciao-controller/internal/datastore/sqlite3db_test.go b/ciao-controller/internal/datastore/sqlite3db_test.go index 6aa140217..ccc791263 100644 --- a/ciao-controller/internal/datastore/sqlite3db_test.go +++ b/ciao-controller/internal/datastore/sqlite3db_test.go @@ -15,6 +15,7 @@ package datastore import ( + "reflect" "testing" "time" @@ -306,3 +307,470 @@ func TestGetAllStorageAttachments(t *testing.T) { } db.disconnect() } + +func TestCreatePool(t *testing.T) { + config := Config{ + PersistentURI: "file:testcreatepool?mode=memory&cache=shared", + TransientURI: "file:testcreatepoolt?mode=memory&cache=shared", + } + + db, err := getPersistentStore(config) + if err != nil { + t.Fatal(err) + } + + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err = db.createPool(pool) + if err != nil { + t.Fatal(err) + } + + pools := db.getAllPools() + if pools == nil { + t.Fatal("pool not stored") + } + + p, ok := pools[pool.ID] + if !ok || (p.Name != "test") { + t.Fatal("pool not stored") + } + + db.disconnect() +} + +func TestUpdatePool(t *testing.T) { + config := Config{ + PersistentURI: "file:testupdatepool?mode=memory&cache=shared", + TransientURI: "file:testupdatepoolt?mode=memory&cache=shared", + } + + db, err := getPersistentStore(config) + if err != nil { + t.Fatal(err) + } + + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err = db.createPool(pool) + if err != nil { + t.Fatal(err) + } + + pool.Free = 2 + pool.TotalIPs = 10 + + err = db.updatePool(pool) + if err != nil { + t.Fatal(err) + } + + pools := db.getAllPools() + if pools == nil { + t.Fatal("pool not stored") + } + + p, ok := pools[pool.ID] + if !ok || p.Free != 2 || p.TotalIPs != 10 { + t.Fatal("pool not updated") + } + + db.disconnect() +} + +func TestDeletePool(t *testing.T) { + config := Config{ + PersistentURI: "file:testdeletepool?mode=memory&cache=shared", + TransientURI: "file:testdeletepoolt?mode=memory&cache=shared", + } + + db, err := getPersistentStore(config) + if err != nil { + t.Fatal(err) + } + + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err = db.createPool(pool) + if err != nil { + t.Fatal(err) + } + + pools := db.getAllPools() + if pools == nil { + t.Fatal("pool not stored") + } + + _, ok := pools[pool.ID] + if !ok { + t.Fatal("pool not updated") + } + + err = db.deletePool(pool.ID) + if err != nil { + t.Fatal("pool not deleted") + } + + pools = db.getAllPools() + if pools == nil { + t.Fatal("pool not stored") + } + + _, ok = pools[pool.ID] + if ok { + t.Fatal("pool not deleted") + } + + db.disconnect() +} + +func TestCreateSubnet(t *testing.T) { + config := Config{ + PersistentURI: "file:testcreatesubnet?mode=memory&cache=shared", + TransientURI: "file:testcreatesubnett?mode=memory&cache=shared", + } + + db, err := getPersistentStore(config) + if err != nil { + t.Fatal(err) + } + + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err = db.createPool(pool) + if err != nil { + t.Fatal(err) + } + + subnet := types.ExternalSubnet{ + ID: uuid.Generate().String(), + CIDR: "192.168.0.0/24", + } + + pool.Subnets = append(pool.Subnets, subnet) + + err = db.updatePool(pool) + if err != nil { + t.Fatal(err) + } + + pools := db.getAllPools() + if pools == nil { + t.Fatal("pool not stored") + } + + p, ok := pools[pool.ID] + if !ok || (p.Name != "test") { + t.Fatal("pool not stored") + } + + subs := p.Subnets + if len(subs) != 1 { + t.Fatal("subnet not saved") + } + + if subs[0].CIDR != subnet.CIDR || subs[0].ID != subnet.ID { + t.Fatal("subnet not saved correctly") + } + + db.disconnect() +} + +func TestDeleteSubnet(t *testing.T) { + config := Config{ + PersistentURI: "file:testdeletesubnet?mode=memory&cache=shared", + TransientURI: "file:testdeletesubnett?mode=memory&cache=shared", + } + + db, err := getPersistentStore(config) + if err != nil { + t.Fatal(err) + } + + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + subnet := types.ExternalSubnet{ + ID: uuid.Generate().String(), + CIDR: "192.168.0.0/24", + } + + pool.Subnets = append(pool.Subnets, subnet) + + err = db.createPool(pool) + if err != nil { + t.Fatal(err) + } + + pool.Subnets = []types.ExternalSubnet{} + err = db.updatePool(pool) + if err != nil { + t.Fatal(err) + } + + pools := db.getAllPools() + if pools == nil { + t.Fatal("pool not stored") + } + + p, ok := pools[pool.ID] + if !ok || (p.Name != "test") { + t.Fatal("pool not stored") + } + + subs := p.Subnets + if len(subs) != 0 { + t.Fatal("subnet not deleted") + } + + db.disconnect() +} + +func TestCreateAddress(t *testing.T) { + config := Config{ + PersistentURI: "file:createaddress?mode=memory&cache=shared", + TransientURI: "file:createaddresst?mode=memory&cache=shared", + } + + db, err := getPersistentStore(config) + if err != nil { + t.Fatal(err) + } + + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + IP := types.ExternalIP{ + ID: uuid.Generate().String(), + Address: "192.168.0.1", + } + + pool.IPs = append(pool.IPs, IP) + + err = db.createPool(pool) + if err != nil { + t.Fatal(err) + } + + pools := db.getAllPools() + if pools == nil { + t.Fatal("pool not stored") + } + + p, ok := pools[pool.ID] + if !ok || (p.Name != "test") { + t.Fatal("pool not stored") + } + + addrs := p.IPs + if len(addrs) != 1 || addrs[0].ID != IP.ID || addrs[0].Address != IP.Address { + t.Fatal("address not stored correctly") + } + + db.disconnect() +} + +func TestDeleteAddress(t *testing.T) { + config := Config{ + PersistentURI: "file:deleteaddress?mode=memory&cache=shared", + TransientURI: "file:deleteaddresst?mode=memory&cache=shared", + } + + db, err := getPersistentStore(config) + if err != nil { + t.Fatal(err) + } + + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + IP := types.ExternalIP{ + ID: uuid.Generate().String(), + Address: "192.168.0.1", + } + + pool.IPs = append(pool.IPs, IP) + + err = db.createPool(pool) + if err != nil { + t.Fatal(err) + } + + pool.IPs = []types.ExternalIP{} + + err = db.updatePool(pool) + if err != nil { + t.Fatal(err) + } + + pools := db.getAllPools() + if pools == nil { + t.Fatal("pool not stored") + } + + p, ok := pools[pool.ID] + if !ok || (p.Name != "test") { + t.Fatal("pool not stored") + } + + addrs := p.IPs + if len(addrs) != 0 { + t.Fatal("address not deleted") + } + + db.disconnect() +} + +func TestCreateMappedIP(t *testing.T) { + config := Config{ + PersistentURI: "file:createmappedaddress?mode=memory&cache=shared", + TransientURI: "file:createmappedaddresst?mode=memory&cache=shared", + } + + db, err := getPersistentStore(config) + if err != nil { + t.Fatal(err) + } + + i := types.Instance{ + ID: uuid.Generate().String(), + TenantID: uuid.Generate().String(), + WorkloadID: uuid.Generate().String(), + IPAddress: "172.16.0.2", + } + + err = db.addInstance(&i) + if err != nil { + t.Fatal("unable to store instance") + } + + instances, err := db.getInstances() + if err != nil || len(instances) != 1 { + t.Fatal(err) + } + + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err = db.createPool(pool) + if err != nil { + t.Fatal(err) + } + + m := types.MappedIP{ + ID: uuid.Generate().String(), + ExternalIP: "192.168.0.1", + InternalIP: i.IPAddress, + InstanceID: i.ID, + TenantID: i.TenantID, + PoolID: pool.ID, + PoolName: pool.Name, + } + + err = db.createMappedIP(m) + if err != nil { + t.Fatal(err) + } + + IPs := db.getMappedIPs() + if len(IPs) != 1 { + t.Fatal("could not get mapped IP") + } + + if reflect.DeepEqual(IPs[m.ExternalIP], m) == false { + t.Fatalf("expected %v, got %v\n", m, IPs[m.ExternalIP]) + } +} + +func TestDeleteMappedIP(t *testing.T) { + config := Config{ + PersistentURI: "file:deletedmappedaddress?mode=memory&cache=shared", + TransientURI: "file:deletemappedaddresst?mode=memory&cache=shared", + } + + db, err := getPersistentStore(config) + if err != nil { + t.Fatal(err) + } + + i := types.Instance{ + ID: uuid.Generate().String(), + TenantID: uuid.Generate().String(), + WorkloadID: uuid.Generate().String(), + IPAddress: "172.16.0.2", + } + + err = db.addInstance(&i) + if err != nil { + t.Fatal("unable to store instance") + } + + instances, err := db.getInstances() + if err != nil || len(instances) != 1 { + t.Fatal(err) + } + + pool := types.Pool{ + ID: uuid.Generate().String(), + Name: "test", + } + + err = db.createPool(pool) + if err != nil { + t.Fatal(err) + } + + m := types.MappedIP{ + ID: uuid.Generate().String(), + ExternalIP: "192.168.0.1", + InternalIP: i.IPAddress, + InstanceID: i.ID, + TenantID: i.TenantID, + PoolID: pool.ID, + PoolName: pool.Name, + } + + err = db.createMappedIP(m) + if err != nil { + t.Fatal(err) + } + + IPs := db.getMappedIPs() + if len(IPs) != 1 { + t.Fatal("could not get mapped IP") + } + + if reflect.DeepEqual(IPs[m.ExternalIP], m) == false { + t.Fatalf("expected %v, got %v\n", m, IPs[m.ExternalIP]) + } + + err = db.deleteMappedIP(m.ID) + if err != nil { + t.Fatal(err) + } + + IPs = db.getMappedIPs() + if len(IPs) != 0 { + t.Fatal("IP not deleted") + } +} diff --git a/ciao-controller/main.go b/ciao-controller/main.go index 0718083e0..939dd3134 100644 --- a/ciao-controller/main.go +++ b/ciao-controller/main.go @@ -25,21 +25,27 @@ package main import ( "context" + "errors" "flag" + "fmt" + "net/http" "os" "strconv" "sync" + "github.com/01org/ciao/ciao-controller/api" datastore "github.com/01org/ciao/ciao-controller/internal/datastore" image "github.com/01org/ciao/ciao-image/client" storage "github.com/01org/ciao/ciao-storage" "github.com/01org/ciao/openstack/block" "github.com/01org/ciao/openstack/compute" + osIdentity "github.com/01org/ciao/openstack/identity" osimage "github.com/01org/ciao/openstack/image" "github.com/01org/ciao/osprepare" "github.com/01org/ciao/ssntp" "github.com/01org/ciao/testutil" "github.com/golang/glog" + "github.com/gorilla/mux" ) type controller struct { @@ -49,6 +55,7 @@ type controller struct { ds *datastore.Datastore id *identity image image.Client + apiURL string } var singleMachine = flag.Bool("single", false, "Enable single machine test") @@ -61,6 +68,7 @@ var servicePassword = "" var volumeAPIPort = block.APIPort var computeAPIPort = compute.APIPort var imageAPIPort = osimage.APIPort +var controllerAPIPort = api.Port var httpsCAcert = "/etc/pki/ciao/ciao-controller-cacert.pem" var httpsKey = "/etc/pki/ciao/ciao-controller-key.pem" var tablesInitPath = flag.String("tables_init_path", "./tables", "path to csv files") @@ -138,6 +146,7 @@ func main() { volumeAPIPort = clusterConfig.Configure.Controller.VolumePort computeAPIPort = clusterConfig.Configure.Controller.ComputePort + controllerAPIPort = clusterConfig.Configure.Controller.CiaoPort httpsCAcert = clusterConfig.Configure.Controller.HTTPSCACert httpsKey = clusterConfig.Configure.Controller.HTTPSKey identityURL = clusterConfig.Configure.IdentityService.URL @@ -202,7 +211,61 @@ func main() { wg.Add(1) go ctl.startImageService() + host, _ := os.Hostname() + ctl.apiURL = fmt.Sprintf("https://%s:%d", host, controllerAPIPort) + + wg.Add(1) + go ctl.startCiaoService() + wg.Wait() ctl.ds.Exit() ctl.client.Disconnect() } + +func (c *controller) startCiaoService() error { + config := api.Config{URL: c.apiURL, CiaoService: c} + + r := api.Routes(config) + if r == nil { + return errors.New("Unable to start Ciao API Service") + } + + // wrap each route in keystone validation. + validServices := []osIdentity.ValidService{ + {ServiceType: "compute", ServiceName: "ciao"}, + {ServiceType: "compute", ServiceName: "nova"}, + } + + validAdmins := []osIdentity.ValidAdmin{ + {Project: "service", Role: "admin"}, + {Project: "admin", Role: "admin"}, + } + + err := r.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { + h := osIdentity.Handler{ + Client: c.id.scV3, + Next: route.GetHandler(), + ValidServices: validServices, + ValidAdmins: validAdmins, + } + + route.Handler(h) + + return nil + }) + + if err != nil { + return err + } + + service := fmt.Sprintf(":%d", controllerAPIPort) + + glog.Infof("Starting ciao API on port %d\n", controllerAPIPort) + + err = http.ListenAndServeTLS(service, httpsCAcert, httpsKey, r) + if err != nil { + glog.Fatal(err) + } + + return nil +} diff --git a/ciao-controller/types/types.go b/ciao-controller/types/types.go index 5a8bd7439..b2c721e8d 100644 --- a/ciao-controller/types/types.go +++ b/ciao-controller/types/types.go @@ -513,4 +513,134 @@ var ( // ErrInstanceNotAssigned is returned when an instance is not assigned to a node. ErrInstanceNotAssigned = errors.New("Cannot perform operation: instance not assigned to Node") + + // ErrDuplicateSubnet is returned when a subnet already exists + ErrDuplicateSubnet = errors.New("Cannot add overlapping subnet") + + // ErrDuplicateIP is returned when a duplicate external IP is added + ErrDuplicateIP = errors.New("Cannot add duplicated external IP") + + // ErrInvalidIP is returned when an IP cannot be parsed + ErrInvalidIP = errors.New("The IP Address is not valid") + + // ErrPoolNotFound is returned when an external IP pool is not found + ErrPoolNotFound = errors.New("Pool not found") + + // ErrPoolNotEmpty is returned when a pool is still in use + ErrPoolNotEmpty = errors.New("Pool has mapped IPs") + + // ErrAddressNotFound is returned when an address isn't found. + ErrAddressNotFound = errors.New("Address Not Found") + + // ErrInvalidPoolAddress is returned when an address isn't part of a pool + ErrInvalidPoolAddress = errors.New("The Address is not found in this pool") + + // ErrBadRequest is returned when we have a malformed request + ErrBadRequest = errors.New("Invalid Request") + + // ErrPoolEmpty is returned when a pool has no free IPs + ErrPoolEmpty = errors.New("Pool has no Free IPs") + + // ErrDuplicatePoolName is returned when a duplicate pool name is used + ErrDuplicatePoolName = errors.New("Pool by that name already exists") ) + +// Link provides a url and relationship for a resource. +type Link struct { + Rel string `json:"rel"` + Href string `json:"href"` +} + +// APILink provides information and links about a supported resource. +type APILink struct { + Rel string `json:"rel"` + Href string `json:"href"` + Version string `json:"version"` + MinVersion string `json:"minimum_version"` +} + +// ExternalSubnet represents a subnet for External IPs. +type ExternalSubnet struct { + ID string `json:"id"` + CIDR string `json:"subnet"` + Links []Link `json:"links"` +} + +// ExternalIP represents an External IP individual address. +type ExternalIP struct { + ID string `json:"id"` + Address string `json:"address"` + Links []Link `json:"links"` +} + +// Pool represents a pool of external IPs. +type Pool struct { + ID string `json:"id"` + Name string `json:"name"` + Free int `json:"free"` + TotalIPs int `json:"total_ips"` + Links []Link `json:"links"` + Subnets []ExternalSubnet `json:"subnets"` + IPs []ExternalIP `json:"ips"` +} + +// NewPoolRequest is used to create a new pool. +type NewPoolRequest struct { + Name string `json:"name"` + Subnet *string `json:"subnet"` + IPs []struct { + IP string `json:"ip"` + } `json:"ips"` +} + +// PoolSummary is a short form of Pool. +type PoolSummary struct { + ID string `json:"id"` + Name string `json:"name"` + Free *int `json:"free,omitempty"` + TotalIPs *int `json:"total_ips,omitempty"` + Links []Link `json:"links,omitempty"` +} + +// ListPoolsResponse respresents a summary list of all pools. +type ListPoolsResponse struct { + Pools []PoolSummary `json:"pools"` +} + +// NewIPAddressRequest is used to add a new external IP to a pool. +type NewIPAddressRequest struct { + IP string `json:"ip"` +} + +// NewAddressRequest is used to add a new IP or new subnet to a pool. +type NewAddressRequest struct { + Subnet *string `json:"subnet"` + IPs []NewIPAddressRequest `json:"ips"` +} + +// MappedIP represents a mapping of external IP -> instance IP. +type MappedIP struct { + ID string `json:"mapping_id"` + ExternalIP string `json:"external_ip"` + InternalIP string `json:"internal_ip"` + InstanceID string `json:"instance_id"` + TenantID string `json:"tenant_id"` + PoolID string `json:"pool_id"` + PoolName string `json:"pool_name"` + Links []Link `json:"links"` +} + +// MappedIPShort is a summary version of a MappedIP. +type MappedIPShort struct { + ID string `json:"mapping_id"` + ExternalIP string `json:"external_ip"` + InternalIP string `json:"internal_ip"` + Links []Link `json:"links"` +} + +// MapIPRequest is used to request that an external IP be assigned from a pool +// to a particular instance. +type MapIPRequest struct { + PoolName *string `json:"pool_name"` + InstanceID string `json:"instance_id"` +} diff --git a/ciao-scheduler/scheduler.go b/ciao-scheduler/scheduler.go index caed26af0..4282ce7ef 100644 --- a/ciao-scheduler/scheduler.go +++ b/ciao-scheduler/scheduler.go @@ -470,7 +470,23 @@ func (sched *ssntpSchedulerServer) sendStartFailureError(clientUUID string, inst glog.Warningf("Unable to dispatch: %v\n", reason) sched.ssntp.SendError(clientUUID, ssntp.StartFailure, payload) } -func (sched *ssntpSchedulerServer) getConcentratorUUID(event ssntp.Event, payload []byte) (string, error) { + +func (sched *ssntpSchedulerServer) getCommandConcentratorUUID(command ssntp.Command, payload []byte) (string, error) { + switch command { + default: + return "", fmt.Errorf("unsupported ssntp.Command type \"%s\"", command) + case ssntp.AssignPublicIP: + var cmd payloads.CommandAssignPublicIP + err := yaml.Unmarshal(payload, &cmd) + return cmd.AssignIP.ConcentratorUUID, err + case ssntp.ReleasePublicIP: + var cmd payloads.CommandReleasePublicIP + err := yaml.Unmarshal(payload, &cmd) + return cmd.ReleaseIP.ConcentratorUUID, err + } +} + +func (sched *ssntpSchedulerServer) getEventConcentratorUUID(event ssntp.Event, payload []byte) (string, error) { switch event { default: return "", fmt.Errorf("unsupported ssntp.Event type \"%s\"", event) @@ -485,18 +501,35 @@ func (sched *ssntpSchedulerServer) getConcentratorUUID(event ssntp.Event, payloa } } +func (sched *ssntpSchedulerServer) fwdCmdToCNCI(command ssntp.Command, payload []byte) (dest ssntp.ForwardDestination) { + // since the scheduler is the primary ssntp server, it needs to + // unwrap CNCI directed command payloads and forward to the right CNCI + + concentratorUUID, err := sched.getCommandConcentratorUUID(command, payload) + if err != nil || concentratorUUID == "" { + glog.Errorf("Bad %s command yaml. Unable to forward to CNCI.\n", command) + dest.SetDecision(ssntp.Discard) + return + } + + glog.V(2).Infof("Forwarding %s command to CNCI Agent %s\n", command.String(), concentratorUUID) + dest.AddRecipient(concentratorUUID) + + return dest +} + func (sched *ssntpSchedulerServer) fwdEventToCNCI(event ssntp.Event, payload []byte) (dest ssntp.ForwardDestination) { // since the scheduler is the primary ssntp server, it needs to - // unwrap event payloads and forward them to the approriate recipient + // unwrap CNCI directed event payloads and forward to the right CNCI - concentratorUUID, err := sched.getConcentratorUUID(event, payload) + concentratorUUID, err := sched.getEventConcentratorUUID(event, payload) if err != nil || concentratorUUID == "" { - glog.Errorf("Bad %s event yaml from, concentratorUUID == %s\n", event, concentratorUUID) + glog.Errorf("Bad %s event yaml. Unable to forward to CNCI.\n", event) dest.SetDecision(ssntp.Discard) return } - glog.V(2).Infof("Forwarding %s to %s\n", event.String(), concentratorUUID) + glog.V(2).Infof("Forwarding %s command to CNCI Agent%s\n", event.String(), concentratorUUID) dest.AddRecipient(concentratorUUID) return dest @@ -709,6 +742,10 @@ func (sched *ssntpSchedulerServer) CommandForward(controllerUUID string, command fallthrough case ssntp.EVACUATE: dest, instanceUUID = sched.fwdCmdToComputeNode(command, payload) + case ssntp.AssignPublicIP: + fallthrough + case ssntp.ReleasePublicIP: + dest = sched.fwdCmdToCNCI(command, payload) default: dest.SetDecision(ssntp.Discard) } @@ -971,6 +1008,18 @@ func setSSNTPForwardRules(sched *ssntpSchedulerServer) { Operand: ssntp.PublicIPAssigned, Dest: ssntp.Controller, }, + { // all AssignPublicIPFailure events go to all Controllers + Operand: ssntp.AssignPublicIPFailure, + Dest: ssntp.Controller, + }, + { // all PublicIPUnassigned events go to all Controllers + Operand: ssntp.PublicIPUnassigned, + Dest: ssntp.Controller, + }, + { // all UnassignPublicIPFailure events go to all Controllers + Operand: ssntp.UnassignPublicIPFailure, + Dest: ssntp.Controller, + }, { // all START command are processed by the Command forwarder Operand: ssntp.START, CommandForward: sched, @@ -1015,6 +1064,14 @@ func setSSNTPForwardRules(sched *ssntpSchedulerServer) { Operand: ssntp.DetachVolumeFailure, Dest: ssntp.Controller, }, + { // all AssignPublicIP commands are processed by the Command forwarder + Operand: ssntp.AssignPublicIP, + CommandForward: sched, + }, + { // all ReleasePublicIP commands are processed by the Command forwarder + Operand: ssntp.ReleasePublicIP, + CommandForward: sched, + }, } } diff --git a/ciao-scheduler/scheduler_ssntp_test.go b/ciao-scheduler/scheduler_ssntp_test.go index 0bf98421a..16c10e250 100644 --- a/ciao-scheduler/scheduler_ssntp_test.go +++ b/ciao-scheduler/scheduler_ssntp_test.go @@ -491,6 +491,17 @@ func TestPublicIPAssigned(t *testing.T) { } } +func TestPublicIPUnassigned(t *testing.T) { + controllerCh := controller.AddEventChan(ssntp.PublicIPUnassigned) + + go cnciAgent.SendPublicIPUnassignedEvent() + + _, err := controller.GetEventChanResult(controllerCh, ssntp.PublicIPUnassigned) + if err != nil { + t.Fatal(err) + } +} + func waitForController(uuid string) { for { server.controllerMutex.Lock() diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 277c8aeaa..7cca1487f 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -19,10 +19,10 @@ package configuration import ( "bytes" "io/ioutil" + "reflect" "syscall" "testing" - "github.com/01org/ciao/networking/libsnnet" "github.com/01org/ciao/payloads" "github.com/google/gofuzz" ) @@ -66,6 +66,7 @@ const fullValidConf = `configure: controller: volume_port: 8776 compute_port: 8774 + ciao_port: 8889 compute_ca: /etc/pki/ciao/compute_ca.pem compute_cert: /etc/pki/ciao/compute_key.pem identity_user: controller @@ -111,47 +112,13 @@ func TestBlobCorrectPayload(t *testing.T) { } func equalPayload(p1, p2 payloads.Configure) bool { - return (p1.Configure.Scheduler.ConfigStorageURI == p2.Configure.Scheduler.ConfigStorageURI && - - p1.Configure.Controller.VolumePort == p2.Configure.Controller.VolumePort && - p1.Configure.Controller.ComputePort == p2.Configure.Controller.ComputePort && - p1.Configure.Controller.HTTPSCACert == p2.Configure.Controller.HTTPSCACert && - p1.Configure.Controller.HTTPSKey == p2.Configure.Controller.HTTPSKey && - p1.Configure.Controller.IdentityUser == p2.Configure.Controller.IdentityUser && - p1.Configure.Controller.IdentityPassword == p2.Configure.Controller.IdentityPassword && - - libsnnet.EqualNetSlice(p1.Configure.Launcher.ComputeNetwork, p2.Configure.Launcher.ComputeNetwork) && - libsnnet.EqualNetSlice(p1.Configure.Launcher.ManagementNetwork, p2.Configure.Launcher.ManagementNetwork) && - p1.Configure.Launcher.DiskLimit == p2.Configure.Launcher.DiskLimit && - p1.Configure.Launcher.MemoryLimit == p2.Configure.Launcher.MemoryLimit && - - p1.Configure.ImageService.Type == p2.Configure.ImageService.Type && - p1.Configure.ImageService.URL == p2.Configure.ImageService.URL && - - p1.Configure.IdentityService.Type == p2.Configure.IdentityService.Type && - p1.Configure.IdentityService.URL == p2.Configure.IdentityService.URL) + return reflect.DeepEqual(p1, p2) } func emptyPayload(p payloads.Configure) bool { - return (p.Configure.Scheduler.ConfigStorageURI != "" && - - p.Configure.Controller.VolumePort != 0 && - p.Configure.Controller.ComputePort != 0 && - p.Configure.Controller.HTTPSCACert != "" && - p.Configure.Controller.HTTPSKey != "" && - p.Configure.Controller.IdentityUser != "" && - p.Configure.Controller.IdentityPassword != "" && - - len(p.Configure.Launcher.ComputeNetwork) > 0 && - len(p.Configure.Launcher.ManagementNetwork) > 0 && - p.Configure.Launcher.DiskLimit != false && - p.Configure.Launcher.MemoryLimit != false && - - p.Configure.ImageService.Type != "" && - p.Configure.ImageService.URL != "" && + p2 := payloads.Configure{} - p.Configure.IdentityService.Type != "" && - p.Configure.IdentityService.URL != "") + return reflect.DeepEqual(p, p2) } func fillPayload(conf *payloads.Configure) { @@ -212,6 +179,7 @@ func TestPayloadCorrectBlob(t *testing.T) { func saneDefaults(conf *payloads.Configure) bool { return (conf.Configure.Controller.VolumePort == 8776 && conf.Configure.Controller.ComputePort == 8774 && + conf.Configure.Controller.CiaoPort == 8889 && conf.Configure.ImageService.Type == payloads.Glance && conf.Configure.IdentityService.Type == payloads.Keystone && conf.Configure.Launcher.DiskLimit == true && diff --git a/networking/ciao-cnci-agent/client.go b/networking/ciao-cnci-agent/client.go index 4815455ce..44119bbc7 100644 --- a/networking/ciao-cnci-agent/client.go +++ b/networking/ciao-cnci-agent/client.go @@ -200,7 +200,14 @@ func processCommand(client *ssntpConn, cmd *cmdWrapper) { glog.Infof("Processing: CiaoCommandAssignPublicIP %v", c) err := assignPubIP(c) if err != nil { - glog.Infof("Error Processing: CiaoCommandAssignPublicIP %v", err) + glog.Errorf("Error Processing: CiaoCommandAssignPublicIP %v", err) + err = sendNetworkError(client, ssntp.AssignPublicIPFailure, c) + } else { + err = sendNetworkEvent(client, ssntp.PublicIPAssigned, c) + } + + if err != nil { + glog.Errorf("Unable to send event : %v", err) } }(cmd) @@ -211,7 +218,14 @@ func processCommand(client *ssntpConn, cmd *cmdWrapper) { glog.Infof("Processing: CiaoCommandReleasePublicIP %v", c) err := releasePubIP(c) if err != nil { - glog.Errorf("Error Processing: CiaoCommandReleasePublicIP %v", err) + glog.Errorf("Error Processing: CiaoCommandReleasePublicIP %v", c) + err = sendNetworkError(client, ssntp.UnassignPublicIPFailure, c) + } else { + err = sendNetworkEvent(client, ssntp.PublicIPUnassigned, c) + } + + if err != nil { + glog.Errorf("Unable to send event : %v", err) } }(cmd) diff --git a/networking/ciao-cnci-agent/network.go b/networking/ciao-cnci-agent/network.go index 20878484a..e551da68b 100644 --- a/networking/ciao-cnci-agent/network.go +++ b/networking/ciao-cnci-agent/network.go @@ -253,6 +253,89 @@ func cnciAddedMarshal(agentUUID string) ([]byte, error) { return yaml.Marshal(&cnciAdded) } +func publicIPAssignedMarshal(cmd *payloads.PublicIPCommand) ([]byte, error) { + var publicIPAssigned payloads.EventPublicIPAssigned + evt := &publicIPAssigned.AssignedIP + + evt.ConcentratorUUID = cmd.ConcentratorUUID + evt.InstanceUUID = cmd.InstanceUUID + evt.PublicIP = cmd.PublicIP + evt.PrivateIP = cmd.PrivateIP + + glog.Infoln("PublicIPAssignedMarshal Event ", publicIPAssigned) + + return yaml.Marshal(&publicIPAssigned) +} + +func publicIPUnassignedMarshal(cmd *payloads.PublicIPCommand) ([]byte, error) { + var publicIPUnassigned payloads.EventPublicIPUnassigned + evt := &publicIPUnassigned.UnassignedIP + + evt.ConcentratorUUID = cmd.ConcentratorUUID + evt.InstanceUUID = cmd.InstanceUUID + evt.PublicIP = cmd.PublicIP + evt.PrivateIP = cmd.PrivateIP + + glog.Infoln("PublicIPUnassignedMarshal Event ", publicIPUnassigned) + + return yaml.Marshal(&publicIPUnassigned) +} + +func publicIPFailureMarshal(reason payloads.PublicIPFailureReason, cmd *payloads.PublicIPCommand) ([]byte, error) { + var failure payloads.ErrorPublicIPFailure + + failure.ConcentratorUUID = cmd.ConcentratorUUID + failure.TenantUUID = cmd.TenantUUID + failure.InstanceUUID = cmd.InstanceUUID + failure.PublicIP = cmd.PublicIP + failure.PrivateIP = cmd.PrivateIP + failure.VnicMAC = cmd.VnicMAC + failure.Reason = reason + + glog.Infoln("publicIPFailureMarshal error ", failure) + + return yaml.Marshal(&failure) +} + +func sendNetworkError(client *ssntpConn, errorType ssntp.Error, errorInfo interface{}) error { + + if !client.isConnected() { + return fmt.Errorf("Unable to send %s %v", errorType, errorInfo) + } + + payload, err := generateNetErrorPayload(errorType, errorInfo) + if err != nil { + return fmt.Errorf("Unable parse ssntpError %s %v", err, errorInfo) + } + + n, err := client.SendError(errorType, payload) + if err != nil { + return fmt.Errorf("Unable to send %s %s %v %d", err.Error(), errorType, errorInfo, n) + } + + return nil +} + +func generateNetErrorPayload(errorType ssntp.Error, errorInfo interface{}) ([]byte, error) { + switch errorType { + case ssntp.AssignPublicIPFailure: + cmd, ok := errorInfo.(*payloads.PublicIPCommand) + if !ok { + return nil, fmt.Errorf("PublicIPAssign Invalid errorInfo [%T] %v", errorInfo, errorInfo) + } + return publicIPFailureMarshal(payloads.PublicIPAssignFailure, cmd) + case ssntp.UnassignPublicIPFailure: + cmd, ok := errorInfo.(*payloads.PublicIPCommand) + if !ok { + return nil, fmt.Errorf("PublicIPUnassign Invalid errorInfo [%T] %v", errorInfo, errorInfo) + } + return publicIPFailureMarshal(payloads.PublicIPReleaseFailure, cmd) + default: + return nil, fmt.Errorf("Unsupported ssntpErrorInfo type: %v", errorType) + } + +} + func sendNetworkEvent(client *ssntpConn, eventType ssntp.Event, eventInfo interface{}) error { if !client.isConnected() { @@ -279,8 +362,19 @@ func generateNetEventPayload(eventType ssntp.Event, eventInfo interface{}, agent glog.Infof("generating cnciAdded Event Payload %s", agentUUID) return cnciAddedMarshal(agentUUID) case ssntp.PublicIPAssigned: - glog.Infof("generating publicIP Assigned Event Payload %s", agentUUID) - return nil, nil + glog.Infof("generating publicIP Assigned Event Payload %v", eventInfo) + cmd, ok := eventInfo.(*payloads.PublicIPCommand) + if !ok { + return nil, fmt.Errorf("PublicIPAssigned Invalid eventInfo [%T] %v", eventInfo, eventInfo) + } + return publicIPAssignedMarshal(cmd) + case ssntp.PublicIPUnassigned: + glog.Infof("generating publicIP Unassigned Event Payload %v", eventInfo) + cmd, ok := eventInfo.(*payloads.PublicIPCommand) + if !ok { + return nil, fmt.Errorf("PublicIPUnassigned Invalid eventInfo [%T] %v", eventInfo, eventInfo) + } + return publicIPUnassignedMarshal(cmd) default: return nil, fmt.Errorf("Unsupported ssntpEventInfo type: %v", eventType) } @@ -306,13 +400,13 @@ func unmarshallPubIP(cmd *payloads.PublicIPCommand) (net.IP, net.IP, error) { func assignPubIP(cmd *payloads.PublicIPCommand) error { prIP, puIP, err := unmarshallPubIP(cmd) - if err != nil { - glog.Errorf("cnci.assignPubIP invalid params %v %v", err, cmd) + return fmt.Errorf("cnci.assignPubIP invalid params %v %v", err, cmd) } - if enableNetwork { - glog.Infof("cnci.assignPubIP success %v %v %v", prIP, puIP, cmd) + err = gFw.PublicIPAccess(libsnnet.FwEnable, prIP, puIP, gCnci.ComputeLink[0].Attrs().Name) + if err != nil { + return fmt.Errorf("%v", err) } return nil @@ -321,13 +415,13 @@ func assignPubIP(cmd *payloads.PublicIPCommand) error { func releasePubIP(cmd *payloads.PublicIPCommand) error { prIP, puIP, err := unmarshallPubIP(cmd) - if err != nil { - glog.Errorf("cnci.releasePubIP invalid params %v %v", err, cmd) + return fmt.Errorf("cnci.releasePubIP invalid params %v %v", err, cmd) } - if enableNetwork { - glog.Infof("cnci.releasePubIP success %v %v %v", prIP, puIP, cmd) + err = gFw.PublicIPAccess(libsnnet.FwDisable, prIP, puIP, gCnci.ComputeLink[0].Attrs().Name) + if err != nil { + return fmt.Errorf("%v", err) } return nil diff --git a/networking/libsnnet/README.md b/networking/libsnnet/README.md index ff4268c0b..17caa09cf 100644 --- a/networking/libsnnet/README.md +++ b/networking/libsnnet/README.md @@ -64,6 +64,12 @@ unit test framework. The tests can be run as follows sudo ip link add testdummy type dummy sudo ip addr add 198.51.100.1/24 dev testdummy export SNNET_ENV=198.51.100.0/24 +export FWIFINT_ENV=testdummy + +sudo ip link add extdummy type dummy +sudo ip addr add 203.0.113.1/24 dev extdummy +export FWIF_ENV=extdummy + sudo -E go test --tags travis -v --short ``` diff --git a/networking/libsnnet/firewall.go b/networking/libsnnet/firewall.go index 2e1ad35c3..95b12626f 100644 --- a/networking/libsnnet/firewall.go +++ b/networking/libsnnet/firewall.go @@ -24,6 +24,7 @@ import ( "strconv" "github.com/coreos/go-iptables/iptables" + "github.com/vishvananda/netlink" ) /* https://wiki.archlinux.org/index.php/iptables @@ -105,6 +106,37 @@ func InitFirewall(devices ...string) (*Firewall, error) { IPTables: ipt, } + // create CIAO Floating IPs user defined chains + floatingIPsChains := []string{"ciao-floating-ip-pre", "ciao-floating-ip-post"} + for _, chain := range floatingIPsChains { + // verify it exists if not create it + _ = ipt.NewChain("nat", chain) + } + + // insert ciao-floating-ip-pre into PREROUTING Chain + ok, err := ipt.Exists("nat", "PREROUTING", "-j", "ciao-floating-ip-pre") + if err != nil { + return nil, fmt.Errorf("Error: InitFirewall could not verify existence of chain ciao-floating-ip-pre, %v", err) + } + if !ok { + err := ipt.Insert("nat", "PREROUTING", 1, "-j", "ciao-floating-ip-pre") + if err != nil { + return nil, fmt.Errorf("Error: InitFirewall could not create ciao-floating-ip-pre chain") + } + } + + // insert ciao-floating-ip-post into POSTROUTING Chain + ok, err = ipt.Exists("nat", "POSTROUTING", "-j", "ciao-floating-ip-post") + if err != nil { + return nil, fmt.Errorf("Error: InitFirewall could not verify existence of chain ciao-floating-ip-post, %v", err) + } + if !ok { + err := ipt.Insert("nat", "POSTROUTING", 1, "-j", "ciao-floating-ip-post") + if err != nil { + return nil, fmt.Errorf("Error: InitFirewall could not create ciao-floating-ip-post chain") + } + } + for _, device := range devices { //iptables -t nat -A POSTROUTING -o $device -j MASQUERADE @@ -276,8 +308,6 @@ func (f *Firewall) ExtPortAccess(action FwAction, protocol string, extDevice str return nil } -/* Not implemented - func ipAssign(action FwAction, ip net.IP, iface string) error { link, err := netlink.LinkByName(iface) @@ -328,95 +358,109 @@ func ipAssign(action FwAction, ip net.IP, iface string) error { } return nil - } -*/ - -/* Not implemented - //PublicIPAccess Enables/Disables public access to an internal IP -//TODO: Consider NATing only when exiting -//TODO: Create our own tables vs using default one func (f *Firewall) PublicIPAccess(action FwAction, internalIP net.IP, publicIP net.IP, extInterface string) error { + intIP := internalIP.String() + pubIP := publicIP.String() + switch action { case FwEnable: - + // assign the pubIP to the cnci agent err := ipAssign(FwEnable, publicIP, extInterface) if err != nil { return fmt.Errorf("Public IP Assignment failure %v", err) } - - //iptables -t nat -A PREROUTING -d $publicIP/32 -j DNAT --to-destination $internalIP - err = f.AppendUnique("nat", "PREROUTING", - "-d", publicIP.String()+"/32", "-j", "DNAT", "--to-destination", internalIP.String()) - + return enablePublicIP(intIP, pubIP) + case FwDisable: + // remove the pubIP from the cnci agent + err := ipAssign(FwDisable, publicIP, extInterface) if err != nil { - ok, err2 := f.Exists("nat", "PREROUTING", - "-d", publicIP.String()+"/32", "-j", "DNAT", "--to-destination", internalIP.String()) - - if !ok { - err = fmt.Errorf("Unable to perform public IP PREROUTING %v %s %s [%v],[%v]", - action, internalIP, publicIP, err, err2) - } + return fmt.Errorf("Public IP Assignment failure %v", err) } - //iptables -t nat -A POSTROUTING -s $internalIP/32 -j SNAT -–to-source $publicIP - err = f.AppendUnique("nat", "POSTROUTING", - "-s", internalIP.String()+"/32", "-j", "SNAT", "--to-source", publicIP.String()) + return disablePublicIP(intIP, pubIP) + default: + return fmt.Errorf("Invalid parameter %v", action) + } +} - if err != nil { - ok, err2 := f.Exists("nat", "POSTROUTING", - "-s", internalIP.String()+"/32", "-j", "SNAT", "--to-source", publicIP.String()) +func enablePublicIP(intIP, pubIP string) error { + ipt, err := iptables.New() + if err != nil { + return fmt.Errorf("initFirewall: Unable to setup iptables %v", err) + } - if !ok { - err = fmt.Errorf("Unable to perform public IP POSTROUTNG %v %s %s [%v],[%v]", - action, internalIP, publicIP, err, err2) - } + // insert DNAT rule of PREROUTING + // iptables -t nat -I ciao-floating-ip-pre -d -j DNAT --to-destination + ok, err := ipt.Exists("nat", "ciao-floating-ip-pre", "-d", pubIP+"/32", "-j", "DNAT", "--to-destination", intIP) + if err != nil { + return fmt.Errorf("Error: InitFirewall could not verify existence of PREROUTING rule %s to %s", pubIP, intIP) + } + + if !ok { + err := ipt.Insert("nat", "ciao-floating-ip-pre", 1, "-d", pubIP+"/32", "-j", "DNAT", "--to-destination", intIP) + if err != nil { + return fmt.Errorf("Could not insert firewall PREROUTING rule %s to %s into chain ciao-floating-ip-pre", pubIP, intIP) } + } - return err + // insert SNAT rule of POSTROUTING + // iptables -t nat -I ciao-floating-ip-post -s -j SNAT --to-source + ok, err = ipt.Exists("nat", "ciao-floating-ip-post", "-s", intIP+"/32", "-j", "SNAT", "--to-source", pubIP) + if err != nil { + return fmt.Errorf("Error: InitFirewall could not verify existence of POSTROUTING rule %s to %s", intIP, pubIP) + } - case FwDisable: - err := ipAssign(FwDisable, publicIP, extInterface) + if !ok { + err := ipt.Insert("nat", "ciao-floating-ip-post", 1, "-s", intIP+"/32", "-j", "SNAT", "--to-source", pubIP) if err != nil { - return fmt.Errorf("Public IP Assignment failure %v", err) + return fmt.Errorf("Could not insert firewall POSTROUTING rule %s to %s into chain ciao-floating-ip-post", intIP, pubIP) } + } - //iptables -t nat -D PREROUTING -d $publicIP/32 -j DNAT –to-destination $internalIP - err = f.Delete("nat", "PREROUTING", - "-d", publicIP.String()+"/32", "-j", "DNAT", "--to-destination", internalIP.String()) - if err != nil { - ok, err1 := f.Exists("nat", "PREROUTING", - "-d", publicIP.String()+"/32", "-j", "DNAT", "--to-destination", internalIP.String()) - if ok { - return fmt.Errorf("Unable to disable public IP PREROUTING %s %s %v %v", - publicIP, internalIP, err, err1) + return nil +} - } +func disablePublicIP(intIP, pubIP string) error { + ipt, err := iptables.New() + if err != nil { + return fmt.Errorf("initFirewall: Unable to setup iptables %v", err) + } + + // delete DNAT PREROUTING rule + // iptables -t nat -D ciao-floating-ip-pre -d -j DNAT --to-destination + ok, err := ipt.Exists("nat", "ciao-floating-ip-pre", "-d", pubIP+"/32", "-j", "DNAT", "--to-destination", intIP) + if err != nil { + return fmt.Errorf("Error: InitFirewall could not verify existence of PREROUTING rule %s to %s", pubIP, intIP) + } + + if ok { + err := ipt.Delete("nat", "ciao-floating-ip-pre", "-d", pubIP+"/32", "-j", "DNAT", "--to-destination", intIP) + if err != nil { + return fmt.Errorf("Could not delete firewall PREROUTING rule %s to %s into chain ciao-floating-ip-pre", pubIP, intIP) } + } - //iptables -t nat -D POSTROUTING -s $internalIP/32 -j SNAT –to-source $publicIP - err = f.Delete("nat", "POSTROUTING", - "-s", internalIP.String()+"/32", "-j", "SNAT", "--to-source", publicIP.String()) + // delete SNAT POSTROUTING rule + // iptables -t nat -D ciao-floating-ip-post -s -j SNAT --to-source + ok, err = ipt.Exists("nat", "ciao-floating-ip-post", "-s", intIP+"/32", "-j", "SNAT", "--to-source", pubIP) + if err != nil { + return fmt.Errorf("Error: InitFirewall could not verify existence of POSTROUTING rule %s to %s", intIP, pubIP) + } + if ok { + err := ipt.Delete("nat", "ciao-floating-ip-post", "-s", intIP+"/32", "-j", "SNAT", "--to-source", pubIP) if err != nil { - ok, err1 := f.Exists("nat", "POSTROUTING", - "-s", internalIP.String()+"/32", "-j", "SNAT", "--to-source", publicIP.String()) - if ok { - return fmt.Errorf("Unable to disable public IP POSTROUTING %s %s %v %v", - publicIP, internalIP, err, err1) - } + return fmt.Errorf("Could not delete firewall POSTROUTING rule %s to %s into chain ciao-floating-ip-post", intIP, pubIP) } - return nil - default: - return fmt.Errorf("Invalid parameter %v", action) } -} -*/ + return nil +} //DumpIPTables provides a utility routine that returns //the current state of the iptables diff --git a/networking/libsnnet/firewall_test.go b/networking/libsnnet/firewall_test.go index c8a30914a..4e9ac1797 100644 --- a/networking/libsnnet/firewall_test.go +++ b/networking/libsnnet/firewall_test.go @@ -32,13 +32,13 @@ func fwinit() { fwIf = os.Getenv("FWIF_ENV") if fwIf == "" { - fwIf = "eth0" + fwIf = "extdummy" } fwIfInt = os.Getenv("FWIFINT_ENV") if fwIfInt == "" { - fwIfInt = "eth1" + fwIfInt = "testdummy" } } @@ -69,11 +69,11 @@ func TestFw_Ssh(t *testing.T) { require.Nil(t, err) err = fw.ExtPortAccess(FwEnable, "tcp", fwIf, 12345, - net.ParseIP("192.168.0.101"), 22) + net.ParseIP("192.51.100.101"), 22) assert.Nil(err) err = fw.ExtPortAccess(FwDisable, "tcp", fwIf, 12345, - net.ParseIP("192.168.0.101"), 22) + net.ParseIP("192.51.100.101"), 22) assert.Nil(err) err = fw.ShutdownFirewall() @@ -104,12 +104,12 @@ func TestFw_Nat(t *testing.T) { assert.Nil(err) } -/* -//Not fully implemented +//Test assigment and removeal of floating IP // -//Not fully implemented +//Test if given a private IP and Public IP can be +//assinged and removed as floating IP // -//Expected to pass +//Test is expected to pass func TestFw_PublicIP(t *testing.T) { fwinit() fw, err := InitFirewall(fwIf) @@ -117,8 +117,8 @@ func TestFw_PublicIP(t *testing.T) { t.Fatalf("Error: InitFirewall %v %v %v", fwIf, err, fw) } - intIP := net.ParseIP("192.168.0.101") - pubIP := net.ParseIP("192.168.0.131") + intIP := net.ParseIP("198.51.100.1") + pubIP := net.ParseIP("198.51.100.100") err = fw.PublicIPAccess(FwEnable, intIP, pubIP, fwIfInt) if err != nil { @@ -135,7 +135,6 @@ func TestFw_PublicIP(t *testing.T) { t.Errorf("Error: Unable to shutdown firewall %v", err) } } -*/ //Exercises all valid CNCI Firewall APIs // @@ -157,7 +156,7 @@ func TestFw_All(t *testing.T) { assert.Nil(err) err = fw.ExtPortAccess(FwEnable, "tcp", fwIf, 12345, - net.ParseIP("192.168.0.101"), 22) + net.ParseIP("192.51.100.101"), 22) assert.Nil(err) procIPFwd := "/proc/sys/net/ipv4/ip_forward" @@ -169,10 +168,10 @@ func TestFw_All(t *testing.T) { } err = fw.ExtPortAccess(FwDisable, "tcp", fwIf, 12345, - net.ParseIP("192.168.0.101"), 22) + net.ParseIP("192.51.100.101"), 22) assert.Nil(err) - _, err = DebugSSHPortForIP(net.ParseIP("192.168.1.101")) + _, err = DebugSSHPortForIP(net.ParseIP("192.51.100.102")) assert.Nil(err) table := DumpIPTables() diff --git a/payloads/configure.go b/payloads/configure.go index 80424c424..c71b25753 100644 --- a/payloads/configure.go +++ b/payloads/configure.go @@ -68,6 +68,7 @@ type ConfigureScheduler struct { type ConfigureController struct { VolumePort int `yaml:"volume_port"` ComputePort int `yaml:"compute_port"` + CiaoPort int `yaml:"ciao_port"` HTTPSCACert string `yaml:"compute_ca"` HTTPSKey string `yaml:"compute_cert"` IdentityUser string `yaml:"identity_user"` @@ -117,6 +118,7 @@ type Configure struct { func (conf *Configure) InitDefaults() { conf.Configure.Controller.VolumePort = 8776 conf.Configure.Controller.ComputePort = 8774 + conf.Configure.Controller.CiaoPort = 8889 conf.Configure.ImageService.Type = Glance conf.Configure.IdentityService.Type = Keystone conf.Configure.Launcher.DiskLimit = true diff --git a/payloads/configure_test.go b/payloads/configure_test.go index 23f5d55ad..f9d3ecb3c 100644 --- a/payloads/configure_test.go +++ b/payloads/configure_test.go @@ -82,6 +82,8 @@ func TestConfigureMarshal(t *testing.T) { cfg.Configure.Controller.VolumePort = p p, _ = strconv.Atoi(testutil.ComputePort) cfg.Configure.Controller.ComputePort = p + p, _ = strconv.Atoi(testutil.CiaoPort) + cfg.Configure.Controller.CiaoPort = p cfg.Configure.Controller.HTTPSCACert = testutil.HTTPSCACert cfg.Configure.Controller.HTTPSKey = testutil.HTTPSKey cfg.Configure.Controller.IdentityUser = testutil.IdentityUser diff --git a/payloads/publicIPassigned.go b/payloads/publicIPassigned.go index 3f99941ae..ac39d9686 100644 --- a/payloads/publicIPassigned.go +++ b/payloads/publicIPassigned.go @@ -28,3 +28,8 @@ type PublicIPEvent struct { type EventPublicIPAssigned struct { AssignedIP PublicIPEvent `yaml:"public_ip_assigned"` } + +// EventPublicIPUnassigned represents the SSNTP PublicIPUnassigned event payload. +type EventPublicIPUnassigned struct { + UnassignedIP PublicIPEvent `yaml:"public_ip_unassigned"` +} diff --git a/payloads/publicIPassigned_test.go b/payloads/publicIPassigned_test.go index 2ce8ba85c..56409f7b9 100644 --- a/payloads/publicIPassigned_test.go +++ b/payloads/publicIPassigned_test.go @@ -65,3 +65,46 @@ func TestPublicIPAssignedMarshal(t *testing.T) { t.Errorf("PublicIPAssigned marshalling failed\n[%s]\n vs\n[%s]", string(y), testutil.AssignedIPYaml) } } + +func TestPublicIPUnassignedUnmarshal(t *testing.T) { + var unassignedIP EventPublicIPUnassigned + + err := yaml.Unmarshal([]byte(testutil.UnassignedIPYaml), &unassignedIP) + if err != nil { + t.Error(err) + } + + if unassignedIP.UnassignedIP.ConcentratorUUID != testutil.CNCIUUID { + t.Errorf("Wrong concentrator UUID field [%s]", unassignedIP.UnassignedIP.ConcentratorUUID) + } + + if unassignedIP.UnassignedIP.InstanceUUID != testutil.InstanceUUID { + t.Errorf("Wrong instance UUID field [%s]", unassignedIP.UnassignedIP.InstanceUUID) + } + + if unassignedIP.UnassignedIP.PublicIP != testutil.InstancePublicIP { + t.Errorf("Wrong public IP field [%s]", unassignedIP.UnassignedIP.PublicIP) + } + + if unassignedIP.UnassignedIP.PrivateIP != testutil.InstancePrivateIP { + t.Errorf("Wrong private IP field [%s]", unassignedIP.UnassignedIP.PrivateIP) + } +} + +func TestPublicIPUnassignedMarshal(t *testing.T) { + var unassignedIP EventPublicIPUnassigned + + unassignedIP.UnassignedIP.ConcentratorUUID = testutil.CNCIUUID + unassignedIP.UnassignedIP.InstanceUUID = testutil.InstanceUUID + unassignedIP.UnassignedIP.PublicIP = testutil.InstancePublicIP + unassignedIP.UnassignedIP.PrivateIP = testutil.InstancePrivateIP + + y, err := yaml.Marshal(&unassignedIP) + if err != nil { + t.Error(err) + } + + if string(y) != testutil.UnassignedIPYaml { + t.Errorf("PublicIPUnassigned marshalling failed\n[%s]\n vs\n[%s]", string(y), testutil.UnassignedIPYaml) + } +} diff --git a/ssntp/ssntp.go b/ssntp/ssntp.go index 3b498a56d..6f8344c6e 100644 --- a/ssntp/ssntp.go +++ b/ssntp/ssntp.go @@ -61,7 +61,7 @@ type Error uint8 // Event is the SSNTP Event operand. // It can be TenantAdded, TenantRemoval, InstanceDeleted, -// ConcentratorInstanceAdded, PublicIPAssigned, TraceReport, +// ConcentratorInstanceAdded, PublicIPAssigned, PublicIPUnassigned, TraceReport, // NodeConnected or NodeDisconnected type Event uint8 @@ -402,6 +402,23 @@ const ( // +----------------------------------------------------------------------------+ PublicIPAssigned + // PublicIPUnassigned events are sent by Networking concentrator + // instances (CNCI) to the Scheduler when they successfully + // unassigned a public IP from a given instance. + // + // The Scheduler must forward those events to the Controller. + // + // The PublicIPUnassigned event payload contains a previously assigned + // public IP, the instance private IP and the instance UUID. + // + // SSNTP PublicIPUnassigned Event frame + // + // +----------------------------------------------------------------------------+ + // | Major | Minor | Type | Operand | Payload Length | YAML formatted payload | + // | | | (0x3) | (0x4) | | | + // +----------------------------------------------------------------------------+ + PublicIPUnassigned + // TraceReport events carry a tracing report payload from one // of the SSNTP clients. // @@ -525,6 +542,14 @@ const ( // DetachVolumeFailure is sent by launcher agents to report a failure to detach // a volume from an instance. DetachVolumeFailure + + // AssignPublicIPFailure is sent by the CNCI when a an external IP + // cannot be assigned. + AssignPublicIPFailure + + // UnassignPublicIPFailure is sent by the CNCI when a an external IP + // cannot be unassigned. + UnassignPublicIPFailure ) // Major is the SSNTP protocol major version @@ -614,6 +639,8 @@ func (status Event) String() string { return "Network Concentrator Instance Added" case PublicIPAssigned: return "Public IP Assigned" + case PublicIPUnassigned: + return "Public IP Unassigned" case TraceReport: return "Trace Report" case NodeConnected: diff --git a/testutil/agent.go b/testutil/agent.go index 2c6b471d0..e857074d7 100644 --- a/testutil/agent.go +++ b/testutil/agent.go @@ -719,6 +719,18 @@ func (client *SsntpTestClient) SendPublicIPAssignedEvent() { go client.SendResultAndDelEventChan(ssntp.PublicIPAssigned, result) } +// SendPublicIPUnassignedEvent allows an SsntpTestClient to push an ssntp.PublicIPUnassigned event frame +func (client *SsntpTestClient) SendPublicIPUnassignedEvent() { + var result Result + + _, err := client.Ssntp.SendEvent(ssntp.PublicIPUnassigned, []byte(UnassignedIPYaml)) + if err != nil { + result.Err = err + } + + go client.SendResultAndDelEventChan(ssntp.PublicIPUnassigned, result) +} + // SendConcentratorAddedEvent allows an SsntpTestClient to push an ssntp.ConcentratorInstanceAdded event frame func (client *SsntpTestClient) SendConcentratorAddedEvent(instanceUUID string, tenantUUID string, ip string, vnicMAC string) { var result Result diff --git a/testutil/payloads.go b/testutil/payloads.go index f2fa40d0f..493629e6b 100644 --- a/testutil/payloads.go +++ b/testutil/payloads.go @@ -84,6 +84,9 @@ const VolumePort = "446" // ComputePort is a test port for the compute service const ComputePort = "443" +// CiaoPort is a test port for ciao's api service +const CiaoPort = "447" + // HTTPSKey is a path to a key for the compute service const HTTPSKey = "/etc/pki/ciao/compute_key.pem" @@ -311,6 +314,14 @@ const AssignedIPYaml = `public_ip_assigned: private_ip: ` + InstancePrivateIP + ` ` +// UnassignedIPYaml is a sample PublicIPUnassigned ssntp.Event payload for test cases +const UnassignedIPYaml = `public_ip_unassigned: + concentrator_uuid: ` + CNCIUUID + ` + instance_uuid: ` + InstanceUUID + ` + public_ip: ` + InstancePublicIP + ` + private_ip: ` + InstancePrivateIP + ` +` + // TenantAddedYaml is a sample TenantAdded ssntp.Event payload for test cases const TenantAddedYaml = `tenant_added: agent_uuid: ` + AgentUUID + ` @@ -346,6 +357,7 @@ const ConfigureYaml = `configure: controller: volume_port: ` + VolumePort + ` compute_port: ` + ComputePort + ` + ciao_port: ` + CiaoPort + ` compute_ca: ` + HTTPSCACert + ` compute_cert: ` + HTTPSKey + ` identity_user: ` + IdentityUser + ` diff --git a/testutil/singlevm/setup.sh b/testutil/singlevm/setup.sh index 6fd6b5f2d..ed15d983a 100755 --- a/testutil/singlevm/setup.sh +++ b/testutil/singlevm/setup.sh @@ -9,6 +9,8 @@ ciao_bin="$HOME/local" ciao_cert="$ciao_bin""/cert-Scheduler-""$ciao_host"".pem" keystone_key="$ciao_bin"/keystone_key.pem keystone_cert="$ciao_bin"/keystone_cert.pem +workload_sshkey="$ciao_bin"/testkey +workload_cloudinit="$ciao_bin"/workloads/test.yaml ciao_pki_path=/etc/pki/ciao export no_proxy=$no_proxy,$ciao_vlan_ip,$ciao_host @@ -56,6 +58,7 @@ export CIAO_ADMIN_USERNAME="$ciao_admin_username" export CIAO_ADMIN_PASSWORD="$ciao_admin_password" export CIAO_CA_CERT_FILE="$ciao_bin"/"CAcert-""$ciao_host"".pem" export CIAO_IDENTITY="$ciao_identity_url" +export CIAO_SSH_KEY="$workload_sshkey" # Save these vars for later use, too > "$ciao_env" # Clean out previous data @@ -217,6 +220,8 @@ then exit 1 fi + + #Generate Certificates "$GOPATH"/bin/ciao-cert -anchor -role scheduler -email="$ciao_email" \ -organization="$ciao_org" -host="$ciao_host" -ip="$ciao_vlan_ip" -verify @@ -276,6 +281,32 @@ cp -a "$ciao_src"/ciao-controller/workloads "$ciao_bin" cp -f "$ciao_scripts"/workloads/* "$ciao_bin"/workloads cp -f "$ciao_scripts"/tables/* "$ciao_bin"/tables +#Over ride the cloud-init configuration +echo "Generating workload ssh key $workload_sshkey" +rm -f "$workload_sshkey" "$workload_sshkey".pub +ssh-keygen -f "$workload_sshkey" -t rsa -N '' +test_sshkey=$(< "$workload_sshkey".pub) +chmod 600 "$workload_sshkey".pub +#Note: Password is set to ciao +test_passwd='$6$rounds=4096$w9I3hR4g/hu$AnYjaC2DfznbPSG3vxsgtgAS4mJwWBkcR74Y/KHNB5OsfAlA4gpU5j6CHWMOkkt9j.9d7OYJXJ4icXHzKXTAO.' + +echo "Generating workload cloud-init file $workload_cloudinit" +( +cat <<-EOF +--- +#cloud-config +users: + - name: demouser + gecos: CIAO Demo User + lock-passwd: false + passwd: ${test_passwd} + sudo: ALL=(ALL) NOPASSWD:ALL + ssh-authorized-keys: + - ${test_sshkey} +... +EOF +) > $workload_cloudinit + #Copy the launch scripts cp "$ciao_scripts"/run_scheduler.sh "$ciao_bin" diff --git a/testutil/singlevm/verify.sh b/testutil/singlevm/verify.sh index d80d598e1..49539606e 100755 --- a/testutil/singlevm/verify.sh +++ b/testutil/singlevm/verify.sh @@ -2,75 +2,188 @@ ciao_bin="$HOME/local" ciao_gobin="$GOPATH"/bin +event_counter=0 -# Read cluster env variables +#Utility functions +function exitOnError { + local exit_code="$1" + local error_string="$2" + + if [ $1 -ne 0 ] + then + echo "FATAL ERROR exiting: " "$error_string" "$exit_code" + exit 1 + fi +} + +#Checks that no network artifacts are left behind +function checkForNetworkArtifacts() { + + #Verify that there are no ciao related artifacts left behind + ciao_networks=`sudo docker network ls --filter driver=ciao -q | wc -l` + if [ $ciao_networks -ne 0 ] + then + echo "FATAL ERROR: ciao docker networks not cleaned up" + sudo docker network ls --filter driver=ciao + exit 1 + fi + + + #The only ciao interfaces left behind should be CNCI VNICs + #Once we can delete tenants we should not even have them around + cnci_vnics=`ip -d link | grep alias | grep cnci | wc -l` + ciao_vnics=`ip -d link | grep alias | wc -l` + + if [ $cnci_vnics -ne $ciao_vnics ] + then + echo "FATAL ERROR: ciao network interfaces not cleaned up" + ip -d link | grep alias + exit 1 + fi +} + +#There are too many failsafes in the CNCI. Hence just disable iptables utility to trigger failure +#This also ensures that the CNCI is always left in a consistent state (sans the permission) +function triggerIPTablesFailure { + ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" <<-EOF + sudo chmod -x /usr/bin/iptables + EOF +} + +#Restore the iptables so that the cluster is usable +function restoreIPTables { + ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$ssh_ip" <<-EOF + sudo chmod +x /usr/bin/iptables + EOF +} + +function clearAllEvents() { + #Clear out all prior events + "$ciao_gobin"/ciao-cli event delete + + #Wait for the event count to drop to 0 + retry=0 + ciao_events=0 + + until [ $retry -ge 6 ] + do + ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'` + + if [ $ciao_events -eq 0 ] + then + break + fi + let retry=retry+1 + sleep 1 + done + + exitOnError $ciao_events "ciao events not deleted properly" +} + +function checkEventStatus { + local event_index="$1" + local event_code="$2" + local retry=0 + local ciao_events=0 + local total_events=0 + local code="" + + #We only need to wait for as many events as the index + total_events=$((event_index + 1)) + + until [ $retry -ge 6 ] + do + ciao_events=`"$ciao_gobin"/ciao-cli event list -f '{{len .}}'` + + if [ $ciao_events -eq $total_events ] + then + break + fi + + let retry=retry+1 + sleep 1 + done + + if [ $ciao_events -ne $total_events ] + then + echo "FATAL ERROR: ciao event not reported. Events seen =" $ciao_events + "$ciao_gobin"/ciao-cli event list + exit 1 + fi + + code=$("$ciao_gobin"/ciao-cli event list -f "{{(index . ${event_index}).Message}}" | cut -d ' ' -f 1) + + if [ "$event_code" != "$code" ] + then + echo "FATAL ERROR: Unknown event $code. Looking for $event_code" + "$ciao_gobin"/ciao-cli event list + exit 1 + fi + + "$ciao_gobin"/ciao-cli event list +} + +function createExternalIPPool() { + # first create a new external IP pool and add a subnet to it. + # this is an admin only operation, so make sure our env variables + # are set accordingly. Since user admin might belong to more than one + # tenant, make sure to specify that we are logging in as part of the + # "admin" tenant/project. + ciao_user=$CIAO_USERNAME + ciao_passwd=$CIAO_PASSWORD + export CIAO_USERNAME=$CIAO_ADMIN_USERNAME + export CIAO_PASSWORD=$CIAO_ADMIN_PASSWORD + "$ciao_gobin"/ciao-cli -tenant-name admin pool create -name test + "$ciao_gobin"/ciao-cli -tenant-name admin pool add -subnet 203.0.113.0/24 -name test + export CIAO_USERNAME=$ciao_user + export CIAO_PASSWORD=$ciao_passwd +} + +function deleteExternalIPPool() { + #Cleanup the pool + export CIAO_USERNAME=$CIAO_ADMIN_USERNAME + export CIAO_PASSWORD=$CIAO_ADMIN_PASSWORD + + "$ciao_gobin"/ciao-cli -tenant-name admin pool delete -name test + exitOnError $? "Unable to delete pool" + + export CIAO_USERNAME=$ciao_user + export CIAO_PASSWORD=$ciao_passwd +} + +# Read cluster env variables . $ciao_bin/demo.sh "$ciao_gobin"/ciao-cli workload list - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to list workloads" - exit 1 -fi +exitOnError $? "Unable to list workloads" "$ciao_gobin"/ciao-cli instance add --workload=e35ed972-c46c-4aad-a1e7-ef103ae079a2 --instances=2 - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to launch VMs" - exit 1 -fi +exitOnError $? "Unable to launch VMs" "$ciao_gobin"/ciao-cli instance list - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to list instances" - exit 1 -fi +exitOnError $? "Unable to list instances" #Launch containers - #Pre-cache the image to reduce the start latency sudo docker pull debian "$ciao_gobin"/ciao-cli instance add --workload=ca957444-fa46-11e5-94f9-38607786d9ec --instances=1 - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to launch containers" - exit 1 -fi +exitOnError $? "Unable to launch containers" sleep 5 "$ciao_gobin"/ciao-cli instance list -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to list instances" - exit 1 -fi +exitOnError $? "Unable to list instances" container_1=`sudo docker ps -q -l` container_1_ip=`sudo docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_1` "$ciao_gobin"/ciao-cli instance add --workload=ca957444-fa46-11e5-94f9-38607786d9ec --instances=1 - -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to launch containers" - exit 1 -fi - +exitOnError $? "Unable to launch containers" sleep 5 "$ciao_gobin"/ciao-cli instance list -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to list instances" - exit 1 -fi +exitOnError $? "Unable to list instances" container_2=`sudo docker ps -q -l` @@ -118,108 +231,93 @@ fi echo "Checking Docker Networking" sudo docker exec $container_2 /bin/ping -c 3 $container_1_ip -if [ $? -ne 0 ] -then - echo "FATAL ERROR: Unable to ping across containers" - exit 1 -else - echo "Container connectivity verified" -fi +exitOnError $? "Unable to ping across containers" +echo "Container connectivity verified" #Clear out all prior events -"$ciao_gobin"/ciao-cli event delete +clearAllEvents -#Wait for the event count to drop to 0 -retry=0 -ciao_events=0 +#Test External IP Assignment support +#Pick the first instance which is a VM, as we can even SSH into it +#We have already checked that the VM is up. +createExternalIPPool -until [ $retry -ge 6 ] -do - ciao_events=`"$ciao_gobin"/ciao-cli event list | grep "0 Ciao event" | wc -l` +testinstance=`"$ciao_gobin"/ciao-cli instance list -f '{{with index . 0}}{{.ID}}{{end}}'` +"$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test - if [ $ciao_events -eq 1 ] - then - break - fi +#Wait for the CNCI to report successful map +checkEventStatus $event_counter "Mapped" - let retry=retry+1 - sleep 1 -done +"$ciao_gobin"/ciao-cli event list +"$ciao_gobin"/ciao-cli external-ip list -if [ $ciao_events -ne 1 ] -then - echo "FATAL ERROR: ciao events not deleted properly" - "$ciao_gobin"/ciao-cli event list - exit 1 -fi +#We checked the event, so the mapping should exist +testip=`"$ciao_gobin"/ciao-cli external-ip list -f '{{with index . 0}}{{.ExternalIP}}{{end}}'` +test_instance=`"$ciao_gobin"/ciao-cli instance list -f '{{with index . 0}}{{.ID}}{{end}}'` -#Now delete all instances -"$ciao_gobin"/ciao-cli instance delete --all +sudo ip route add 203.0.113.0/24 dev ciaovlan +ping -c 3 $testip +ping_result=$? +#Make sure we are able to reach the VM +test_hostname=`ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$testip" hostname` +sudo ip route del 203.0.113.0/24 dev ciaovlan -if [ $? -ne 0 ] +exitOnError $ping_result "Unable to ping external IP" + +if [ "$test_hostname" == "$test_instance" ] then - echo "FATAL ERROR: Unable to delete instances" + echo "SSH connectivity using external IP verified" +else + echo "FATAL ERROR: Unable to ssh via external IP" exit 1 fi -"$ciao_gobin"/ciao-cli instance list +"$ciao_gobin"/ciao-cli external-ip unmap -address $testip -#Wait for all the instance deletions to be reported back -retry=0 -ciao_events=0 +#Wait for the CNCI to report successful unmap +event_counter=$((event_counter+1)) +checkEventStatus $event_counter "Unmapped" -until [ $retry -ge 6 ] -do - ciao_events=`"$ciao_gobin"/ciao-cli event list | grep "4 Ciao event(s)" | wc -l` +"$ciao_gobin"/ciao-cli external-ip list - if [ $ciao_events -eq 1 ] - then - break - fi +#Test for External IP Failures - let retry=retry+1 - sleep 1 -done +#Map failure +triggerIPTablesFailure +"$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test +#Wait for the CNCI to report unsuccessful map +event_counter=$((event_counter+1)) +checkEventStatus $event_counter "Failed" +restoreIPTables -if [ $ciao_events -ne 1 ] -then - echo "FATAL ERROR: ciao instances not deleted properly" - "$ciao_gobin"/ciao-cli event list - exit 1 -fi +#Unmap failure +"$ciao_gobin"/ciao-cli external-ip map -instance $testinstance -pool test +event_counter=$((event_counter+1)) +checkEventStatus $event_counter "Mapped" -#Wait around a bit as instance delete is asynchronous -retry=0 -ciao_networks=0 -until [ $retry -ge 6 ] -do - #Verify that there are no ciao related artifacts left behind - ciao_networks=`sudo docker network ls --filter driver=ciao -q | wc -l` +triggerIPTablesFailure +"$ciao_gobin"/ciao-cli external-ip unmap -address $testip +event_counter=$((event_counter+1)) +checkEventStatus $event_counter "Failed" +restoreIPTables - if [ $ciao_networks -eq 0 ] - then - break - fi - let retry=retry+1 - sleep 1 -done +#Cleanup +"$ciao_gobin"/ciao-cli external-ip unmap -address $testip +event_counter=$((event_counter+1)) +checkEventStatus $event_counter "Unmapped" -if [ $ciao_networks -ne 0 ] -then - echo "FATAL ERROR: ciao docker networks not cleaned up" - sudo docker network ls --filter driver=ciao - exit 1 -fi +#Cleanup pools +deleteExternalIPPool + +#Now delete all instances +"$ciao_gobin"/ciao-cli instance delete --all +exitOnError $? "Unable to delete instances" +"$ciao_gobin"/ciao-cli instance list -#The only ciao interfaces left behind should be CNCI VNICs -#Once we can delete tenants we should not even have them around -cnci_vnics=`ip -d link | grep alias | grep cnci | wc -l` -ciao_vnics=`ip -d link | grep alias | wc -l` +#Wait for all the instance deletions to be reported back +event_counter=$((event_counter+4)) +checkEventStatus $event_counter "Deleted" -if [ $cnci_vnics -ne $ciao_vnics ] -then - echo "FATAL ERROR: ciao network interfaces not cleaned up" - ip -d link | grep alias - exit 1 -fi +#Verify that there are no ciao related artifacts left behind +checkForNetworkArtifacts