Add datasources CLI stubs (#5020)

* add: datasources cli stubs

* update: refactoring & enhancements

* add: licenses to the new files

* update: lints

* add: operations by name

* remove: redundant ctx from requests

* add: include cmd main file

cmd/cli/app/datasource/apply.go

@@ -0,0 +1,117 @@
+// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// Package datasource provides functionalities to manage and apply data sources.
+package datasource
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/mindersec/minder/internal/util"
+	"github.com/mindersec/minder/internal/util/cli"
+	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+)
+
+// applyCmd represents the datasource apply command
+var applyCmd = &cobra.Command{
+	Use:   "apply",
+	Short: "Apply a data source",
+	Long:  `The datasource apply subcommand lets you create or update data sources for a project within Minder.`,
+	RunE:  cli.GRPCClientWrapRunE(applyCommand),
+}
+
+func init() {
+	DataSourceCmd.AddCommand(applyCmd)
+	// Flags
+	applyCmd.Flags().StringArrayP("file", "f", []string{},
+		"Path to the YAML defining the data source (or - for stdin). Can be specified multiple times. Can be a directory.")
+	// Required
+	if err := applyCmd.MarkFlagRequired("file"); err != nil {
+		applyCmd.Printf("Error marking flag required: %s", err)
+		os.Exit(1)
+	}
+}
+
+// applyCommand is the datasource apply subcommand
+func applyCommand(_ context.Context, cmd *cobra.Command, _ []string, conn *grpc.ClientConn) error {
+	client := minderv1.NewDataSourceServiceClient(conn)
+
+	project := viper.GetString("project")
+
+	fileFlag, err := cmd.Flags().GetStringArray("file")
+	if err != nil {
+		return cli.MessageAndError("Error parsing file flag", err)
+	}
+
+	if err = validateFilesArg(fileFlag); err != nil {
+		return cli.MessageAndError("Error validating file flag", err)
+	}
+
+	files, err := util.ExpandFileArgs(fileFlag...)
+	if err != nil {
+		return cli.MessageAndError("Error expanding file args", err)
+	}
+
+	// No longer print usage on returned error, since we've parsed our inputs
+	// See https://github.com/spf13/cobra/issues/340#issuecomment-374617413
+	cmd.SilenceUsage = true
+
+	table := initializeTableForList()
+
+	applyFunc := func(ctx context.Context, fileName string, ds *minderv1.DataSource) (*minderv1.DataSource, error) {
+		createResp, err := client.CreateDataSource(ctx, &minderv1.CreateDataSourceRequest{
+			DataSource: ds,
+		})
+
+		if err == nil {
+			return createResp.DataSource, nil
+		}
+
+		st, ok := status.FromError(err)
+		if !ok {
+			// We can't parse the error, so just return it
+			return nil, fmt.Errorf("error creating data source from %s: %w", fileName, err)
+		}
+
+		if st.Code() != codes.AlreadyExists {
+			return nil, fmt.Errorf("error creating data source from %s: %w", fileName, err)
+		}
+
+		updateResp, err := client.UpdateDataSource(ctx, &minderv1.UpdateDataSourceRequest{
+			DataSource: ds,
+		})
+
+		if err != nil {
+			return nil, fmt.Errorf("error updating data source from %s: %w", fileName, err)
+		}
+
+		return updateResp.DataSource, nil
+	}
+
+	for _, f := range files {
+		if f.Path != "-" && shouldSkipFile(f.Path) {
+			continue
+		}
+		// cmd.Context() is the root context. We need to create a new context for each file
+		// so we can avoid the timeout.
+		if err = executeOnOneDataSource(cmd.Context(), table, f.Path, os.Stdin, project, applyFunc); err != nil {
+			if f.Expanded && minderv1.YouMayHaveTheWrongResource(err) {
+				cmd.PrintErrf("Skipping file %s: not a data source\n", f.Path)
+				// We'll skip the file if it's not a data source
+				continue
+			}
+			return cli.MessageAndError(fmt.Sprintf("error applying data source from %s", f.Path), err)
+		}
+	}
+	// Render the table
+	table.Render()
+	return nil
+}

cmd/cli/app/datasource/common.go

@@ -0,0 +1,148 @@
+// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package datasource
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/viper"
+
+	"github.com/mindersec/minder/internal/util"
+	"github.com/mindersec/minder/internal/util/cli"
+	"github.com/mindersec/minder/internal/util/cli/table"
+	"github.com/mindersec/minder/internal/util/cli/table/layouts"
+	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+)
+
+// executeOnOneDataSource executes a function on a single data source
+func executeOnOneDataSource(
+	ctx context.Context,
+	t table.Table,
+	f string,
+	dashOpen io.Reader,
+	proj string,
+	exec func(context.Context, string, *minderv1.DataSource) (*minderv1.DataSource, error),
+) error {
+	ctx, cancel := cli.GetAppContext(ctx, viper.GetViper())
+	defer cancel()
+
+	reader, closer, err := util.OpenFileArg(f, dashOpen)
+	if err != nil {
+		return fmt.Errorf("error opening file arg: %w", err)
+	}
+	defer closer()
+
+	ds := &minderv1.DataSource{}
+	if err := minderv1.ParseResourceProto(reader, ds); err != nil {
+		return fmt.Errorf("error parsing data source: %w", err)
+	}
+
+	// Override the YAML specified project with the command line argument
+	if proj != "" {
+		if ds.Context == nil {
+			ds.Context = &minderv1.ContextV2{}
+		}
+
+		ds.Context.ProjectId = proj
+	}
+
+	// create or update the data source
+	createdDS, err := exec(ctx, f, ds)
+	if err != nil {
+		return err
+	}
+
+	// add the data source to the table rows
+	name := appendDataSourcePropertiesToName(createdDS)
+	t.AddRow(
+		createdDS.Context.ProjectId,
+		createdDS.Id,
+		name,
+		cli.ConcatenateAndWrap(createdDS.Name, 20),
+	)
+
+	return nil
+}
+
+// validateFilesArg validates the file arguments
+func validateFilesArg(files []string) error {
+	if files == nil {
+		return fmt.Errorf("error: file must be set")
+	}
+
+	if contains(files, "") {
+		return fmt.Errorf("error: file must be set")
+	}
+
+	if contains(files, "-") && len(files) > 1 {
+		return fmt.Errorf("error: cannot use stdin with other files")
+	}
+
+	return nil
+}
+
+// contains checks if a slice contains a specific string
+func contains(slice []string, item string) bool {
+	for _, s := range slice {
+		if s == item {
+			return true
+		}
+	}
+	return false
+}
+
+// shouldSkipFile determines if a file should be skipped based on its extension
+func shouldSkipFile(f string) bool {
+	// if the file is not json or yaml, skip it
+	// Get file extension
+	ext := filepath.Ext(f)
+	switch ext {
+	case ".yaml", ".yml", ".json":
+		return false
+	default:
+		fmt.Fprintf(os.Stderr, "Skipping file %s: not a yaml or json file\n", f)
+		return true
+	}
+}
+
+// appendDataSourcePropertiesToName appends the data source properties to the name. The format is:
+// <name> (<properties>)
+// where <properties> is a comma separated list of properties.
+func appendDataSourcePropertiesToName(ds *minderv1.DataSource) string {
+	name := ds.Name
+	properties := []string{}
+	// add the type property if it is present
+	dType := getDataSourceType(ds)
+	if dType != "" {
+		properties = append(properties, fmt.Sprintf("type: %s", dType))
+	}
+
+	// add other properties as needed
+
+	// return the name with the properties if any
+	if len(properties) != 0 {
+		return fmt.Sprintf("%s (%s)", name, strings.Join(properties, ", "))
+	}
+
+	// return only name otherwise
+	return name
+}
+
+// getDataSourceType returns the type of data source
+func getDataSourceType(ds *minderv1.DataSource) string {
+	if ds.GetRest() != nil {
+		return "REST"
+	}
+	return "Unknown"
+}
+
+// initializeTableForList initializes the table for listing data sources
+func initializeTableForList() table.Table {
+	return table.New(table.Simple, layouts.DataSourceList, nil)
+}

cmd/cli/app/datasource/create.go

@@ -0,0 +1,98 @@
+// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package datasource
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
+
+	"github.com/mindersec/minder/internal/util"
+	"github.com/mindersec/minder/internal/util/cli"
+	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+)
+
+// createCmd represents the datasource create command
+var createCmd = &cobra.Command{
+	Use:   "create",
+	Short: "Create a data source",
+	Long:  `The datasource create subcommand lets you create new data sources for a project within Minder.`,
+	RunE:  cli.GRPCClientWrapRunE(createCommand),
+}
+
+func init() {
+	DataSourceCmd.AddCommand(createCmd)
+	// Flags
+	createCmd.Flags().StringArrayP("file", "f", []string{},
+		"Path to the YAML defining the data source (or - for stdin). Can be specified multiple times. Can be a directory.")
+	// Required
+	if err := createCmd.MarkFlagRequired("file"); err != nil {
+		createCmd.Printf("Error marking flag required: %s", err)
+		os.Exit(1)
+	}
+}
+
+// createCommand is the datasource create subcommand
+func createCommand(_ context.Context, cmd *cobra.Command, _ []string, conn *grpc.ClientConn) error {
+	client := minderv1.NewDataSourceServiceClient(conn)
+
+	project := viper.GetString("project")
+
+	fileFlag, err := cmd.Flags().GetStringArray("file")
+	if err != nil {
+		return cli.MessageAndError("Error parsing file flag", err)
+	}
+
+	if err = validateFilesArg(fileFlag); err != nil {
+		return cli.MessageAndError("Error validating file flag", err)
+	}
+
+	files, err := util.ExpandFileArgs(fileFlag...)
+	if err != nil {
+		return cli.MessageAndError("Error expanding file args", err)
+	}
+
+	// No longer print usage on returned error, since we've parsed our inputs
+	// See https://github.com/spf13/cobra/issues/340#issuecomment-374617413
+	cmd.SilenceUsage = true
+
+	table := initializeTableForList()
+
+	createFunc := func(ctx context.Context, _ string, ds *minderv1.DataSource) (*minderv1.DataSource, error) {
+		resp, err := client.CreateDataSource(ctx, &minderv1.CreateDataSourceRequest{
+			DataSource: ds,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		return resp.DataSource, nil
+	}
+
+	for _, f := range files {
+		if f.Path != "-" && shouldSkipFile(f.Path) {
+			continue
+		}
+		// cmd.Context() is the root context. We need to create a new context for each file
+		// so we can avoid the timeout.
+		if err = executeOnOneDataSource(cmd.Context(), table, f.Path, os.Stdin, project, createFunc); err != nil {
+			// We swallow errors if you're loading a directory to avoid failing
+			// on test files.
+			if f.Expanded && minderv1.YouMayHaveTheWrongResource(err) {
+				cmd.PrintErrf("Skipping file %s: not a data source\n", f.Path)
+				// We'll skip the file if it's not a data source
+				continue
+			}
+			return cli.MessageAndError(fmt.Sprintf("Error creating data source from %s", f.Path), err)
+		}
+	}
+
+	// Render the table
+	table.Render()
+	return nil
+}

cmd/cli/app/datasource/datasource.go

@@ -0,0 +1,24 @@
+// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package datasource
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/mindersec/minder/cmd/cli/app"
+)
+
+// DataSourceCmd is the root command for the data source subcommands
+var DataSourceCmd = &cobra.Command{
+	Use:   "datasource",
+	Short: "Manage data sources within a minder control plane",
+	Long:  "The data source subcommand allows the management of data sources within Minder.",
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		return cmd.Usage()
+	},
+}
+
+func init() {
+	app.RootCmd.AddCommand(DataSourceCmd)
+}