feat(recipe)!: add version support in recipes (#322)

* feat: add version to new recipe cmd * chore: keep agent out of touch for recipe * feat: add v1beta1 as base recipe version across all templates * chore: add tests for incorrect version * docs: add version field in examples and docs * fix: typo in error message Co-authored-by: Abduh <[email protected]> * test: check for error message * refactor: remove multiple declaration of same err Co-authored-by: Abduh <[email protected]>
raystack · Mar 9, 2022 · d6a65c8 · d6a65c8
1 parent adb20c4
commit d6a65c8
Show file tree

Hide file tree

Showing 22 changed files with 101 additions and 18 deletions.
diff --git a/cmd/run.go b/cmd/run.go
@@ -3,6 +3,12 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+
 	"github.com/MakeNowJust/heredoc"
 	"github.com/odpf/meteor/agent"
 	"github.com/odpf/meteor/config"
@@ -14,11 +20,6 @@ import (
 	"github.com/odpf/salt/term"
 	"github.com/schollz/progressbar/v3"
 	"github.com/spf13/cobra"
-	"os"
-	"os/signal"
-	"strconv"
-	"syscall"
-	"time"
 )
 
 // RunCmd creates a command object for the "run" action.

diff --git a/docs/docs/concepts/recipe.md b/docs/docs/concepts/recipe.md
@@ -12,6 +12,7 @@ Recipe is a yaml file, follows a structure as shown below and needs to passed as
 
 ```yaml
 name: main-kafka-production # unique recipe name as an ID
+version: v1beta1 #recipe version
 source: # required - for fetching input from sources
  name: kafka # required - collector to use (e.g. bigquery, kafka)
  config:
@@ -36,6 +37,7 @@ Contains details about the ingridients of our recipe. The `config` of each sourc
 | Key | Description | Requirement | further reference |
 | :--- | :--- | :--- | :--- |
 | `name` | **unique** recipe name, will be used as ID for job | required | N/A |
+| `version` | Specify the version of recipe being used | required | N/A |
 | `source` | contains details about the source of metadata extraction | required | [source](source.md) |
 | `sinks` | defines the final destination's of extracted and processed metadata | required | [sink](sink.md) |
 | `processors` | used process the metadata before sinking | optional | [processor](processor.md) |
@@ -48,6 +50,7 @@ Meteor reads recipe using [go template](https://golang.org/pkg/text/template/),
 
 ```yaml
 name: sample-recipe
+version: v1beta1
 source:
   name: mongodb
   config:

diff --git a/docs/docs/example/date-kafka.yaml b/docs/docs/example/date-kafka.yaml
@@ -1,4 +1,5 @@
 name: date-kafka-recipe
+version: v1beta1
 source:
   name: date
 sinks:

diff --git a/docs/docs/example/kafka-console.yaml b/docs/docs/example/kafka-console.yaml
@@ -1,4 +1,5 @@
 name: sample-recipe
+version: v1beta1
 source:
   name: kafka
   config:

diff --git a/generator/recipe.go b/generator/recipe.go
@@ -16,6 +16,7 @@ var file embed.FS
 // Template represents the template for generating a recipe.
 type Template struct {
 	Name       string
+	Version    string
 	Source     map[string]string
 	Sinks      map[string]string
 	Processors map[string]string
@@ -25,10 +26,13 @@ var templateFuncs = map[string]interface{}{
 	"indent": indent,
 }
 
+var recipeVersions = [1]string{"v1beta1"}
+
 // Recipe checks if the recipe is valid and returns a Template
 func Recipe(name string, source string, sinks []string, processors []string) (err error) {
 	tem := Template{
-		Name: name,
+		Name:    name,
+		Version: recipeVersions[len(recipeVersions)-1],
 	}
 
 	if source != "" {
@@ -73,3 +77,7 @@ func indent(spaces int, v string) string {
 	pad := strings.Repeat(" ", spaces)
 	return pad + strings.Replace(v, "\n", "\n"+pad, -1)
 }
+
+func GetRecipeVersions() [1]string {
+	return recipeVersions
+}
diff --git a/generator/recipe.yaml b/generator/recipe.yaml
@@ -1,4 +1,5 @@
 name: {{.Name}}
+version: {{.Version}}
 source:
 {{- range $key, $value := .Source }}
   type: {{$key}}

diff --git a/recipe/node.go b/recipe/node.go
@@ -9,6 +9,7 @@ import (
 // RecipeNode contains the json data for a recipe node
 type RecipeNode struct {
 	Name       yaml.Node    `json:"name" yaml:"name"`
+	Version    yaml.Node    `json:"version" yaml:"version"`
 	Source     PluginNode   `json:"source" yaml:"source"`
 	Sinks      []PluginNode `json:"sinks" yaml:"sinks"`
 	Processors []PluginNode `json:"processors" yaml:"processors"`
@@ -60,7 +61,8 @@ func (node RecipeNode) toRecipe() (recipe Recipe, err error) {
 		return
 	}
 	recipe = Recipe{
-		Name: node.Name.Value,
+		Name:    node.Name.Value,
+		Version: node.Version.Value,
 		Source: PluginRecipe{
 			Name:   node.Source.Name.Value,
 			Config: sourceConfig,

diff --git a/recipe/reader.go b/recipe/reader.go
@@ -2,11 +2,14 @@ package recipe
 
 import (
 	"bytes"
+	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	"text/template"
 
+	"github.com/odpf/meteor/generator"
 	"gopkg.in/yaml.v3"
 )
 
@@ -15,6 +18,10 @@ type Reader struct {
 	data map[string]string
 }
 
+var (
+	ErrInvalidRecipeVersion = errors.New("recipe version is invalid or not found")
+)
+
 // NewReader returns a new Reader.
 func NewReader() *Reader {
 	reader := &Reader{}
@@ -69,6 +76,12 @@ func (r *Reader) readFile(path string) (recipe Recipe, err error) {
 		node.Name.Value = fileName
 	}
 
+	versions := generator.GetRecipeVersions()
+	err = validateRecipeVersion(node.Version.Value, versions[len(versions)-1])
+	if err != nil {
+		return
+	}
+
 	recipe, err = node.toRecipe()
 	if err != nil {
 		return
@@ -94,3 +107,10 @@ func (r *Reader) readDir(path string) (recipes []Recipe, err error) {
 
 	return
 }
+
+func validateRecipeVersion(receivedVersion, expectedVersion string) (err error) {
+	if strings.Compare(receivedVersion, expectedVersion) == 0 {
+		return
+	}
+	return fmt.Errorf("received recipe version %s does not match to the expected version %s", receivedVersion, expectedVersion)
+}
diff --git a/recipe/reader_test.go b/recipe/reader_test.go
@@ -1,6 +1,7 @@
 package recipe_test
 
 import (
+	"errors"
 	"os"
 	"testing"
 
@@ -63,7 +64,8 @@ func TestReaderRead(t *testing.T) {
 			}
 			expectedRecipes := []recipe.Recipe{
 				{
-					Name: "test-recipe-no-name",
+					Name:    "test-recipe-no-name",
+					Version: "v1beta1",
 					Source: recipe.PluginRecipe{
 						Name: "test-source",
 						Config: map[string]interface{}{
@@ -221,6 +223,15 @@ func TestReaderRead(t *testing.T) {
 			compareRecipes(t, expected[i], r)
 		}
 	})
+
+	t.Run("should return error if version is missing/incorrect", func(t *testing.T) {
+		reader := recipe.NewReader()
+		_, err := reader.Read("./testdata/missing-version.yaml")
+		errors.Is(err, recipe.ErrInvalidRecipeVersion)
+
+		_, err = reader.Read("./testdata/incorrect-version.yaml")
+		errors.Is(err, recipe.ErrInvalidRecipeVersion)
+	})
 }
 
 func compareRecipes(t *testing.T, expected, actual recipe.Recipe) {

diff --git a/recipe/recipe.go b/recipe/recipe.go
@@ -3,6 +3,7 @@ package recipe
 // Recipe contains the json data for a recipe
 type Recipe struct {
 	Name       string         `json:"name" yaml:"name" validate:"required"`
+	Version    string         `json:"version" yaml:"version" validate:"required"`
 	Source     PluginRecipe   `json:"source" yaml:"source" validate:"required"`
 	Sinks      []PluginRecipe `json:"sinks" yaml:"sinks" validate:"required,min=1"`
 	Processors []PluginRecipe `json:"processors" yaml:"processors"`

diff --git a/recipe/recipe_test.go b/recipe/recipe_test.go
@@ -18,12 +18,12 @@ func TestRecipeGetLine(t *testing.T) {
 	rcp := r[0]
 
 	t.Run("should return source line and column", func(t *testing.T) {
-		assert.Equal(t, 3, rcp.Source.Node.Name.Line)
+		assert.Equal(t, 4, rcp.Source.Node.Name.Line)
 		assert.Equal(t, 9, rcp.Source.Node.Name.Column)
 	})
 
 	t.Run("should return config source lines", func(t *testing.T) {
-		expectedLineNum := []int{5, 6, 7}
+		expectedLineNum := []int{6, 7, 8}
 		var lineNum []int
 		srcConfig := rcp.Source.Node.Config
 		for _, j := range srcConfig {
@@ -34,24 +34,24 @@ func TestRecipeGetLine(t *testing.T) {
 	})
 
 	t.Run("should return config source line for a specific config key", func(t *testing.T) {
-		expectedLineNum := 6
+		expectedLineNum := 7
 		srcConfigKey := rcp.Source.Node.Config["srcKey2"]
 		assert.Equal(t, expectedLineNum, srcConfigKey.Line)
 	})
 
 	t.Run("should return processors line and column", func(t *testing.T) {
-		assert.Equal(t, 9, rcp.Processors[0].Node.Name.Line)
+		assert.Equal(t, 10, rcp.Processors[0].Node.Name.Line)
 		assert.Equal(t, 11, rcp.Processors[0].Node.Name.Column)
 
-		assert.Equal(t, 14, rcp.Processors[1].Node.Name.Line)
+		assert.Equal(t, 15, rcp.Processors[1].Node.Name.Line)
 		assert.Equal(t, 11, rcp.Processors[1].Node.Name.Column)
 	})
 
 	t.Run("should return sinks line and column", func(t *testing.T) {
-		assert.Equal(t, 20, rcp.Sinks[0].Node.Name.Line)
+		assert.Equal(t, 21, rcp.Sinks[0].Node.Name.Line)
 		assert.Equal(t, 11, rcp.Sinks[0].Node.Name.Column)
 
-		assert.Equal(t, 25, rcp.Sinks[1].Node.Name.Line)
+		assert.Equal(t, 26, rcp.Sinks[1].Node.Name.Line)
 		assert.Equal(t, 11, rcp.Sinks[1].Node.Name.Column)
 	})
 }
@@ -65,12 +65,12 @@ func TestRecipeGetLineBySrcTypeTag(t *testing.T) {
 	rcp := r[0]
 
 	t.Run("should return source line and column", func(t *testing.T) {
-		assert.Equal(t, 3, rcp.Source.Node.Type.Line)
+		assert.Equal(t, 4, rcp.Source.Node.Type.Line)
 		assert.Equal(t, 9, rcp.Source.Node.Type.Column)
 	})
 
 	t.Run("should return config source lines", func(t *testing.T) {
-		expectedLineNum := []int{5, 6, 7}
+		expectedLineNum := []int{6, 7, 8}
 		var lineNum []int
 		srcConfig := rcp.Source.Node.Config
 		for _, j := range srcConfig {
@@ -81,7 +81,7 @@ func TestRecipeGetLineBySrcTypeTag(t *testing.T) {
 	})
 
 	t.Run("should return config source line for a specific config key", func(t *testing.T) {
-		expectedLineNum := 6
+		expectedLineNum := 7
 		srcConfigKey := rcp.Source.Node.Config["srcKey2"]
 		assert.Equal(t, expectedLineNum, srcConfigKey.Line)
 	})

diff --git a/recipe/testdata/generator/expected-2.yaml b/recipe/testdata/generator/expected-2.yaml
@@ -1,4 +1,5 @@
 name: recipe-2
+version: v1beta1
 source:
   type: kafka
   config:

diff --git a/recipe/testdata/generator/expected-3.yaml b/recipe/testdata/generator/expected-3.yaml
@@ -1,4 +1,5 @@
 name: recipe-three
+version: v1beta1
 source:
   type: kafka
   config:

diff --git a/recipe/testdata/generator/expected.yaml b/recipe/testdata/generator/expected.yaml
@@ -1,4 +1,5 @@
 name: recipe-1
+version: v1beta1
 source:
   type: kafka
   config:

diff --git a/recipe/testdata/generator/template.yaml b/recipe/testdata/generator/template.yaml
@@ -1,4 +1,5 @@
 name: {{ .Data.name }}
+version: v1beta1
 source:
   type: kafka
   config:

diff --git a/recipe/testdata/incorrect-version.yaml b/recipe/testdata/incorrect-version.yaml
@@ -0,0 +1,13 @@
+name: recipe-three
+version: v1alpha0
+source:
+  type: kafka
+  config:
+    broker: "main-broker.com:9092"
+sinks:
+  - name: console
+processors:
+  - name: enrich
+    config:
+      host:  main-broker.com:9092
+      owner:  [email protected]
diff --git a/recipe/testdata/missing-version.yaml b/recipe/testdata/missing-version.yaml
@@ -0,0 +1,12 @@
+name: recipe-three
+source:
+  type: kafka
+  config:
+    broker: "main-broker.com:9092"
+sinks:
+  - name: console
+processors:
+  - name: enrich
+    config:
+      host:  main-broker.com:9092
+      owner:  [email protected]
diff --git a/recipe/testdata/recipe-read-line.yaml b/recipe/testdata/recipe-read-line.yaml
@@ -1,4 +1,5 @@
 name: recipe-read-line
+version: v1beta1
 source:
   name: srcA
   config:

diff --git a/recipe/testdata/src- typeTag-recipe-read-line.yaml b/recipe/testdata/src- typeTag-recipe-read-line.yaml
@@ -1,4 +1,5 @@
 name: recipe-read-line
+version: v1beta1
 source:
   type: srcA
   config:

diff --git a/recipe/testdata/testdir/test-recipe-no-name.yaml b/recipe/testdata/testdir/test-recipe-no-name.yaml
@@ -1,3 +1,4 @@
+version: v1beta1
 source:
   name: test-source
   config:

diff --git a/recipe/testdata/testdir/test-recipe-variables.yaml b/recipe/testdata/testdir/test-recipe-variables.yaml
@@ -1,4 +1,5 @@
 name: test-recipe
+version: v1beta1
 source:
   name: test-source
   config:

diff --git a/recipe/testdata/testdir/test-recipe.yaml b/recipe/testdata/testdir/test-recipe.yaml
@@ -1,4 +1,5 @@
 name: test-recipe
+version: v1beta1
 source:
   name: test-source
   config: