Update project to Cloudquery plugin v4

README.md

@@ -1,7 +1,7 @@
 # CloudQuery File Source Plugin
 
-[![test](https://github.com/github.com/dihedron/cq-source-file/actions/workflows/test.yaml/badge.svg)](https://github.com/github.com/dihedron/cq-source-file/actions/workflows/test.yaml)
-[![Go Report Card](https://goreportcard.com/badge/github.com/github.com/dihedron/cq-source-file)](https://goreportcard.com/report/github.com/github.com/dihedron/cq-source-file)
+[![test](https://github.com/dihedron/cq-source-file/actions/workflows/test.yaml/badge.svg)](https://github.com/dihedron/cq-source-file/actions/workflows/test.yaml)
+[![Go Report Card](https://goreportcard.com/badge/github.com/dihedron/cq-source-file)](https://goreportcard.com/report/github.com/dihedron/cq-source-file)
 
 A local file source plugin for CloudQuery that loads data from a file (in JSON, YAML, CSV or XLSX format) to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more.
 
@@ -90,8 +90,8 @@ The first part of the configuration file sepcifies that the plugin (`dihedron/fi
 
 The following `spec` provides plugin-specific configuration:
 
-1. first of all it specifies the path to the file that provides the information to be imported into CloudQuery (`./test.csv`), that the file format is `csv` (other supported values are `jsonm`, `yaml` and `xlsx`) and that the records are separated by a `,` (only required for `csv`); CSV and XLSX files *must* provide a header row that is used to name the columns, whereas JSON and YAML files must be arrays of objects (i.e. start with `[]` or with a list element `- ...`); the column (CSV, XLSX) or object attribute (YAML; JSON) names are used in the following column specification section;
-2. secondly, the configuration specifies the main table to be imported; each table has a `name` and can provide an optional `filter`, which is an expression that is applied to each row and should return either `true` (in which case the row is sent to CloudQuery) or `false` (the row is dropped); the expression is based on [this rule engine grammar](https://github.com/antonmedv/expr); the current row is addressed via the `_` identifier and the fields are accessed as properties (`_.color`);
+1. first of all it specifies the path to the file that provides the information to be imported into CloudQuery (`./test.csv`), that the file format is `csv` (other supported values are `json`, `yaml` and `xlsx`) and that the records are separated by a `,` (only required for `csv`); CSV and XLSX files *must* provide a header row that is used to name the columns, whereas JSON and YAML files must be arrays of objects (i.e. start with `[]` or with a list element `- ...`); the column (CSV, XLSX) or object attribute (YAML; JSON) names are used in the following column specification section;
+2. secondly, the configuration specifies the main table to be imported; each table has a `name` and can provide an optional `filter`, which is an expression that is applied to each row and should return either `true` (in which case the row is sent to CloudQuery) or `false` (the row is dropped); the expression is based on [this rule engine grammar](https://github.com/expr-lang/expr); the current row is addressed via the `_` identifier and the fields are accessed as properties (`_.color`);
 3. the table's colums are enumerated and decribed; each column has a `name`, a `type`, and can additionally specify whether it is part of the primary key (`key: true`), whether it has unique values (`unique: true`) and is non nullable (`notnull: true`); moreover, there is a `transform` property that provides a way to transform (or set to a constant value) the extracted value, both for data cleansing or for conditional extraction, according to Golang templates syntax; The templating engine has the whole of [Sprig](http://masterminds.github.io/sprig/) functions available;
 4. last, a table can have dependent tables (`relations`), which are weak entities that are related to the main one; relations are useful when a single line in a CSV, XLSX, YAML or JSON file actually embeds multiple entities, e.g. a host (`hostname`, `serial`, `ram`, `cpus` ...) and its (possibly multiple) dependent NICs (`mac_address`, `ip_address`, `port_type` ...).
 

_test/import-csv.yaml

@@ -5,15 +5,14 @@ spec:
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
-  tables: 
-    ["*"]
+  tables: ["*"]
   destinations:
     - sqlite
   spec:
     file: ./test.csv
     format: csv
     separator: ","
-    table: 
+    table:
       name: T1
       filter: _.color startsWith 'b'
       columns:
@@ -41,10 +40,10 @@ spec:
             key: true
             unique: true
             notnull: true
-            transform: "{{index .Row \"color\" | toString | upper}}"
+            transform: '{{index .Row "color" | toString | upper}}'
           - name: upper_value
             type: string
-            transform: "{{index .Row \"value\" | toString | upper}}"
+            transform: '{{index .Row "value" | toString | upper}}'
       - name: T1_UPPER
         filter: _.color startsWith 'bla'
         columns:
@@ -53,27 +52,26 @@ spec:
             key: true
             unique: true
             notnull: true
-            transform: "{{index .Row \"color\" | toString | upper}}"
+            transform: '{{index .Row "color" | toString | upper}}'
           - name: upper_value
             type: string
-            transform: "{{index .Row \"value\" | toString | upper}}"
+            transform: '{{index .Row "value" | toString | upper}}'
 ---
 kind: source
 spec:
   name: test2
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
-  tables: 
-    ["*"]
+  tables: ["*"]
   destinations:
     - sqlite
   spec:
     file: ./test.csv
     format: csv
     separator: ","
-    table: 
-      name: T2    
+    table:
+      name: T2
       filter: _.color not startsWith 'b'
       columns:
         - name: color
@@ -96,6 +94,6 @@ kind: destination
 spec:
   name: sqlite
   path: cloudquery/sqlite
-  version: v2.0.5
+  version: v2.7.5
   spec:
     connection_string: ./import.sqlite

_test/import-json.yaml

@@ -3,16 +3,15 @@ kind: source
 spec:
   name: test1
   registry: local
-  path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
+  path: ../dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
-  tables: 
-    ["*"]
+  tables: ["*"]
   destinations:
     - sqlite
   spec:
     file: ./test.json
     format: json
-    table: 
+    table:
       name: T1
       filter: _.color startsWith 'b'
       columns:
@@ -40,38 +39,37 @@ spec:
             key: true
             unique: true
             notnull: true
-            transform: "{{index .Row \"color\" | toString | upper}}"
+            transform: '{{index .Row "color" | toString | upper}}'
           - name: upper_value
             type: string
-            transform: "{{index .Row \"value\" | toString | upper}}"
-      - name: T1_UPPER
+            transform: '{{index .Row "value" | toString | upper}}'
+      - name: T2_UPPER
         filter: _.color startsWith 'bla'
         columns:
           - name: upper_color
             type: string
             key: true
             unique: true
             notnull: true
-            transform: "{{index .Row \"color\" | toString | upper}}"
+            transform: '{{index .Row "color" | toString | upper}}'
           - name: upper_value
             type: string
-            transform: "{{index .Row \"value\" | toString | upper}}"
+            transform: '{{index .Row "value" | toString | upper}}'
 ---
 kind: source
 spec:
   name: test2
   registry: local
-  path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
+  path: ../dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
-  tables: 
-    ["*"]
+  tables: ["*"]
   destinations:
     - sqlite
   spec:
     file: ./test.json
     format: json
-    table: 
-      name: T2    
+    table:
+      name: T2
       filter: _.color not startsWith 'b'
       columns:
         - name: color
@@ -94,6 +92,6 @@ kind: destination
 spec:
   name: sqlite
   path: cloudquery/sqlite
-  version: v2.0.5
+  version: v2.7.5
   spec:
     connection_string: ./import.sqlite

_test/import-xlsx.yaml

@@ -5,14 +5,13 @@ spec:
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
-  tables: 
-    ["*"]
+  tables: ["*"]
   destinations:
     - sqlite
   spec:
     file: ./test.xlsx
     format: xlsx
-    table: 
+    table:
       name: T1
       filter: _.color startsWith 'b'
       columns:
@@ -40,10 +39,10 @@ spec:
             key: true
             unique: true
             notnull: true
-            transform: "{{index .Row \"color\" | toString | upper}}"
+            transform: '{{index .Row "color" | toString | upper}}'
           - name: upper_value
             type: string
-            transform: "{{index .Row \"value\" | toString | upper}}"
+            transform: '{{index .Row "value" | toString | upper}}'
       - name: T1_UPPER
         filter: _.color startsWith 'bla'
         columns:
@@ -52,26 +51,25 @@ spec:
             key: true
             unique: true
             notnull: true
-            transform: "{{index .Row \"color\" | toString | upper}}"
+            transform: '{{index .Row "color" | toString | upper}}'
           - name: upper_value
             type: string
-            transform: "{{index .Row \"value\" | toString | upper}}"
+            transform: '{{index .Row "value" | toString | upper}}'
 ---
 kind: source
 spec:
   name: test2
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
-  tables: 
-    ["*"]
+  tables: ["*"]
   destinations:
     - sqlite
   spec:
     file: ./test.xlsx
     format: xlsx
-    table: 
-      name: T2    
+    table:
+      name: T2
       filter: _.color not startsWith 'b'
       columns:
         - name: color
@@ -94,6 +92,6 @@ kind: destination
 spec:
   name: sqlite
   path: cloudquery/sqlite
-  version: v2.0.5
+  version: v2.7.5
   spec:
     connection_string: ./import.sqlite

_test/import-yaml.yaml

@@ -5,14 +5,13 @@ spec:
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
-  tables: 
-    ["*"]
+  tables: ["*"]
   destinations:
     - sqlite
   spec:
     file: ./test.yaml
     format: yaml
-    table: 
+    table:
       name: T1
       filter: _.color startsWith 'b'
       columns:
@@ -40,10 +39,10 @@ spec:
             key: true
             unique: true
             notnull: true
-            transform: "{{index .Row \"color\" | toString | upper}}"
+            transform: '{{index .Row "color" | toString | upper}}'
           - name: upper_value
             type: string
-            transform: "{{index .Row \"value\" | toString | upper}}"
+            transform: '{{index .Row "value" | toString | upper}}'
       - name: T1_UPPER
         filter: _.color startsWith 'bla'
         columns:
@@ -52,26 +51,25 @@ spec:
             key: true
             unique: true
             notnull: true
-            transform: "{{index .Row \"color\" | toString | upper}}"
+            transform: '{{index .Row "color" | toString | upper}}'
           - name: upper_value
             type: string
-            transform: "{{index .Row \"value\" | toString | upper}}"
+            transform: '{{index .Row "value" | toString | upper}}'
 ---
 kind: source
 spec:
   name: test2
   registry: local
   path: /data/workspaces/gomods/cq-source-file/dist/cq-source-file_linux_amd64_v1/cq-source-file
   version: v1.0.0
-  tables: 
-    ["*"]
+  tables: ["*"]
   destinations:
     - sqlite
   spec:
     file: ./test.yaml
     format: yaml
-    table: 
-      name: T2    
+    table:
+      name: T2
       filter: _.color not startsWith 'b'
       columns:
         - name: color
@@ -94,6 +92,6 @@ kind: destination
 spec:
   name: sqlite
   path: cloudquery/sqlite
-  version: v2.0.5
+  version: v2.7.5
   spec:
     connection_string: ./import.sqlite

_test/test.yaml

@@ -26,4 +26,4 @@
 - color: black
   value: "#000"
   optimized: true
-  count: 789
+  count: 789

client/client.go

@@ -4,33 +4,36 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/cloudquery/plugin-sdk/plugins/source"
-	"github.com/cloudquery/plugin-sdk/schema"
-	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/dihedron/cq-plugin-utils/format"
 	"github.com/rs/zerolog"
 )
 
 type Client struct {
-	Logger zerolog.Logger
-	Specs  *Spec
+	Specs Spec
+
+	logger zerolog.Logger
 }
 
 func (c *Client) ID() string {
 	return "github.com/dihedron/cq-source-file"
 }
 
-func New(ctx context.Context, logger zerolog.Logger, s specs.Source, opts source.Options) (schema.ClientMeta, error) {
-	var pluginSpec Spec
+func (c *Client) Logger() *zerolog.Logger {
+	return &c.logger
+}
 
-	if err := s.UnmarshalSpec(&pluginSpec); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal plugin spec: %w", err)
-	}
+func New(ctx context.Context, logger zerolog.Logger, spec *Spec) (*Client, error) {
 
-	logger.Debug().Str("spec", format.ToJSON(pluginSpec)).Msg("plugin spec")
+	logger.Debug().Str("specs", format.ToJSON(spec)).Msg("plugin configuration")
+
+	err := spec.Validate()
+	if err != nil {
+		logger.Error().Err(err).Msg("invalid spec configuration")
+		return nil, fmt.Errorf("error spec not valid: %w", err)
+	}
 
 	return &Client{
-		Logger: logger,
-		Specs:  &pluginSpec,
+		Specs:  *spec,
+		logger: logger,
 	}, nil
 }

client/spec.go

@@ -3,7 +3,7 @@ package client
 import (
 	"text/template"
 
-	"github.com/antonmedv/expr/vm"
+	"github.com/expr-lang/expr/vm"
 )
 
 type Column struct {
@@ -24,6 +24,7 @@ type Table struct {
 	Evaluator   *vm.Program `json:"-,omitempty" yaml:"-,omitempty"`
 	Columns     []*Column   `json:"columns,omitempty" yaml:"columns,omitempty"`
 }
+
 type Spec struct {
 	File      string   `json:"file,omitempty" yaml:"file,omitempty"`
 	Format    string   `json:"format,omitempty" yaml:"format,omitempty"`
@@ -32,3 +33,11 @@ type Spec struct {
 	Table     Table    `json:"table,omitempty" yaml:"table,omitempty"`
 	Relations []Table  `json:"relations,omitempty" yaml:"relations,omitempty"`
 }
+
+func (s *Spec) Validate() error {
+	// TODO: implement
+	return nil
+}
+
+func (s *Spec) SetDefaults() {
+}