From 38b58224c0d2d5d546d25220e6f97aca86d5ea3d Mon Sep 17 00:00:00 2001 From: pomdtr Date: Thu, 4 May 2023 12:13:57 +0200 Subject: [PATCH 1/2] cleanup code scanning code --- pkg/scanner/frameworks.go | 113 ---------------------- pkg/scanner/scan.go | 17 ++-- pkg/scanner/scan_test.go | 17 ++-- pkg/scanner/{runtimes.go => scanners.go} | 116 +++++++++++++++++++---- pkg/scanner/types.go | 20 ---- 5 files changed, 110 insertions(+), 173 deletions(-) delete mode 100644 pkg/scanner/frameworks.go rename pkg/scanner/{runtimes.go => scanners.go} (51%) delete mode 100644 pkg/scanner/types.go diff --git a/pkg/scanner/frameworks.go b/pkg/scanner/frameworks.go deleted file mode 100644 index 717d96e..0000000 --- a/pkg/scanner/frameworks.go +++ /dev/null @@ -1,113 +0,0 @@ -package scanner - -import ( - "io/ioutil" - "path/filepath" - "regexp" - - "github.com/deta/space/pkg/util/fs" - "github.com/deta/space/shared" -) - -var NodeFrameworks = [...]NodeFramework{ - { - Name: shared.React, - Detectors: Detectors{ - Matches: []Match{ - {Path: "package.json", MatchContent: `"(dev)?(d|D)ependencies":\s*{[^}]*"react-scripts":\s*".+?"[^}]*}`}, - {Path: "package.json", MatchContent: `"(dev)?(d|D)ependencies":\s*{[^}]*"react-dev-utils":\s*".+?"[^}]*}`}, - }, - }, - }, - { - Name: shared.Svelte, - Detectors: Detectors{ - Matches: []Match{ - {Path: "package.json", MatchContent: `"(dev)?(d|D)ependencies":\s*{[^}]*"svelte":\s*".+?"[^}]*}`}, - {Path: "package.json", MatchContent: `"(dev)?(d|D)ependencies":\s*{[^}]*"@sveltejs/vite-plugin-svelte":\s*".+?"[^}]*}`}, - }, - Strict: true, - }, - }, - { - Name: shared.Vue, - Detectors: Detectors{ - Matches: []Match{ - {Path: "package.json", MatchContent: `"(dev)?(d|D)ependencies":\s*{[^}]*"@vue\/cli-service":\s*".+?"[^}]*}`}, - }, - Strict: true, - }, - }, - { - Name: shared.SvelteKit, - Detectors: Detectors{ - Matches: []Match{ - {Path: "package.json", MatchContent: `"(dev)?(d|D)ependencies":\s*{[^}]*"@sveltejs\/kit":\s*".+?"[^}]*}`}, - }, - Strict: true, - }, - }, - { - Name: shared.Next, - Detectors: Detectors{ - Matches: []Match{ - {Path: "package.json", MatchContent: `"(dev)?(d|D)ependencies":\s*{[^}]*"next":\s*".+?"[^}]*}`}, - }, - Strict: true, - }, - }, - { - Name: shared.Nuxt, - Detectors: Detectors{ - Matches: []Match{ - {Path: "package.json", MatchContent: `"(dev)?(d|D)ependencies":\s*{[^}]*"nuxt3?(-edge)?":\s*".+?"[^}]*}`}, - }, - Strict: true, - }, - }, -} - -func check(dir string, framework *NodeFramework) (bool, error) { - passed := false - for _, match := range (*framework).Detectors.Matches { - - // check to see if the file exists before checking for pattern match - exists, err := fs.FileExists(dir, match.Path) - if err != nil { - return false, err - } - if !exists { - return false, nil - } - - path := filepath.Join(dir, match.Path) - - b, err := ioutil.ReadFile(path) - if err != nil { - return false, err - } - - pass, _ := regexp.MatchString(match.MatchContent, string(b)) - - if !pass && (*framework).Detectors.Strict { - return false, nil - } - if pass { - passed = true - } - } - return passed, nil -} - -func detectFramework(dir string) (string, error) { - for _, framework := range NodeFrameworks { - check, err := check(dir, &framework) - if err != nil { - return "", err - } - if check { - return framework.Name, nil - } - } - return "nodejs16", nil -} diff --git a/pkg/scanner/scan.go b/pkg/scanner/scan.go index d7e1953..aec05e9 100644 --- a/pkg/scanner/scan.go +++ b/pkg/scanner/scan.go @@ -9,11 +9,6 @@ import ( ) func Scan(sourceDir string) ([]*shared.Micro, error) { - files, err := os.ReadDir(sourceDir) - if err != nil { - return nil, err - } - var micros []*shared.Micro // scan root source dir for a micro @@ -21,7 +16,6 @@ func Scan(sourceDir string) ([]*shared.Micro, error) { if err != nil { return nil, err } - if m != nil { // root folder has a micro return as a single micro app micros = append(micros, m) @@ -29,9 +23,14 @@ func Scan(sourceDir string) ([]*shared.Micro, error) { } // scan subfolders for micros - for _, file := range files { - if file.IsDir() { - m, err = scanDir(filepath.Join(sourceDir, file.Name())) + entries, err := os.ReadDir(sourceDir) + if err != nil { + return nil, err + } + + for _, entry := range entries { + if entry.IsDir() { + m, err = scanDir(filepath.Join(sourceDir, entry.Name())) if err != nil { return nil, err } diff --git a/pkg/scanner/scan_test.go b/pkg/scanner/scan_test.go index 17b72f4..d1e92ff 100644 --- a/pkg/scanner/scan_test.go +++ b/pkg/scanner/scan_test.go @@ -14,8 +14,8 @@ type ScanTestInfo struct { ExpectedEngine string } -var ( - microsTestInfo = []ScanTestInfo{ +func TestScanSingleMicroProjects(t *testing.T) { + microsTestInfo := []ScanTestInfo{ {Name: "python", Path: "testdata/micros/python", ExpectedEngine: shared.Python39}, {Name: "go", Path: "testdata/micros/go", ExpectedEngine: shared.Custom}, {Name: "next", Path: "testdata/micros/next", ExpectedEngine: shared.Next}, @@ -27,9 +27,7 @@ var ( {Name: "svelte-kit", Path: "testdata/micros/svelte-kit", ExpectedEngine: shared.SvelteKit}, {Name: "vue", Path: "testdata/micros/vue", ExpectedEngine: shared.Vue}, } -) -func TestScanSingleMicroProjects(t *testing.T) { for _, project := range microsTestInfo { t.Run(project.Path, func(t *testing.T) { micros, err := Scan(project.Path) @@ -44,7 +42,6 @@ func TestScanSingleMicroProjects(t *testing.T) { } func TestScanMultiMicroProject(t *testing.T) { - expectedMicros := []string{"python", "go", "next", "node", "nuxt", "react", "static", "svelte", "svelte-kit", "vue"} expectedMicrosToEngines := map[string]string{ "python": shared.Python39, @@ -69,12 +66,10 @@ func TestScanMultiMicroProject(t *testing.T) { assert.Equal(t, len(micros), len(expectedMicros), "detected %d micros, but expected %d", len(micros), len(expectedMicros)) for _, micro := range micros { - t.Run(micro.Name, func(t *testing.T) { - if !slices.Contains(expectedMicros, micro.Name) { - t.Fatalf("micro %s at %s is detected, but should not be detected as part of a multi-micro project", micro.Name, micro.Src) - } - assert.Equal(t, micro.Engine, expectedMicrosToEngines[micro.Name], "detected engine for micro %s as %s, but expected %s", micro.Name, micro.Engine, expectedMicrosToEngines[micro.Name]) - }) + if !slices.Contains(expectedMicros, micro.Name) { + t.Fatalf("micro %s at %s is detected, but should not be detected as part of a multi-micro project", micro.Name, micro.Src) + } + assert.Equal(t, micro.Engine, expectedMicrosToEngines[micro.Name], "detected engine for micro %s as %s, but expected %s", micro.Name, micro.Engine, expectedMicrosToEngines[micro.Name]) } } diff --git a/pkg/scanner/runtimes.go b/pkg/scanner/scanners.go similarity index 51% rename from pkg/scanner/runtimes.go rename to pkg/scanner/scanners.go index 9b73e69..676119b 100644 --- a/pkg/scanner/runtimes.go +++ b/pkg/scanner/scanners.go @@ -1,12 +1,18 @@ package scanner import ( + "encoding/json" "fmt" + "os" + "path/filepath" + "regexp" "github.com/deta/space/pkg/util/fs" "github.com/deta/space/shared" ) +type engineScanner func(dir string) (*shared.Micro, error) + func pythonScanner(dir string) (*shared.Micro, error) { // if any of the following files exist detect as python app exists, err := fs.CheckIfAnyFileExists(dir, "requirements.txt", "Pipfile", "setup.py", "main.py") @@ -31,49 +37,119 @@ func pythonScanner(dir string) (*shared.Micro, error) { return m, nil } -func nodeScanner(dir string) (*shared.Micro, error) { - // if any of the following files exist detect as a node app - exists, err := fs.CheckIfAnyFileExists(dir, "package.json") +type PackageJSON struct { + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` +} + +func (p *PackageJSON) HasDependency(pattern string) bool { + reg, err := regexp.Compile(pattern) if err != nil { - return nil, err + return false } - if !exists { - return nil, nil + + for dep := range p.Dependencies { + if reg.MatchString(dep) { + return true + } } + for dep := range p.DevDependencies { + if reg.MatchString(dep) { + return true + } + } + + return false +} + +func nodeScanner(dir string) (*shared.Micro, error) { name, err := getMicroNameFromPath(dir) if err != nil { return nil, fmt.Errorf("failed to extract micro name from it's path, %v", err) } - m := &shared.Micro{ - Name: name, - Src: dir, - Engine: shared.Node16x, + // if any of the following files exist detect as a node app + manifestPath := filepath.Join(dir, "package.json") + if _, err := os.Stat(manifestPath); err != nil { + return nil, nil } - framework, err := detectFramework(dir) + manifestBytes, err := os.ReadFile(manifestPath) if err != nil { return nil, err } - m.Engine = framework - return m, nil -} - -func goScanner(dir string) (*shared.Micro, error) { - exists, err := fs.CheckIfAnyFileExists(dir, "go.mod") - if err != nil { + var manifest PackageJSON + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { return nil, err } - if !exists { - return nil, nil + + if manifest.HasDependency("react-scripts") || manifest.HasDependency("react-dev-utils") { + return &shared.Micro{ + Name: name, + Src: dir, + Engine: shared.React, + }, nil + } + + if manifest.HasDependency("svelte") && manifest.HasDependency("@sveltejs/vite-plugin-svelte") { + return &shared.Micro{ + Name: name, + Src: dir, + Engine: shared.Svelte, + }, nil + } + + if manifest.HasDependency("@vue/cli-service") { + return &shared.Micro{ + Name: name, + Src: dir, + Engine: shared.Vue, + }, nil + } + + if manifest.HasDependency("@sveltejs/kit") { + return &shared.Micro{ + Name: name, + Src: dir, + Engine: shared.SvelteKit, + }, nil + } + + if manifest.HasDependency("next") { + return &shared.Micro{ + Name: name, + Src: dir, + Engine: shared.Next, + }, nil } + if manifest.HasDependency("nuxt3?(-edge)?") { + return &shared.Micro{ + Name: name, + Src: dir, + Engine: shared.Nuxt, + }, nil + } + + return &shared.Micro{ + Name: name, + Src: dir, + Engine: "nodejs16", + }, nil +} + +func goScanner(dir string) (*shared.Micro, error) { name, err := getMicroNameFromPath(dir) if err != nil { return nil, fmt.Errorf("failed to extract micro name from it's path, %v", err) } + + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil { + return nil, nil + } + m := &shared.Micro{ Name: name, Src: dir, diff --git a/pkg/scanner/types.go b/pkg/scanner/types.go deleted file mode 100644 index 4730061..0000000 --- a/pkg/scanner/types.go +++ /dev/null @@ -1,20 +0,0 @@ -package scanner - -import "github.com/deta/space/shared" - -type engineScanner func(dir string) (*shared.Micro, error) - -type Match struct { - Path string - MatchContent string -} - -type Detectors struct { - Matches []Match - Strict bool // strict requires all the matches to pass -} - -type NodeFramework struct { - Name string - Detectors Detectors -} From 2f6e65a36d75b5f074957644a9f153976f950e3b Mon Sep 17 00:00:00 2001 From: pomdtr Date: Thu, 4 May 2023 12:16:20 +0200 Subject: [PATCH 2/2] add rust scanner --- pkg/scanner/scan.go | 1 + pkg/scanner/scanners.go | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/pkg/scanner/scan.go b/pkg/scanner/scan.go index aec05e9..b95d318 100644 --- a/pkg/scanner/scan.go +++ b/pkg/scanner/scan.go @@ -48,6 +48,7 @@ func scanDir(dir string) (*shared.Micro, error) { pythonScanner, nodeScanner, goScanner, + rustScanner, staticScanner, } diff --git a/pkg/scanner/scanners.go b/pkg/scanner/scanners.go index 676119b..734382a 100644 --- a/pkg/scanner/scanners.go +++ b/pkg/scanner/scanners.go @@ -162,6 +162,26 @@ func goScanner(dir string) (*shared.Micro, error) { return m, nil } +func rustScanner(dir string) (*shared.Micro, error) { + name, err := getMicroNameFromPath(dir) + if err != nil { + return nil, fmt.Errorf("failed to extract micro name from it's path, %v", err) + } + + if _, err := os.Stat(filepath.Join(dir, "Cargo.toml")); err != nil { + return nil, nil + } + + return &shared.Micro{ + Name: name, + Src: dir, + Engine: "custom", + Commands: []string{"cargo build --release --bin server"}, + Include: []string{"target/release/server"}, + Run: "./server", + }, nil +} + func staticScanner(dir string) (*shared.Micro, error) { // if any of the following files exist, detect as a static app exists, err := fs.CheckIfAnyFileExists(dir, "index.html")