Merge pull request #254 from jumppad-labs/f-copy-go-getter

Add the capability to the copy resource to copy from remote sources

.github/workflows/build_and_deploy.yaml

@@ -64,6 +64,7 @@ jobs:
           '/certificates',
           '/terraform',
           '/registries',
+          "/copy"
         ]
 
     steps:

README.md

@@ -38,7 +38,7 @@ The following snippets are examples of things you can build with Jumppad, for mo
 
 ```hcl
 resource "network" "cloud" {
-  subnet = "10.5.0.0/16"
+  subnet = "10.5.0.0/24"
 }
 
 resource "k8s_cluster" "k3s" {

examples/copy/files/foo

@@ -0,0 +1 @@
+bar

examples/copy/main.hcl

@@ -0,0 +1,25 @@
+resource "copy" "local" {
+  source = "${dir()}/files/foo"
+  destination = "${data("copy")}/local"
+}
+
+resource "copy" "local_relative" {
+  source = "./files/foo"
+  destination = "${data("copy")}/local_relative"
+}
+
+
+resource "copy" "http" {
+  source = "https://www.foundanimals.org/wp-content/uploads/2023/02/twenty20_b4e89a76-af70-4567-b92a-9c3bbf335cb3.jpg"
+  destination = "${data("copy")}/http"
+}
+
+resource "copy" "git" {
+  source = "github.com/jumppad-labs/examples"
+  destination = "${data("copy")}/git"
+}
+
+resource "copy" "zip" {
+  source = "https://releases.hashicorp.com/nomad/1.6.3/nomad_1.6.3_linux_amd64.zip"
+  destination = "${data("copy")}/zip"
+}

examples/copy/test/copy.feature

@@ -0,0 +1,27 @@
+Feature: Copying files
+  In order to test copy capabilities
+  I should apply a blueprint
+  And check that the files have been copied
+
+  Scenario: Copy files from multiple sources
+    Given I have a running blueprint
+    When I run the script
+    ```
+    #!/bin/bash
+    if [ ! -f $HOME/.jumppad/data/copy/local/foo ]; then
+      exit 1
+    fi
+
+    if [ ! -f $HOME/.jumppad/data/copy/http/twenty20_b4e89a76-af70-4567-b92a-9c3bbf335cb3.jpg ]; then
+      exit 1
+    fi
+
+    if [ ! -f $HOME/.jumppad/data/copy/git/README.md ]; then
+      exit 1
+    fi
+
+    if [ ! -f $HOME/.jumppad/data/copy/zip/nomad ]; then
+      exit 1
+    fi
+    ```
+    Then I expect the exit code to be 0

pkg/config/functions.go

@@ -0,0 +1,52 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strconv"
+
+	"github.com/jumppad-labs/jumppad/pkg/utils"
+)
+
+func customHCLFuncJumppad() (string, error) {
+	return utils.JumppadHome(), nil
+}
+
+// returns the docker host ip address
+func customHCLFuncDockerIP() (string, error) {
+	return utils.GetDockerIP(), nil
+}
+
+func customHCLFuncDockerHost() (string, error) {
+	return utils.GetDockerHost(), nil
+}
+
+func customHCLFuncDataFolderWithPermissions(name string, permissions int) (string, error) {
+	if permissions > 0 && permissions < 778 {
+		return "", fmt.Errorf("permissions must be a three digit number less than 777")
+	}
+
+	// convert the permissions to an octal e.g. 777 to 0777
+	strInt := fmt.Sprintf("0%d", permissions)
+	oInt, _ := strconv.ParseInt(strInt, 8, 32)
+
+	perms := os.FileMode(uint32(oInt))
+	return utils.DataFolder(name, perms), nil
+}
+
+func customHCLFuncDataFolder(name string) (string, error) {
+	perms := os.FileMode(0775)
+	return utils.DataFolder(name, perms), nil
+}
+
+func customHCLFuncSystem(property string) (string, error) {
+	switch property {
+	case "os":
+		return runtime.GOOS, nil
+	case "arch":
+		return runtime.GOARCH, nil
+	default:
+		return "", fmt.Errorf("unknown system property %s", property)
+	}
+}

pkg/config/resources/copy/provider.go

@@ -5,10 +5,14 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
 	htypes "github.com/jumppad-labs/hclconfig/types"
+	"github.com/jumppad-labs/jumppad/pkg/clients"
+	"github.com/jumppad-labs/jumppad/pkg/clients/getter"
+	"github.com/jumppad-labs/jumppad/pkg/utils"
 	sdk "github.com/jumppad-labs/plugin-sdk"
 	cp "github.com/otiai10/copy"
 	"golang.org/x/xerrors"
@@ -17,6 +21,7 @@ import (
 type Provider struct {
 	log    sdk.Logger
 	config *Copy
+	getter getter.Getter
 }
 
 func (p *Provider) Init(cfg htypes.Resource, l sdk.Logger) error {
@@ -25,6 +30,12 @@ func (p *Provider) Init(cfg htypes.Resource, l sdk.Logger) error {
 		return fmt.Errorf("unable to initialize Copy provider, resource is not an instance of Copy")
 	}
 
+	cli, err := clients.GenerateClients(l)
+	if err != nil {
+		return err
+	}
+
+	p.getter = cli.Getter
 	p.config = c
 	p.log = l
 
@@ -39,10 +50,17 @@ func (p *Provider) Create(ctx context.Context) error {
 
 	p.log.Info("Creating Copy", "ref", p.config.Meta.Name, "source", p.config.Source, "destination", p.config.Destination, "perms", p.config.Permissions)
 
+	tempPath := filepath.Join(utils.JumppadTemp(), "copy", p.config.Meta.Name)
+
+	err := p.getter.Get(p.config.Source, tempPath)
+	if err != nil {
+		return fmt.Errorf("error getting source from %s: %v", p.config.Source, err)
+	}
+
 	// Check source exists
-	_, err := os.Stat(p.config.Source)
+	_, err = os.Stat(tempPath)
 	if err != nil {
-		p.log.Debug("Error discovering source directory", "ref", p.config.Meta.Name, "source", p.config.Source, "error", err)
+		p.log.Debug("Error discovering source directory", "ref", p.config.Meta.Name, "source", tempPath, "error", err)
 		return xerrors.Errorf("unable to find source directory for copy resource, ref=%s: %w", p.config.Meta.Name, err)
 	}
 
@@ -67,9 +85,9 @@ func (p *Provider) Create(ctx context.Context) error {
 		return false, nil
 	}
 
-	err = cp.Copy(p.config.Source, p.config.Destination, opts)
+	err = cp.Copy(tempPath, p.config.Destination, opts)
 	if err != nil {
-		p.log.Debug("Error copying source directory", "ref", p.config.Meta.Name, "source", p.config.Source, "error", err)
+		p.log.Debug("Error copying source directory", "ref", p.config.Meta.Name, "source", tempPath, "error", err)
 
 		return xerrors.Errorf("unable to copy files, ref=%s: %w", p.config.Meta.Name, err)
 	}
@@ -81,11 +99,11 @@ func (p *Provider) Create(ctx context.Context) error {
 		perms, err := strconv.ParseInt(p.config.Permissions, 8, 64)
 		if err != nil {
 			p.log.Debug("Invalid destination permissions", "ref", p.config.Meta.Name, "permissions", p.config.Permissions, "error", err)
-			return xerrors.Errorf("Invalid destination permissions for copy resource, ref=%s %s: %w", p.config.Meta.Name, p.config.Permissions, err)
+			return xerrors.Errorf("invalid destination permissions for copy resource, ref=%s %s: %w", p.config.Meta.Name, p.config.Permissions, err)
 		}
 
 		for _, f := range p.config.CopiedFiles {
-			fn := strings.Replace(f, p.config.Source, p.config.Destination, -1)
+			fn := strings.Replace(f, tempPath, p.config.Destination, -1)
 			p.log.Debug("Setting permissions for file", "ref", p.config.Meta.Name, "file", fn, "permissions", p.config.Permissions)
 
 			os.Chmod(fn, os.FileMode(perms))
@@ -97,6 +115,12 @@ func (p *Provider) Create(ctx context.Context) error {
 		os.Chmod(p.config.Destination, originalPerms)
 	}
 
+	// clean up temporary files
+	err = os.RemoveAll(tempPath)
+	if err != nil {
+		p.log.Debug("Error removing temporary files", "ref", p.config.Meta.Name, "path", tempPath, "error", err)
+	}
+
 	return nil
 }
 
@@ -130,12 +154,10 @@ func (p *Provider) Lookup() ([]string, error) {
 
 func (p *Provider) Refresh(ctx context.Context) error {
 	p.log.Debug("Refresh Copied files", "ref", p.config.Meta.Name)
-
 	return nil
 }
 
 func (p *Provider) Changed() (bool, error) {
 	p.log.Debug("Checking changes", "ref", p.config.Meta.Name)
-
 	return false, nil
 }

pkg/config/resources/copy/resource.go

@@ -1,6 +1,8 @@
 package copy
 
 import (
+	"os"
+
 	"github.com/jumppad-labs/hclconfig/types"
 	"github.com/jumppad-labs/jumppad/pkg/config"
 	"github.com/jumppad-labs/jumppad/pkg/utils"
@@ -17,7 +19,7 @@ type Copy struct {
 
 	Depends []string `hcl:"depends_on,optional" json:"depends,omitempty"`
 
-	Source      string `hcl:"source" json:"source"`                              // Source file, folder or glob
+	Source      string `hcl:"source" json:"source"`                              // Source file, folder, url, git repo, etc
 	Destination string `hcl:"destination" json:"destination"`                    // Destination to write file or files to
 	Permissions string `hcl:"permissions,optional" json:"permissions,omitempty"` // Permissions 0777 to set for written file
 
@@ -26,7 +28,12 @@ type Copy struct {
 }
 
 func (t *Copy) Process() error {
-	t.Source = utils.EnsureAbsolute(t.Source, t.Meta.File)
+	// If the source is a local file, ensure it is absolute
+	tempSource := utils.EnsureAbsolute(t.Source, t.Meta.File)
+	if _, err := os.Stat(tempSource); err == nil {
+		t.Source = tempSource
+	}
+
 	t.Destination = utils.EnsureAbsolute(t.Destination, t.Meta.File)
 
 	cfg, err := config.LoadState()

pkg/config/resources/copy/resource_test.go

@@ -14,7 +14,7 @@ func init() {
 	config.RegisterResource(TypeCopy, &Copy{}, &Provider{})
 }
 
-func TestCopyProcessSetsAbsolute(t *testing.T) {
+func TestCopyProcessSetsAbsoluteIfLocal(t *testing.T) {
 	wd, err := os.Getwd()
 	require.NoError(t, err)
 
@@ -30,6 +30,22 @@ func TestCopyProcessSetsAbsolute(t *testing.T) {
 	require.Equal(t, wd, c.Destination)
 }
 
+func TestCopyProcessSetsAbsoluteIfNotLocal(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	c := &Copy{
+		ResourceBase: types.ResourceBase{Meta: types.Meta{File: "./"}},
+		Source:       "github.com/jumppad-labs/jumppad",
+		Destination:  "./",
+	}
+
+	c.Process()
+
+	require.Equal(t, "github.com/jumppad-labs/jumppad", c.Source)
+	require.Equal(t, wd, c.Destination)
+}
+
 func TestCopySetsOutputsFromState(t *testing.T) {
 	testutils.SetupState(t, `
 {
@@ -60,5 +76,4 @@ func TestCopySetsOutputsFromState(t *testing.T) {
 	c.Process()
 
 	require.Equal(t, []string{"a", "b"}, c.CopiedFiles)
-
 }

pkg/config/zz_hclparser.go

@@ -1,13 +1,8 @@
 package config
 
 import (
-	"fmt"
-	"os"
-	"strconv"
-
 	"github.com/jumppad-labs/hclconfig"
 	"github.com/jumppad-labs/hclconfig/types"
-	"github.com/jumppad-labs/jumppad/pkg/utils"
 	sdk "github.com/jumppad-labs/plugin-sdk"
 )
 
@@ -57,37 +52,7 @@ func NewParser(callback hclconfig.WalkCallback, variables map[string]string, var
 	p.RegisterFunction("docker_host", customHCLFuncDockerHost)
 	p.RegisterFunction("data", customHCLFuncDataFolder)
 	p.RegisterFunction("data_with_permissions", customHCLFuncDataFolderWithPermissions)
+	p.RegisterFunction("system", customHCLFuncSystem)
 
 	return p
 }
-
-func customHCLFuncJumppad() (string, error) {
-	return utils.JumppadHome(), nil
-}
-
-// returns the docker host ip address
-func customHCLFuncDockerIP() (string, error) {
-	return utils.GetDockerIP(), nil
-}
-
-func customHCLFuncDockerHost() (string, error) {
-	return utils.GetDockerHost(), nil
-}
-
-func customHCLFuncDataFolderWithPermissions(name string, permissions int) (string, error) {
-	if permissions > 0 && permissions < 778 {
-		return "", fmt.Errorf("permissions must be a three digit number less than 777")
-	}
-
-	// convert the permissions to an octal e.g. 777 to 0777
-	strInt := fmt.Sprintf("0%d", permissions)
-	oInt, _ := strconv.ParseInt(strInt, 8, 64)
-
-	perms := os.FileMode(oInt)
-	return utils.DataFolder(name, perms), nil
-}
-
-func customHCLFuncDataFolder(name string) (string, error) {
-	perms := os.FileMode(0775)
-	return utils.DataFolder(name, perms), nil
-}