Skip to content

Commit

Permalink
Merge pull request nephio-project#139 from Nordix/tls-registry
Browse files Browse the repository at this point in the history 
Adding TLS configuration for fn-runner image registries
  • Loading branch information
@efiacor
efiacor authored Nov 21, 2024
2 parents d2fcfb1 + 77aec11 commit c06b695
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 35 deletions.
123 changes: 102 additions & 21 deletions func/internal/podevaluator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,12 @@ package internal

import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
Expand All @@ -28,6 +31,7 @@ import (

"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
containerregistry "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/nephio-project/porch/func/evaluator"
util "github.com/nephio-project/porch/pkg/util"
Expand Down Expand Up @@ -71,7 +75,20 @@ type podEvaluator struct {

var _ Evaluator = &podEvaluator{}

func NewPodEvaluator(namespace, wrapperServerImage string, interval, ttl time.Duration, podTTLConfig string, functionPodTemplateName string, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string) (Evaluator, error) {
func NewPodEvaluator(
namespace,
wrapperServerImage string,
interval,
ttl time.Duration,
podTTLConfig string,
functionPodTemplateName string,
enablePrivateRegistries bool,
registryAuthSecretPath string,
registryAuthSecretName string,
enablePrivateRegistriesTls bool,
tlsSecretPath string,
) (Evaluator, error) {

restCfg, err := config.GetConfig()
if err != nil {
return nil, fmt.Errorf("failed to get rest config: %w", err)
Expand Down Expand Up @@ -100,15 +117,17 @@ func NewPodEvaluator(namespace, wrapperServerImage string, interval, ttl time.Du
pe := &podEvaluator{
requestCh: reqCh,
podCacheManager: &podCacheManager{
gcScanInternal: interval,
podTTL: ttl,
enablePrivateRegistries: enablePrivateRegistries,
registryAuthSecretPath: registryAuthSecretPath,
registryAuthSecretName: registryAuthSecretName,
requestCh: reqCh,
podReadyCh: readyCh,
cache: map[string]*podAndGRPCClient{},
waitlists: map[string][]chan<- *clientConnAndError{},
gcScanInternal: interval,
podTTL: ttl,
enablePrivateRegistries: enablePrivateRegistries,
registryAuthSecretPath: registryAuthSecretPath,
registryAuthSecretName: registryAuthSecretName,
enablePrivateRegistriesTls: enablePrivateRegistriesTls,
tlsSecretPath: tlsSecretPath,
requestCh: reqCh,
podReadyCh: readyCh,
cache: map[string]*podAndGRPCClient{},
waitlists: map[string][]chan<- *clientConnAndError{},

podManager: &podManager{
kubeClient: cl,
Expand Down Expand Up @@ -177,6 +196,9 @@ type podCacheManager struct {
registryAuthSecretPath string
registryAuthSecretName string

enablePrivateRegistriesTls bool
tlsSecretPath string

// requestCh is a receive-only channel to receive
requestCh <-chan *clientConnRequest
// podReadyCh is a channel to receive the information when a pod is ready.
Expand Down Expand Up @@ -245,7 +267,7 @@ func (pcm *podCacheManager) warmupCache(podTTLConfig string) error {

// We invoke the function with useGenerateName=false so that the pod name is fixed,
// since we want to ensure only one pod is created for each function.
pcm.podManager.getFuncEvalPodClient(ctx, fnImage, ttl, false, pcm.enablePrivateRegistries, pcm.registryAuthSecretPath, pcm.registryAuthSecretName)
pcm.podManager.getFuncEvalPodClient(ctx, fnImage, ttl, false, pcm.enablePrivateRegistries, pcm.registryAuthSecretPath, pcm.registryAuthSecretName, pcm.enablePrivateRegistriesTls, pcm.tlsSecretPath)
klog.Infof("preloaded pod cache for function %v", fnImage)
})

Expand Down Expand Up @@ -313,7 +335,7 @@ func (pcm *podCacheManager) podCacheManager() {
pcm.waitlists[req.image] = append(list, req.grpcClientCh)
// We invoke the function with useGenerateName=true to avoid potential name collision, since if pod foo is
// being deleted and we can't use the same name.
go pcm.podManager.getFuncEvalPodClient(context.Background(), req.image, pcm.podTTL, true, pcm.enablePrivateRegistries, pcm.registryAuthSecretPath, pcm.registryAuthSecretName)
go pcm.podManager.getFuncEvalPodClient(context.Background(), req.image, pcm.podTTL, true, pcm.enablePrivateRegistries, pcm.registryAuthSecretPath, pcm.registryAuthSecretName, pcm.enablePrivateRegistriesTls, pcm.tlsSecretPath)
case resp := <-pcm.podReadyCh:
if resp.err != nil {
klog.Warningf("received error from the pod manager: %v", resp.err)
Expand Down Expand Up @@ -445,9 +467,9 @@ type digestAndEntrypoint struct {
// time-to-live period for the pod. If useGenerateName is false, it will try to
// create a pod with a fixed name. Otherwise, it will create a pod and let the
// apiserver to generate the name from a template.
func (pm *podManager) getFuncEvalPodClient(ctx context.Context, image string, ttl time.Duration, useGenerateName bool, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string) {
func (pm *podManager) getFuncEvalPodClient(ctx context.Context, image string, ttl time.Duration, useGenerateName bool, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string, enablePrivateRegistriesTls bool, tlsSecretPath string) {
c, err := func() (*podAndGRPCClient, error) {
podKey, err := pm.retrieveOrCreatePod(ctx, image, ttl, useGenerateName, enablePrivateRegistries, registryAuthSecretPath, registryAuthSecretName)
podKey, err := pm.retrieveOrCreatePod(ctx, image, ttl, useGenerateName, enablePrivateRegistries, registryAuthSecretPath, registryAuthSecretName, enablePrivateRegistriesTls, tlsSecretPath)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -539,7 +561,7 @@ type DockerConfig struct {
}

// imageDigestAndEntrypoint gets the entrypoint of a container image by looking at its metadata.
func (pm *podManager) imageDigestAndEntrypoint(ctx context.Context, image string, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string) (*digestAndEntrypoint, error) {
func (pm *podManager) imageDigestAndEntrypoint(ctx context.Context, image string, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string, enablePrivateRegistriesTls bool, tlsSecretPath string) (*digestAndEntrypoint, error) {
start := time.Now()
defer func() {
klog.Infof("getting image metadata for %v took %v", image, time.Since(start))
Expand Down Expand Up @@ -569,7 +591,7 @@ func (pm *podManager) imageDigestAndEntrypoint(ctx context.Context, image string
}
}

return pm.getImageMetadata(ctx, ref, auth, image)
return pm.getImageMetadata(ctx, ref, auth, image, enablePrivateRegistries, enablePrivateRegistriesTls, tlsSecretPath)
}

// ensureCustomAuthSecret ensures that, if an image from a custom registry is requested, the appropriate credentials are passed into a secret for function pods to use when pulling. If the secret does not already exist, it is created.
Expand All @@ -590,16 +612,16 @@ func (pm *podManager) getCustomAuth(ref name.Reference, registryAuthSecretPath s

var dockerConfig DockerConfig
if err := json.Unmarshal(dockerConfigBytes, &dockerConfig); err != nil {
klog.Errorf("error unmarshalling authentication file %v", err)
klog.Errorf("error unmarshaling authentication file %v", err)
return nil, err
}

return authn.FromConfig(dockerConfig.Auths[ref.Context().RegistryStr()]), nil
}

// getImageMetadata retrieves the image digest and entrypoint.
func (pm *podManager) getImageMetadata(ctx context.Context, ref name.Reference, auth authn.Authenticator, image string) (*digestAndEntrypoint, error) {
img, err := remote.Image(ref, remote.WithAuth(auth), remote.WithContext(ctx))
func (pm *podManager) getImageMetadata(ctx context.Context, ref name.Reference, auth authn.Authenticator, image string, enablePrivateRegistries bool, enablePrivateRegistriesTls bool, tlsSecretPath string) (*digestAndEntrypoint, error) {
img, err := getImage(ctx, ref, auth, image, enablePrivateRegistries, enablePrivateRegistriesTls, tlsSecretPath)
if err != nil {
return nil, err
}
Expand All @@ -625,15 +647,74 @@ func (pm *podManager) getImageMetadata(ctx context.Context, ref name.Reference,
return de, nil
}

func getImage(ctx context.Context, ref name.Reference, auth authn.Authenticator, image string, enablePrivateRegistries bool, enablePrivateRegistriesTls bool, tlsSecretPath string) (containerregistry.Image, error) {
// if private registries or their appropriate tls configuration are disabled in the config we pull image with default operation otherwise try and use their tls cert's
if !enablePrivateRegistries || strings.HasPrefix(image, defaultRegistry) || !enablePrivateRegistriesTls {
return remote.Image(ref, remote.WithAuth(auth), remote.WithContext(ctx))
}
tlsFile := "ca.crt"
// Check if mounted secret location contains CA file.
if _, err := os.Stat(tlsSecretPath); os.IsNotExist(err) {
return nil, err
}
if _, errCRT := os.Stat(filepath.Join(tlsSecretPath, "ca.crt")); os.IsNotExist(errCRT) {
if _, errPEM := os.Stat(filepath.Join(tlsSecretPath, "ca.pem")); os.IsNotExist(errPEM) {
return nil, fmt.Errorf("ca.crt not found: %v, and ca.pem also not found: %v", errCRT, errPEM)
}
tlsFile = "ca.pem"
}
// Load the custom TLS configuration
tlsConfig, err := loadTLSConfig(filepath.Join(tlsSecretPath, tlsFile))
if err != nil {
return nil, err
}
// Create a custom HTTPS transport
transport := createTransport(tlsConfig)

// Attempt image pull with given custom TLS cert
img, tlsErr := remote.Image(ref, remote.WithAuth(auth), remote.WithContext(ctx), remote.WithTransport(transport))
if tlsErr != nil {
// Attempt without given custom TLS cert but with default keychain
klog.Errorf("Pulling image %s with the provided TLS Cert has failed with error %v", image, tlsErr)
klog.Infof("Attempting image pull with default keychain instead of provided TLS Cert")
return remote.Image(ref, remote.WithAuth(auth), remote.WithContext(ctx))
}
return img, tlsErr
}

func loadTLSConfig(caCertPath string) (*tls.Config, error) {
// Read the CA certificate file
caCert, err := os.ReadFile(caCertPath)
if err != nil {
return nil, err
}
// Append the CA certificate to the system pool
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM(caCert) {
return nil, fmt.Errorf("failed to append certificates from PEM")
}
// Create a tls.Config with the CA pool
tlsConfig := &tls.Config{
RootCAs: caCertPool,
}
return tlsConfig, nil
}

func createTransport(tlsConfig *tls.Config) *http.Transport {
return &http.Transport{
TLSClientConfig: tlsConfig,
}
}

// retrieveOrCreatePod retrieves or creates a pod for an image.
func (pm *podManager) retrieveOrCreatePod(ctx context.Context, image string, ttl time.Duration, useGenerateName bool, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string) (client.ObjectKey, error) {
func (pm *podManager) retrieveOrCreatePod(ctx context.Context, image string, ttl time.Duration, useGenerateName bool, enablePrivateRegistries bool, registryAuthSecretPath string, registryAuthSecretName string, enablePrivateRegistriesTls bool, tlsSecretPath string) (client.ObjectKey, error) {
var de *digestAndEntrypoint
var replacePod bool
var currentPod *corev1.Pod
var err error
val, found := pm.imageMetadataCache.Load(image)
if !found {
de, err = pm.imageDigestAndEntrypoint(ctx, image, enablePrivateRegistries, registryAuthSecretPath, registryAuthSecretName)
de, err = pm.imageDigestAndEntrypoint(ctx, image, enablePrivateRegistries, registryAuthSecretPath, registryAuthSecretName, enablePrivateRegistriesTls, tlsSecretPath)
if err != nil {
return client.ObjectKey{}, fmt.Errorf("unable to get the entrypoint for %v: %w", image, err)
}
Expand Down
2 changes: 1 addition & 1 deletion func/internal/podevaluator_podmanager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -644,7 +644,7 @@ func TestPodManager(t *testing.T) {
fakeServer.evalFunc = tt.evalFunc

//Execute the function under test
go pm.getFuncEvalPodClient(ctx, tt.functionImage, time.Hour, tt.useGenerateName, false, "", "auth-secret")
go pm.getFuncEvalPodClient(ctx, tt.functionImage, time.Hour, tt.useGenerateName, false, "/var/tmp/config-secret/.dockerconfigjson", "auth-secret", false, "/var/tmp/tls-secret/")

if tt.podPatch != nil {
go func() {
Expand Down
28 changes: 15 additions & 13 deletions func/server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,18 +38,20 @@ const (
)

var (
port = flag.Int("port", 9445, "The server port")
functions = flag.String("functions", "./functions", "Path to cached functions.")
config = flag.String("config", "./config.yaml", "Path to the config file.")
enablePrivateRegistries = flag.Bool("enable-private-registry", false, "if true enables the use of private registries and their authentication")
registryAuthSecretPath = flag.String("registry-auth-secret-path", "/var/tmp/config-secret/.dockerconfigjson", "The path of the secret used in custom registry authentication")
registryAuthSecretName = flag.String("registry-auth-secret-name", "auth-secret", "The name of the secret used in custom registry authentication")
podCacheConfig = flag.String("pod-cache-config", "/pod-cache-config/pod-cache-config.yaml", "Path to the pod cache config file. The file is map of function name to TTL.")
podNamespace = flag.String("pod-namespace", "porch-fn-system", "Namespace to run KRM functions pods.")
podTTL = flag.Duration("pod-ttl", 30*time.Minute, "TTL for pods before GC.")
scanInterval = flag.Duration("scan-interval", time.Minute, "The interval of GC between scans.")
disableRuntimes = flag.String("disable-runtimes", "", fmt.Sprintf("The runtime(s) to disable. Multiple runtimes should separated by `,`. Available runtimes: `%v`, `%v`.", execRuntime, podRuntime))
functionPodTemplateName = flag.String("function-pod-template", "", "Configmap that contains a pod specification")
port = flag.Int("port", 9445, "The server port")
functions = flag.String("functions", "./functions", "Path to cached functions.")
config = flag.String("config", "./config.yaml", "Path to the config file.")
enablePrivateRegistries = flag.Bool("enable-private-registries", false, "if true enables the use of private registries and their authentication")
registryAuthSecretPath = flag.String("registry-auth-secret-path", "/var/tmp/config-secret/.dockerconfigjson", "The path of the secret used for authenticating to custom registries")
registryAuthSecretName = flag.String("registry-auth-secret-name", "auth-secret", "The name of the secret used for authenticating to custom registries")
enablePrivateRegistriesTls = flag.Bool("enable-private-registries-tls", false, "if enabled, will prioritize use of user provided TLS secret when accessing registries")
tlsSecretPath = flag.String("tls-secret-path", "/var/tmp/tls-secret/", "The path of the secret used in tls configuration")
podCacheConfig = flag.String("pod-cache-config", "/pod-cache-config/pod-cache-config.yaml", "Path to the pod cache config file. The file is map of function name to TTL.")
podNamespace = flag.String("pod-namespace", "porch-fn-system", "Namespace to run KRM functions pods.")
podTTL = flag.Duration("pod-ttl", 30*time.Minute, "TTL for pods before GC.")
scanInterval = flag.Duration("scan-interval", time.Minute, "The interval of GC between scans.")
disableRuntimes = flag.String("disable-runtimes", "", fmt.Sprintf("The runtime(s) to disable. Multiple runtimes should separated by `,`. Available runtimes: `%v`, `%v`.", execRuntime, podRuntime))
functionPodTemplateName = flag.String("function-pod-template", "", "Configmap that contains a pod specification")
)

func main() {
Expand Down Expand Up @@ -92,7 +94,7 @@ func run() error {
if wrapperServerImage == "" {
return fmt.Errorf("environment variable %v must be set to use pod function evaluator runtime", wrapperServerImageEnv)
}
podEval, err := internal.NewPodEvaluator(*podNamespace, wrapperServerImage, *scanInterval, *podTTL, *podCacheConfig, *functionPodTemplateName, *enablePrivateRegistries, *registryAuthSecretPath, *registryAuthSecretName)
podEval, err := internal.NewPodEvaluator(*podNamespace, wrapperServerImage, *scanInterval, *podTTL, *podCacheConfig, *functionPodTemplateName, *enablePrivateRegistries, *registryAuthSecretPath, *registryAuthSecretName, *enablePrivateRegistriesTls, *tlsSecretPath)
if err != nil {
return fmt.Errorf("failed to initialize pod evaluator: %w", err)
}
Expand Down

0 comments on commit c06b695

Please sign in to comment.