Mirror repository rewrites (v1.1)

Support CRI configuration to allow for request-time rewrite rules
applicable only to the repository portion of resource paths when pulling
images. Because the rewrites are applied at request time, images
themselves will not be "rewritten" -- images as stored by CRI (and the
underlying containerd facility) will continue to present as normal.

As an example, if you use the following config for your containerd:
```toml
[plugins]
  [plugins."io.containerd.grpc.v1.cri"]
    [plugins."io.containerd.grpc.v1.cri".registry]
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://registry-1.docker.io/v2"]
       	  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io".rewrite]
            "^library/(.*)" = "my-org/$1"
```

And then subsequently invoke `crictl pull alpine:3.13` it will pull
content from `docker.io/my-org/alpine:3.13` but still show up as
`docker.io/library/alpine:3.13` in the `crictl images` listing.

This commit has been reworked from the original implementation. Rewites
are now done when resolving instead of when building the request, so
that auth token scopes stored in the context properly reflect the
rewritten repository path. For the original implementation, see
06c4ea9.
Ref: k3s-io/k3s#11191 (comment)

Signed-off-by: Jacob Blain Christen <[email protected]>
Co-authored-by: Brad Davidson <[email protected]>
Signed-off-by: Brad Davidson <[email protected]>

pkg/cri/config/config.go

@@ -199,6 +199,23 @@ type Mirror struct {
 	// with host specified.
 	// The scheme, host and path from the endpoint URL will be used.
 	Endpoints []string `toml:"endpoint" json:"endpoint"`
+
+	// Rewrites are repository rewrite rules for a namespace. When fetching image resources
+	// from an endpoint and a key matches the repository via regular expression matching
+	// it will be replaced with the corresponding value from the map in the resource request.
+	//
+	// This example configures CRI to pull docker.io/library/* images from docker.io/my-org/*:
+	//
+	// [plugins]
+	//   [plugins."io.containerd.grpc.v1.cri"]
+	//     [plugins."io.containerd.grpc.v1.cri".registry]
+	//       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+	//         [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+	//           endpoint = ["https://registry-1.docker.io/v2"]
+	//           [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io".rewrite]
+	//             "^library/(.*)" = "my-org/$1"
+	//
+	Rewrites map[string]string `toml:"rewrite" json:"rewrite"`
 }
 
 // AuthConfig contains the config related to authentication to a specific registry

pkg/cri/server/image_pull.go

@@ -449,6 +449,10 @@ func (c *criService) registryHosts(ctx context.Context, auth *runtime.AuthConfig
 		if err != nil {
 			return nil, fmt.Errorf("get registry endpoints: %w", err)
 		}
+		rewrites, err := c.registryRewrites(host)
+		if err != nil {
+			return nil, fmt.Errorf("get registry rewrites: %w", err)
+		}
 		for _, e := range endpoints {
 			u, err := url.Parse(e)
 			if err != nil {
@@ -503,6 +507,7 @@ func (c *criService) registryHosts(ctx context.Context, auth *runtime.AuthConfig
 				Scheme:       u.Scheme,
 				Path:         u.Path,
 				Capabilities: docker.HostCapabilityResolve | docker.HostCapabilityPull,
+				Rewrites:     rewrites,
 			})
 		}
 		return registries, nil
@@ -565,6 +570,20 @@ func (c *criService) registryEndpoints(host string) ([]string, error) {
 	return append(endpoints, defaultScheme(defaultHost)+"://"+defaultHost), nil
 }
 
+func (c *criService) registryRewrites(host string) (map[string]string, error) {
+	var rewrites map[string]string
+	_, ok := c.config.Registry.Mirrors[host]
+	if ok {
+		rewrites = c.config.Registry.Mirrors[host].Rewrites
+	} else {
+		rewrites = c.config.Registry.Mirrors["*"].Rewrites
+	}
+	if rewrites == nil {
+		rewrites = map[string]string{}
+	}
+	return rewrites, nil
+}
+
 // newTransport returns a new HTTP transport used to pull image.
 // TODO(random-liu): Create a library and share this code with `ctr`.
 func newTransport() *http.Transport {

remotes/docker/registry.go

@@ -74,6 +74,7 @@ type RegistryHost struct {
 	Path         string
 	Capabilities HostCapabilities
 	Header       http.Header
+	Rewrites     map[string]string
 }
 
 func (h RegistryHost) isProxy(refhost string) bool {

remotes/docker/resolver.go

@@ -26,6 +26,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"regexp"
 	"strings"
 
 	"github.com/containerd/log"
@@ -237,15 +238,14 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp
 	if err != nil {
 		return "", ocispec.Descriptor{}, err
 	}
-	refspec := base.refspec
-	if refspec.Object == "" {
+	if base.refspec.Object == "" {
 		return "", ocispec.Descriptor{}, reference.ErrObjectRequired
 	}
 
 	var (
 		firstErr error
 		paths    [][]string
-		dgst     = refspec.Digest()
+		dgst     = base.refspec.Digest()
 		caps     = HostCapabilityPull
 	)
 
@@ -263,7 +263,7 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp
 		paths = append(paths, []string{"blobs", dgst.String()})
 	} else {
 		// Add
-		paths = append(paths, []string{"manifests", refspec.Object})
+		paths = append(paths, []string{"manifests", base.refspec.Object})
 		caps |= HostCapabilityResolve
 	}
 
@@ -272,15 +272,14 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp
 		return "", ocispec.Descriptor{}, fmt.Errorf("no resolve hosts: %w", errdefs.ErrNotFound)
 	}
 
-	ctx, err = ContextWithRepositoryScope(ctx, refspec, false)
-	if err != nil {
-		return "", ocispec.Descriptor{}, err
-	}
-
 	for _, u := range paths {
 		for _, host := range hosts {
 			ctx := log.WithLogger(ctx, log.G(ctx).WithField("host", host.Host))
-
+			base := base.withRewritesFromHost(host)
+			ctx, err = ContextWithRepositoryScope(ctx, base.refspec, false)
+			if err != nil {
+				return "", ocispec.Descriptor{}, err
+			}
 			req := base.request(host, http.MethodHead, u...)
 			if err := req.addNamespace(base.refspec.Hostname()); err != nil {
 				return "", ocispec.Descriptor{}, err
@@ -481,11 +480,22 @@ func (r *dockerBase) request(host RegistryHost, method string, ps ...string) *re
 	if header == nil {
 		header = http.Header{}
 	}
-
 	for key, value := range host.Header {
 		header[key] = append(header[key], value...)
 	}
-	parts := append([]string{"/", host.Path, r.repository}, ps...)
+	repository := r.repository
+	for pattern, replace := range host.Rewrites {
+		exp, err := regexp.Compile(pattern)
+		if err != nil {
+			log.L.WithError(err).Warnf("Failed to compile rewrite, `%s`, for %s", pattern, host.Host)
+			continue
+		}
+		if rr := exp.ReplaceAllString(repository, replace); rr != repository {
+			repository = rr
+			break
+		}
+	}
+	parts := append([]string{"/", host.Path, repository}, ps...)
 	p := path.Join(parts...)
 	// Join strips trailing slash, re-add ending "/" if included
 	if len(parts) > 0 && strings.HasSuffix(parts[len(parts)-1], "/") {
@@ -499,6 +509,27 @@ func (r *dockerBase) request(host RegistryHost, method string, ps ...string) *re
 	}
 }
 
+func (r *dockerBase) withRewritesFromHost(host RegistryHost) *dockerBase {
+	for pattern, replace := range host.Rewrites {
+		exp, err := regexp.Compile(pattern)
+		if err != nil {
+			log.L.WithError(err).Warnf("Failed to compile rewrite, `%s`, for %s", pattern, host.Host)
+			continue
+		}
+		if rr := exp.ReplaceAllString(r.repository, replace); rr != r.repository {
+			return &dockerBase{
+				refspec: reference.Spec{
+					Locator: r.refspec.Hostname() + "/" + rr,
+					Object:  r.refspec.Object,
+				},
+				repository: rr,
+				header:     r.header,
+			}
+		}
+	}
+	return r
+}
+
 func (r *request) authorize(ctx context.Context, req *http.Request) error {
 	// Check if has header for host
 	if r.host.Authorizer != nil {