Skip to content

Commit 4555b08

Browse files
csweichelcsweichel
committed
WIP: use registry mirror for image builds
1 parent f68ad77 commit 4555b08

File tree

7 files changed

+218
-7
lines changed

7 files changed

+218
-7
lines changed

‎components/image-builder-bob/cmd/mirror.go

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright (c) 2021 Gitpod GmbH. All rights reserved.
2+
// Licensed under the GNU Affero General Public License (AGPL).
3+
// See License.AGPL.txt in the project root for license information.
4+
5+
package cmd
6+
7+
import (
8+
"net/http"
9+
"os"
10+
11+
"github.com/spf13/cobra"
12+
13+
log "github.com/gitpod-io/gitpod/common-go/log"
14+
"github.com/gitpod-io/gitpod/image-builder/bob/pkg/proxy"
15+
)
16+
17+
var mirrorOpts struct {
18+
Auth string
19+
AdditionalAuth string
20+
}
21+
22+
// mirrorCmd represents the build command
23+
var mirrorCmd = &cobra.Command{
24+
Use: "mirror",
25+
Short: "Runs an authenticating mirror",
26+
Run: func(cmd *cobra.Command, args []string) {
27+
log.Init("bob", "", true, os.Getenv("SUPERVISOR_DEBUG_ENABLE") == "true")
28+
log := log.WithField("command", "mirror")
29+
30+
authP, err := proxy.NewAuthorizerFromDockerEnvVar(mirrorOpts.Auth)
31+
if err != nil {
32+
log.WithError(err).WithField("auth", mirrorOpts.Auth).Fatal("cannot unmarshal auth")
33+
}
34+
authA, err := proxy.NewAuthorizerFromEnvVar(mirrorOpts.AdditionalAuth)
35+
if err != nil {
36+
log.WithError(err).WithField("auth", mirrorOpts.Auth).Fatal("cannot unmarshal auth")
37+
}
38+
authP = authP.AddIfNotExists(authA)
39+
40+
prx := proxy.NewRegistryMirror(authP)
41+
http.Handle("/", prx)
42+
log.Info("starting bob mirror on :5000")
43+
err = http.ListenAndServe(":5000", nil)
44+
if err != nil {
45+
log.Fatal(err)
46+
}
47+
},
48+
}
49+
50+
func init() {
51+
rootCmd.AddCommand(mirrorCmd)
52+
53+
// These env vars start with `WORKSPACEKIT_` so that they aren't passed on to ring2
54+
mirrorCmd.Flags().StringVar(&mirrorOpts.Auth, "auth", os.Getenv("WORKSPACEKIT_BOBmirror_AUTH"), "authentication to use")
55+
mirrorCmd.Flags().StringVar(&mirrorOpts.AdditionalAuth, "additional-auth", os.Getenv("WORKSPACEKIT_BOBmirror_ADDITIONALAUTH"), "additional authentication to use")
56+
}

‎components/image-builder-bob/go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ require (
99
github.com/docker/distribution v2.8.1+incompatible
1010
github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
1111
github.com/gofrs/flock v0.8.1 // indirect
12-
github.com/hashicorp/go-retryablehttp v0.7.1
12+
github.com/hashicorp/go-retryablehttp v0.7.4
1313
github.com/moby/buildkit v0.11.6
1414
github.com/opencontainers/runtime-spec v1.1.0
1515
github.com/sirupsen/logrus v1.9.3
@@ -32,7 +32,7 @@ require (
3232
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
3333
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
3434
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
35-
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
35+
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
3636
github.com/inconshreveable/mousetrap v1.0.1 // indirect
3737
github.com/klauspost/compress v1.15.12 // indirect
3838
github.com/kr/text v0.2.0 // indirect

‎components/image-builder-bob/go.sum

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎components/image-builder-bob/pkg/proxy/mirror.go

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// Copyright (c) 2023 Gitpod GmbH. All rights reserved.
2+
// Licensed under the GNU Affero General Public License (AGPL).
3+
// See License.AGPL.txt in the project root for license information.
4+
5+
package proxy
6+
7+
import (
8+
"context"
9+
"net/http"
10+
"net/http/httputil"
11+
"net/url"
12+
"sync"
13+
14+
"github.com/containerd/containerd/remotes/docker"
15+
"github.com/gitpod-io/gitpod/common-go/log"
16+
"github.com/hashicorp/go-retryablehttp"
17+
)
18+
19+
func NewRegistryMirror(auth Authorizer) *RegistryMirror {
20+
return &RegistryMirror{
21+
Auth: auth,
22+
proxies: make(map[string]*httputil.ReverseProxy),
23+
}
24+
}
25+
26+
type RegistryMirror struct {
27+
Auth Authorizer
28+
29+
mu sync.Mutex
30+
proxies map[string]*httputil.ReverseProxy
31+
}
32+
33+
// ServeHTTP serves the proxy
34+
func (mirror *RegistryMirror) ServeHTTP(w http.ResponseWriter, r *http.Request) {
35+
ns := r.URL.Query().Get("ns")
36+
if ns == "" {
37+
http.Error(w, "no namespace (ns query parameter) present", http.StatusBadRequest)
38+
}
39+
40+
auth := docker.NewDockerAuthorizer(docker.WithAuthCreds(mirror.Auth.Authorize))
41+
r = r.WithContext(context.WithValue(r.Context(), authKey, auth))
42+
43+
r.RequestURI = ""
44+
45+
mirror.reverse(ns).ServeHTTP(w, r)
46+
}
47+
48+
// reverse produces an authentication-adding reverse proxy for a given repo alias
49+
func (mirror *RegistryMirror) reverse(host string) *httputil.ReverseProxy {
50+
mirror.mu.Lock()
51+
defer mirror.mu.Unlock()
52+
53+
if rp, ok := mirror.proxies[host]; ok {
54+
return rp
55+
}
56+
57+
rp := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "https", Host: host})
58+
59+
client := retryablehttp.NewClient()
60+
client.RetryMax = 3
61+
client.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) {
62+
if err != nil {
63+
log.WithError(err).Warn("saw error during CheckRetry")
64+
return false, err
65+
}
66+
auth, ok := ctx.Value(authKey).(docker.Authorizer)
67+
if !ok || auth == nil {
68+
return false, nil
69+
}
70+
if resp.StatusCode == http.StatusUnauthorized {
71+
err := auth.AddResponses(context.Background(), []*http.Response{resp})
72+
if err != nil {
73+
log.WithError(err).WithField("URL", resp.Request.URL.String()).Warn("cannot add responses although response was Unauthorized")
74+
return false, nil
75+
}
76+
return true, nil
77+
}
78+
79+
return false, nil
80+
}
81+
client.RequestLogHook = func(l retryablehttp.Logger, r *http.Request, i int) {
82+
// Total hack: we need a place to modify the request before retrying, and this log
83+
// hook seems to be the only place. We need to modify the request, because
84+
// maybe we just added the host authorizer in the previous CheckRetry call.
85+
//
86+
// The ReverseProxy sets the X-Forwarded-For header with the host machine
87+
// address. If on a cluster with IPV6 enabled, this will be "::1" (IPV6 equivalent
88+
// of "127.0.0.1"). This can have the knock-on effect of receiving an IPV6
89+
// URL, e.g. auth.ipv6.docker.com instead of auth.docker.com which may not
90+
// exist. By forcing the value to be "127.0.0.1", we ensure consistency
91+
// across clusters.
92+
//
93+
// @link https://golang.org/src/net/http/httputil/reversemirror.go
94+
r.Header.Set("X-Forwarded-For", "127.0.0.1")
95+
96+
auth, ok := r.Context().Value(authKey).(docker.Authorizer)
97+
if !ok || auth == nil {
98+
return
99+
}
100+
_ = auth.Authorize(r.Context(), r)
101+
}
102+
client.ResponseLogHook = func(l retryablehttp.Logger, r *http.Response) {}
103+
104+
rp.Transport = &retryablehttp.RoundTripper{
105+
Client: client,
106+
}
107+
rp.ModifyResponse = func(r *http.Response) error {
108+
if r.StatusCode == http.StatusBadGateway {
109+
// BadGateway makes containerd retry - we don't want that because we retry the upstream
110+
// requests internally.
111+
r.StatusCode = http.StatusInternalServerError
112+
r.Status = http.StatusText(http.StatusInternalServerError)
113+
}
114+
115+
return nil
116+
}
117+
mirror.proxies[host] = rp
118+
return rp
119+
}

‎components/image-builder-bob/pkg/proxy/proxy.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@ import (
1818
"github.com/hashicorp/go-retryablehttp"
1919
)
2020

21-
const authKey = "authKey"
21+
const authKey authKeyType = "authKey"
22+
23+
type authKeyType string
2224

2325
func NewProxy(host *url.URL, aliases map[string]Repo) (*Proxy, error) {
2426
if host.Host == "" || host.Scheme == "" {

‎components/image-builder-mk3/pkg/auth/auth.go

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/docker/cli/cli/config/configfile"
2121
"github.com/docker/distribution/reference"
2222
"github.com/docker/docker/api/types"
23+
"github.com/sirupsen/logrus"
2324
"golang.org/x/xerrors"
2425

2526
"github.com/gitpod-io/gitpod/common-go/log"
@@ -349,7 +350,7 @@ func (a AllowedAuthFor) additionalAuth(domain string) *Authentication {
349350
type ImageBuildAuth map[string]types.AuthConfig
350351

351352
// GetImageBuildAuthFor produces authentication in the format an image builds needs
352-
func (a AllowedAuthFor) GetImageBuildAuthFor(blocklist []string) (res ImageBuildAuth) {
353+
func (a AllowedAuthFor) GetImageBuildAuthFor(ctx context.Context, auth RegistryAuthenticator, blocklist []string) (res ImageBuildAuth) {
353354
res = make(ImageBuildAuth)
354355
for reg := range a.Additional {
355356
var blocked bool
@@ -363,8 +364,36 @@ func (a AllowedAuthFor) GetImageBuildAuthFor(blocklist []string) (res ImageBuild
363364
continue
364365
}
365366
ath := a.additionalAuth(reg)
367+
if ath.Empty() {
368+
continue
369+
}
370+
res[reg] = types.AuthConfig(*ath)
371+
}
372+
for _, reg := range a.Explicit {
373+
ath, err := auth.Authenticate(ctx, reg)
374+
if err != nil {
375+
// TODO(cw): swalloing this error is not the obvious choice. We have to weigh failing all workspace image builds because of
376+
// a single misconfiguration, vs essentially silently ignoring that misconfig.
377+
log.WithError(err).WithField("registry", reg).Warn("cannot add image build auth for explicit registry - image build might fail")
378+
continue
379+
}
380+
if ath.Empty() {
381+
continue
382+
}
366383
res[reg] = types.AuthConfig(*ath)
367384
}
368385

369-
return
386+
if log.Log.Logger.IsLevelEnabled(logrus.DebugLevel) {
387+
keys := make([]string, 0, len(res))
388+
for k := range res {
389+
keys = append(keys, k)
390+
}
391+
log.WithField("registries", trustedStrings(keys)).Debug("providing additional auth for image build")
392+
}
393+
394+
return res
370395
}
396+
397+
type trustedStrings []string
398+
399+
func (trustedStrings) IsTrustedValue() {}

‎components/image-builder-mk3/pkg/orchestrator/orchestrator.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -336,9 +336,10 @@ func (o *Orchestrator) Build(req *protocol.BuildRequest, resp protocol.ImageBuil
336336
wsref, err := reference.ParseNamed(wsrefstr)
337337
var additionalAuth []byte
338338
if err == nil {
339-
additionalAuth, err = json.Marshal(reqauth.GetImageBuildAuthFor([]string{
339+
imgbldAuth := reqauth.GetImageBuildAuthFor(ctx, o.Auth, []string{
340340
reference.Domain(wsref),
341-
}))
341+
})
342+
additionalAuth, err = json.Marshal(imgbldAuth)
342343
if err != nil {
343344
return xerrors.Errorf("cannot marshal additional auth: %w", err)
344345
}

0 commit comments

Comments
 (0)