gitrepo: Add support for specifying proxy per GitRepository

Add `.spec.proxy.secretRef.name` to `GitRepository` to allow referencing
a secret containing the proxy settings to be used for all remote Git
operations for a particular object.

Signed-off-by: Sanskar Jaiswal <[email protected]>

Author: aryan9600

api/v1/gitrepository_types.go

@@ -78,6 +78,11 @@ type GitRepositorySpec struct {
 	// +optional
 	Verification *GitRepositoryVerification `json:"verify,omitempty"`
 
+	// Proxy specifies the proxy configuration to use while contacting
+	// the Git server.
+	// +optional
+	Proxy *GitRepositoryProxy `json:"proxy,omitempty"`
+
 	// Ignore overrides the set of excluded patterns in the .sourceignore format
 	// (which is the same as .gitignore). If not provided, a default will be used,
 	// consult the documentation for your version to find out what those are.
@@ -175,6 +180,13 @@ type GitRepositoryVerification struct {
 	SecretRef meta.LocalObjectReference `json:"secretRef"`
 }
 
+type GitRepositoryProxy struct {
+	// SecretRef specifies the Secret containing the proxy address and optionally
+	// the proxy username and password.
+	// +required
+	SecretRef meta.LocalObjectReference `json:"secretRef"`
+}
+
 // GitRepositoryStatus records the observed state of a Git repository.
 type GitRepositoryStatus struct {
 	// ObservedGeneration is the last observed generation of the GitRepository

api/v1/zz_generated.deepcopy.go

@@ -90,6 +90,23 @@ spec:
                 description: Interval at which to check the GitRepository for updates.
                 pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
                 type: string
+              proxy:
+                description: Proxy specifies the proxy configuration to use while
+                  contacting the Git server.
+                properties:
+                  secretRef:
+                    description: SecretRef specifies the Secret containing the proxy
+                      address and optionally the proxy username and password.
+                    properties:
+                      name:
+                        description: Name of the referent.
+                        type: string
+                    required:
+                    - name
+                    type: object
+                required:
+                - secretRef
+                type: object
               recurseSubmodules:
                 description: RecurseSubmodules enables the initialization of all submodules
                   within the GitRepository as cloned from the URL, using their default

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -157,6 +157,21 @@ signature(s).</p>
 </tr>
 <tr>
 <td>
+<code>proxy</code><br>
+<em>
+<a href="#source.toolkit.fluxcd.io/v1.GitRepositoryProxy">
+GitRepositoryProxy
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Proxy specifies the proxy configuration to use while contacting
+the Git server.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>ignore</code><br>
 <em>
 string
@@ -401,6 +416,40 @@ the GitRepositoryRef.</p>
 </table>
 </div>
 </div>
+<h3 id="source.toolkit.fluxcd.io/v1.GitRepositoryProxy">GitRepositoryProxy
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#source.toolkit.fluxcd.io/v1.GitRepositorySpec">GitRepositorySpec</a>)
+</p>
+<div class="md-typeset__scrollwrap">
+<div class="md-typeset__table">
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>secretRef</code><br>
+<em>
+<a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">
+github.com/fluxcd/pkg/apis/meta.LocalObjectReference
+</a>
+</em>
+</td>
+<td>
+<p>SecretRef specifies the Secret containing the proxy address and optionally
+the proxy username and password.</p>
+</td>
+</tr>
+</tbody>
+</table>
+</div>
+</div>
 <h3 id="source.toolkit.fluxcd.io/v1.GitRepositoryRef">GitRepositoryRef
 </h3>
 <p>
@@ -593,6 +642,21 @@ signature(s).</p>
 </tr>
 <tr>
 <td>
+<code>proxy</code><br>
+<em>
+<a href="#source.toolkit.fluxcd.io/v1.GitRepositoryProxy">
+GitRepositoryProxy
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Proxy specifies the proxy configuration to use while contacting
+the Git server.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>ignore</code><br>
 <em>
 string

docs/api/v1/source.md

@@ -451,10 +451,53 @@ controller with the argument `--feature-gates=OptimizedGitClones=false`.
 NB: GitRepository objects configured for SemVer or Commit clones are
 not affected by this functionality.
 
-#### Proxy support
+### Proxy
 
-When a proxy is configured in the source-controller Pod through the appropriate
-environment variables, for example `HTTPS_PROXY`, `NO_PROXY`, etc.
+`.spec.proxy.secretRef.name` is an optional field to specify a name of a Secret
+containing the proxy settings that need to be used for all remote Git operations
+for the particular GitRepository object. The Secret can contain three keys:
+
+- `address`: The address of the proxy server. This is a required key.
+- `username`: The username to use if the proxy server is protected by basic
+   authentication. This is an optinal key.
+- `password`: The password to use if the proxy server is protected by basic
+   authentication. This is an optinal key.
+
+The proxy server must be either HTTP/S or SOCKS5. You can use a SOCKS5 proxy
+with a HTTP/S Git repository url.
+
+Examples:
+
+```yaml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: http-proxy
+type: Opaque
+stringData:
+  address: http://proxy.com
+  username: mandalorian
+  password: grogu
+```
+
+```yaml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ssh-proxy
+type: Opaque
+stringData:
+  address: socks5://proxy.com
+  username: mandalorian
+  password: grogu
+```
+
+Proxying can also be configured in the source-controller pod directly by using
+the standard environment variables such as `HTTPS_PROXY`, `ALL_PROXY`, etc.
+
+`.spec.proxy.secretRef.name` takes precedence over all environment variables.
 
 ### Recurse submodules
 

docs/spec/v1/gitrepositories.md

@@ -21,6 +21,9 @@ replace github.com/docker/docker => github.com/docker/docker v23.0.6+incompatibl
 // is compatible with github.com/google/go-containerregistry v0.15.x.
 replace github.com/google/go-containerregistry => github.com/google/go-containerregistry v0.14.1-0.20230409045903-ed5c185df419
 
+// temp replace directive to enable the use of `WithProxy()`
+replace github.com/fluxcd/pkg/git/gogit => github.com/fluxcd/pkg/git/gogit v0.11.2-0.20230530083208-12339fbae0ae
+
 require (
 	cloud.google.com/go/storage v1.30.1
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1

go.mod

@@ -382,8 +382,8 @@ github.com/fluxcd/pkg/apis/meta v1.1.0 h1:vYU1mvUzztnQyTzZOLHQ3wm/tXd7E1QZ2V91zu
 github.com/fluxcd/pkg/apis/meta v1.1.0/go.mod h1:/QwCotRKL/BT6RSa4O75FlYW14fU8eRfKnoagzbkmL4=
 github.com/fluxcd/pkg/git v0.12.2 h1:96xH3hy3WfwiD0DioyJZcGapYT3lmPc2s7jU5UM8buw=
 github.com/fluxcd/pkg/git v0.12.2/go.mod h1:9TG4fEfGCF1XHLt9Xs7X2YOmkmWOiwfjH9tdGIQs8/8=
-github.com/fluxcd/pkg/git/gogit v0.11.1 h1:17UbHEPQovLOhlrsPaDoJa3J7jX0I7G92TWXeEDf2eU=
-github.com/fluxcd/pkg/git/gogit v0.11.1/go.mod h1:Hh358WYfwmvGf6Aaj1wjGZMN2AWlAcXRR6aubMQYq8M=
+github.com/fluxcd/pkg/git/gogit v0.11.2-0.20230530083208-12339fbae0ae h1:JCU+9lTKHPIE7bS5Q9DUoGNa2QuGNhIch/3rt5v8Pso=
+github.com/fluxcd/pkg/git/gogit v0.11.2-0.20230530083208-12339fbae0ae/go.mod h1:Kn+GfYfZBBIaXmQj39cQvrDxT/6y8leQxXZ5/B+YYTQ=
 github.com/fluxcd/pkg/gittestserver v0.8.4 h1:rA/QUZnfH77ZZG+5xfMqjgEHJdzeeE6Nn1o8cops/bU=
 github.com/fluxcd/pkg/gittestserver v0.8.4/go.mod h1:i3Vng3Stl5zOuGhN4+RuP2NWf5snJCeGUKA7pzAvcHU=
 github.com/fluxcd/pkg/helmtestserver v0.13.0 h1:bRzOO955nDKWKJZvDORfmDvRdb/558BX4ffgx1vT4LI=

go.sum

@@ -28,6 +28,7 @@ import (
 
 	securejoin "github.com/cyphar/filepath-securejoin"
 	"github.com/fluxcd/pkg/runtime/logger"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -473,24 +474,50 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch
 		conditions.Delete(obj, sourcev1.SourceVerifiedCondition)
 	}
 
+	var proxyOpts *transport.ProxyOptions
+	if obj.Spec.Proxy != nil {
+		proxySecretName := obj.Spec.Proxy.SecretRef.Name
+		proxyData, err := r.getSecretData(ctx, proxySecretName, obj.GetNamespace())
+		if err != nil {
+			e := serror.NewGeneric(
+				fmt.Errorf("failed to get secret '%s/%s': %w", obj.GetNamespace(), proxySecretName, err),
+				sourcev1.AuthenticationFailedReason,
+			)
+			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
+			// Return error as the world as observed may change
+			return sreconcile.ResultEmpty, e
+		}
+		address, ok := proxyData["address"]
+		if !ok {
+			e := serror.NewGeneric(
+				fmt.Errorf("invalid proxy secret '%s/%s': key 'address' is missing", obj.GetNamespace(), proxySecretName),
+				sourcev1.AuthenticationFailedReason,
+			)
+			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
+			// Return error as the world as observed may change
+			return sreconcile.ResultEmpty, e
+		}
+		proxyOpts = &transport.ProxyOptions{
+			URL: string(address),
+		}
+		proxyOpts.Username = string(proxyData["username"])
+		proxyOpts.Password = string(proxyData["password"])
+	}
+
 	var authData map[string][]byte
 	if obj.Spec.SecretRef != nil {
 		// Attempt to retrieve secret
-		name := types.NamespacedName{
-			Namespace: obj.GetNamespace(),
-			Name:      obj.Spec.SecretRef.Name,
-		}
-		var secret corev1.Secret
-		if err := r.Client.Get(ctx, name, &secret); err != nil {
+		var err error
+		authData, err = r.getSecretData(ctx, obj.Spec.SecretRef.Name, obj.GetNamespace())
+		if err != nil {
 			e := serror.NewGeneric(
-				fmt.Errorf("failed to get secret '%s': %w", name.String(), err),
+				fmt.Errorf("failed to get secret '%s/%s': %w", obj.GetNamespace(), obj.Spec.SecretRef.Name, err),
 				sourcev1.AuthenticationFailedReason,
 			)
 			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
 			// Return error as the world as observed may change
 			return sreconcile.ResultEmpty, e
 		}
-		authData = secret.Data
 	}
 
 	u, err := url.Parse(obj.Spec.URL)
@@ -541,7 +568,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch
 		optimizedClone = true
 	}
 
-	c, err := r.gitCheckout(ctx, obj, authOpts, dir, optimizedClone)
+	c, err := r.gitCheckout(ctx, obj, authOpts, proxyOpts, dir, optimizedClone)
 	if err != nil {
 		return sreconcile.ResultEmpty, err
 	}
@@ -583,7 +610,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch
 
 		// If we can't skip the reconciliation, checkout again without any
 		// optimization.
-		c, err := r.gitCheckout(ctx, obj, authOpts, dir, false)
+		c, err := r.gitCheckout(ctx, obj, authOpts, proxyOpts, dir, false)
 		if err != nil {
 			return sreconcile.ResultEmpty, err
 		}
@@ -611,6 +638,18 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch
 	return sreconcile.ResultSuccess, nil
 }
 
+func (r *GitRepositoryReconciler) getSecretData(ctx context.Context, name, namespace string) (map[string][]byte, error) {
+	key := types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}
+	var secret corev1.Secret
+	if err := r.Client.Get(ctx, key, &secret); err != nil {
+		return nil, err
+	}
+	return secret.Data, nil
+}
+
 // reconcileArtifact archives a new Artifact to the Storage, if the current
 // (Status) data on the object does not match the given.
 //
@@ -782,8 +821,8 @@ func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, sp *patc
 // gitCheckout builds checkout options with the given configurations and
 // performs a git checkout.
 func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
-	obj *sourcev1.GitRepository, authOpts *git.AuthOptions, dir string,
-	optimized bool) (*git.Commit, error) {
+	obj *sourcev1.GitRepository, authOpts *git.AuthOptions, proxyOpts *transport.ProxyOptions,
+	dir string, optimized bool) (*git.Commit, error) {
 	// Configure checkout strategy.
 	cloneOpts := repository.CloneConfig{
 		RecurseSubmodules: obj.Spec.RecurseSubmodules,
@@ -813,6 +852,9 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
 	if authOpts.Transport == git.HTTP {
 		clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
 	}
+	if proxyOpts != nil {
+		clientOpts = append(clientOpts, gogit.WithProxy(*proxyOpts))
+	}
 
 	gitReader, err := gogit.NewClient(dir, authOpts, clientOpts...)
 	if err != nil {