scrub struct before log

Author: iQQBot

components/scrubber/scrubber.go

@@ -10,6 +10,7 @@ import (
 	"reflect"
 	"regexp"
 	"strings"
+	"unsafe"
 
 	"github.com/mitchellh/reflectwalk"
 )
@@ -86,6 +87,8 @@ type Scrubber interface {
 	//   }
 	//
 	Struct(val any) error
+
+	DeepCopyStruct(val any) any
 }
 
 // Default is the default scrubber consumers of this package should use
@@ -189,6 +192,127 @@ func (s *scrubberImpl) Struct(val any) error {
 	return nil
 }
 
+func (s *scrubberImpl) deepCopyStruct(fieldName string, src reflect.Value, scrubTag string) reflect.Value {
+	if src.Kind() == reflect.Ptr && src.IsNil() {
+		return reflect.New(src.Type()).Elem()
+	}
+
+	if src.Kind() == reflect.String {
+		dst := reflect.New(src.Type())
+		var (
+			setExplicitValue bool
+			explicitValue    string
+		)
+		switch scrubTag {
+		case "ignore":
+			return dst
+		case "hash":
+			setExplicitValue = true
+			explicitValue = SanitiseHash(src.String())
+		case "redact":
+			setExplicitValue = true
+			explicitValue = SanitiseRedact(src.String())
+		}
+
+		if setExplicitValue {
+			dst.Elem().SetString(explicitValue)
+		} else {
+			sanitisatiser := s.getSanitisatiser(fieldName)
+			if sanitisatiser != nil {
+				dst.Elem().SetString(sanitisatiser(src.String()))
+			}
+		}
+		if !dst.CanInterface() {
+			return dst
+		}
+		return dst.Elem()
+	}
+
+	switch src.Kind() {
+
+	case reflect.Struct:
+		dst := reflect.New(src.Type())
+		t := src.Type()
+
+		for i := 0; i < t.NumField(); i++ {
+			f := t.Field(i)
+			srcValue := src.Field(i)
+			dstValue := dst.Elem().Field(i)
+
+			if !srcValue.CanInterface() {
+				dstValue = reflect.NewAt(dstValue.Type(), unsafe.Pointer(dstValue.UnsafeAddr())).Elem()
+
+				if !srcValue.CanAddr() {
+					switch {
+					case srcValue.CanInt():
+						dstValue.SetInt(srcValue.Int())
+					case srcValue.CanUint():
+						dstValue.SetUint(srcValue.Uint())
+					case srcValue.CanFloat():
+						dstValue.SetFloat(srcValue.Float())
+					case srcValue.CanComplex():
+						dstValue.SetComplex(srcValue.Complex())
+					case srcValue.Kind() == reflect.Bool:
+						dstValue.SetBool(srcValue.Bool())
+					}
+
+					continue
+				}
+
+				srcValue = reflect.NewAt(srcValue.Type(), unsafe.Pointer(srcValue.UnsafeAddr())).Elem()
+			}
+
+			tagValue := f.Tag.Get("scrub")
+			copied := s.deepCopyStruct(f.Name, srcValue, tagValue)
+			dstValue.Set(copied)
+		}
+		return dst.Elem()
+
+	case reflect.Map:
+		dst := reflect.MakeMap(src.Type())
+		keys := src.MapKeys()
+		for i := 0; i < src.Len(); i++ {
+			mValue := src.MapIndex(keys[i])
+			dst.SetMapIndex(keys[i], s.deepCopyStruct(keys[i].String(), mValue, ""))
+		}
+		return dst
+
+	case reflect.Slice:
+		dst := reflect.MakeSlice(src.Type(), src.Len(), src.Cap())
+		for i := 0; i < src.Len(); i++ {
+			dst.Index(i).Set(s.deepCopyStruct(fieldName, src.Index(i), ""))
+		}
+		return dst
+
+	case reflect.Array:
+		if src.Len() == 0 {
+			return src // can not access to src.Index(0)
+		}
+
+		dst := reflect.New(src.Type()).Elem()
+		for i := 0; i < src.Len(); i++ {
+			dst.Index(i).Set(s.deepCopyStruct(fieldName, src.Index(i), ""))
+		}
+		return dst
+
+	case reflect.Ptr:
+		dst := reflect.New(src.Elem().Type())
+		copied := s.deepCopyStruct(fieldName, src.Elem(), scrubTag)
+		dst.Elem().Set(copied)
+		return dst
+
+	default:
+		dst := reflect.New(src.Type())
+		dst.Elem().Set(src)
+		return dst.Elem()
+	}
+}
+
+// Struct implements Scrubber
+func (s *scrubberImpl) DeepCopyStruct(val any) any {
+	return s.deepCopyStruct("", reflect.ValueOf(val), "").Interface()
+}
+
 func (s *scrubberImpl) scrubJsonObject(val map[string]interface{}) error {
 	// fix https://github.com/gitpod-io/security/issues/64
 	name, _ := val["name"].(string)

components/ws-manager-mk2/controllers/workspace_controller.go

@@ -29,7 +29,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
-	"github.com/gitpod-io/gitpod/components/scrubber"
 	"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
 	config "github.com/gitpod-io/gitpod/ws-manager/api/config"
 	workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
@@ -126,20 +125,13 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	r.updateMetrics(ctx, &workspace)
 	r.emitPhaseEvents(ctx, &workspace, oldStatus)
 
-	var scrubbedPodStatus *corev1.PodStatus
+	var podStatus *corev1.PodStatus
 	if len(workspacePods.Items) > 0 {
-		scrubbedPodStatus = workspacePods.Items[0].Status.DeepCopy()
-		if err = scrubber.Default.Struct(scrubbedPodStatus); err != nil {
-			log.Error(err, "failed to scrub pod status")
-		}
-	}
-	scrubbedStatus := workspace.Status.DeepCopy()
-	if err = scrubber.Default.Struct(scrubbedStatus); err != nil {
-		log.Error(err, "failed to scrub workspace status")
+		podStatus = &workspacePods.Items[0].Status
 	}
 
 	if !equality.Semantic.DeepDerivative(oldStatus, workspace.Status) {
-		log.Info("updating workspace status", "status", scrubbedStatus, "podStatus", scrubbedPodStatus)
+		log.Info("updating workspace status", "status", workspace.Status, "podStatus", podStatus)
 	}
 
 	err = r.Status().Update(ctx, &workspace)

components/ws-manager-mk2/go.mod

@@ -62,6 +62,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

components/ws-manager-mk2/go.sum

@@ -22,6 +22,7 @@ import (
 
 	"github.com/bombsimon/logrusr/v4"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
+	"github.com/mitchellh/copystructure"
 	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -43,6 +44,7 @@ import (
 	config "github.com/gitpod-io/gitpod/ws-manager/api/config"
 	workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
 
+	"github.com/gitpod-io/gitpod/components/scrubber"
 	"github.com/gitpod-io/gitpod/ws-manager-mk2/controllers"
 	"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
 	imgproxy "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/proxy"
@@ -77,7 +79,16 @@ func main() {
 	flag.Parse()
 
 	log.Init(ServiceName, Version, jsonLog, verbose)
-	baseLogger := logrusr.New(log.Log)
+	baseLogger := logrusr.New(log.Log, logrusr.WithFormatter(func(i interface{}) interface{} {
+		copied, err := copystructure.Copy(i)
+		if err != nil {
+			return "[error:failed to copy struct]"
+		}
+		if err = scrubber.Default.Struct(copied); err != nil {
+			return "[error:failed to scrub]"
+		}
+		return copied
+	}))
 	ctrl.SetLogger(baseLogger)
 	// Set the logger used by k8s (e.g. client-go).
 	klog.SetLogger(baseLogger)