[ws-manager-mk2] redact log (#18906)

* update to logrusr v4

* Add DeepCopyStruct to scrubber

* use DeepCopyStruct to scrub log

* mark wrokspace.stauts.url redact

Author: iQQBot

components/common-go/log/log.go

@@ -231,3 +231,15 @@ type jsonEntry struct {
 	Msg     string       `json:"msg,omitempty"`
 	Time    *time.Time   `json:"time,omitempty"`
 }
+
+// TrustedValueWrap is a simple wrapper that treats the entire value as trusted, which are not processed by the scrubber.
+// During JSON marshal, only the Value itself will be processed, without including Wrap.
+type TrustedValueWrap struct {
+	Value any
+}
+
+func (TrustedValueWrap) IsTrustedValue() {}
+
+func (t TrustedValueWrap) MarshalJSON() ([]byte, error) {
+	return json.Marshal(t.Value)
+}

components/scrubber/scrubber.go

@@ -10,6 +10,7 @@ import (
 	"reflect"
 	"regexp"
 	"strings"
+	"unsafe"
 
 	"github.com/mitchellh/reflectwalk"
 )
@@ -86,6 +87,12 @@ type Scrubber interface {
 	//   }
 	//
 	Struct(val any) error
+
+	// DeepCopyStruct scrubes a struct with a deep copy.
+	// The difference between `DeepCopyStruct` and `Struct`` is that DeepCopyStruct does not modify the structure directly,
+	// but creates a deep copy instead.
+	// Also, val can be a pointer or a structure.
+	DeepCopyStruct(val any) any
 }
 
 // Default is the default scrubber consumers of this package should use
@@ -189,6 +196,145 @@ func (s *scrubberImpl) Struct(val any) error {
 	return nil
 }
 
+func (s *scrubberImpl) deepCopyStruct(fieldName string, src reflect.Value, scrubTag string, skipScrub bool) reflect.Value {
+	if src.Kind() == reflect.Ptr && src.IsNil() {
+		return reflect.New(src.Type()).Elem()
+	}
+
+	if src.CanInterface() {
+		value := src.Interface()
+		if _, ok := value.(TrustedValue); ok {
+			skipScrub = true
+		}
+	}
+
+	if src.Kind() == reflect.String && !skipScrub {
+		dst := reflect.New(src.Type())
+		var (
+			setExplicitValue bool
+			explicitValue    string
+		)
+		switch scrubTag {
+		case "ignore":
+			dst.Elem().SetString(src.String())
+			if !dst.CanInterface() {
+				return dst
+			}
+			return dst.Elem()
+		case "hash":
+			setExplicitValue = true
+			explicitValue = SanitiseHash(src.String())
+		case "redact":
+			setExplicitValue = true
+			explicitValue = SanitiseRedact(src.String())
+		}
+
+		if setExplicitValue {
+			dst.Elem().SetString(explicitValue)
+		} else {
+			sanitisatiser := s.getSanitisatiser(fieldName)
+			if sanitisatiser != nil {
+				dst.Elem().SetString(sanitisatiser(src.String()))
+			} else {
+				dst.Elem().SetString(s.Value(src.String()))
+			}
+		}
+		if !dst.CanInterface() {
+			return dst
+		}
+		return dst.Elem()
+	}
+
+	switch src.Kind() {
+	case reflect.Struct:
+		dst := reflect.New(src.Type())
+		t := src.Type()
+
+		for i := 0; i < t.NumField(); i++ {
+			f := t.Field(i)
+			srcValue := src.Field(i)
+			dstValue := dst.Elem().Field(i)
+
+			if !srcValue.CanInterface() {
+				dstValue = reflect.NewAt(dstValue.Type(), unsafe.Pointer(dstValue.UnsafeAddr())).Elem()
+
+				if !srcValue.CanAddr() {
+					switch {
+					case srcValue.CanInt():
+						dstValue.SetInt(srcValue.Int())
+					case srcValue.CanUint():
+						dstValue.SetUint(srcValue.Uint())
+					case srcValue.CanFloat():
+						dstValue.SetFloat(srcValue.Float())
+					case srcValue.CanComplex():
+						dstValue.SetComplex(srcValue.Complex())
+					case srcValue.Kind() == reflect.Bool:
+						dstValue.SetBool(srcValue.Bool())
+					}
+
+					continue
+				}
+
+				srcValue = reflect.NewAt(srcValue.Type(), unsafe.Pointer(srcValue.UnsafeAddr())).Elem()
+			}
+
+			tagValue := f.Tag.Get("scrub")
+			copied := s.deepCopyStruct(f.Name, srcValue, tagValue, skipScrub)
+			dstValue.Set(copied)
+		}
+		return dst.Elem()
+
+	case reflect.Map:
+		dst := reflect.MakeMap(src.Type())
+		keys := src.MapKeys()
+		for i := 0; i < src.Len(); i++ {
+			mValue := src.MapIndex(keys[i])
+			dst.SetMapIndex(keys[i], s.deepCopyStruct(keys[i].String(), mValue, "", skipScrub))
+		}
+		return dst
+
+	case reflect.Slice:
+		dst := reflect.MakeSlice(src.Type(), src.Len(), src.Cap())
+		for i := 0; i < src.Len(); i++ {
+			dst.Index(i).Set(s.deepCopyStruct(fieldName, src.Index(i), "", skipScrub))
+		}
+		return dst
+
+	case reflect.Array:
+		if src.Len() == 0 {
+			return src
+		}
+
+		dst := reflect.New(src.Type()).Elem()
+		for i := 0; i < src.Len(); i++ {
+			dst.Index(i).Set(s.deepCopyStruct(fieldName, src.Index(i), "", skipScrub))
+		}
+		return dst
+
+	case reflect.Interface:
+		dst := reflect.New(src.Elem().Type())
+		copied := s.deepCopyStruct(fieldName, src.Elem(), scrubTag, skipScrub)
+		dst.Elem().Set(copied)
+		return dst.Elem()
+
+	case reflect.Ptr:
+		dst := reflect.New(src.Elem().Type())
+		copied := s.deepCopyStruct(fieldName, src.Elem(), scrubTag, skipScrub)
+		dst.Elem().Set(copied)
+		return dst
+
+	default:
+		dst := reflect.New(src.Type())
+		dst.Elem().Set(src)
+		return dst.Elem()
+	}
+}
+
+// Struct implements Scrubber
+func (s *scrubberImpl) DeepCopyStruct(val any) any {
+	return s.deepCopyStruct("", reflect.ValueOf(val), "", false).Interface()
+}
+
 func (s *scrubberImpl) scrubJsonObject(val map[string]interface{}) error {
 	// fix https://github.com/gitpod-io/security/issues/64
 	name, _ := val["name"].(string)

components/scrubber/scrubber_test.go

@@ -5,6 +5,7 @@
 package scrubber
 
 import (
+	"encoding/json"
 	"math/rand"
 	"testing"
 
@@ -68,6 +69,10 @@ type TrustedStructToTest struct {
 	StructToTest
 }
 
+type TestWrap struct {
+	Test *StructToTest
+}
+
 type UnexportedStructToTest struct {
 	Exported      string
 	unexportedPtr *string
@@ -293,6 +298,171 @@ func TestJSON(t *testing.T) {
 	}
 }
 
+func TestDeepCopyStruct(t *testing.T) {
+	type Expectation struct {
+		Error  string
+		Result any
+	}
+	tests := []struct {
+		Name        string
+		Struct      any
+		Expectation Expectation
+		CmpOpts     []cmp.Option
+	}{
+		{
+			Name: "basic happy path",
+			Struct: &struct {
+				Username     string
+				Email        string
+				Password     string
+				WorkspaceID  string
+				LeaveMeAlone string
+			}{Username: "foo", Email: "[email protected]", Password: "foobar", WorkspaceID: "gitpodio-gitpod-uesaddev73c", LeaveMeAlone: "foo"},
+			Expectation: Expectation{
+				Result: &struct {
+					Username     string
+					Email        string
+					Password     string
+					WorkspaceID  string
+					LeaveMeAlone string
+				}{Username: "[redacted:md5:acbd18db4cc2f85cedef654fccc4a4d8]", Email: "[redacted]", Password: "[redacted]", WorkspaceID: "[redacted:md5:a35538939333def8477b5c19ac694b35]", LeaveMeAlone: "foo"},
+			},
+		},
+		{
+			Name: "stuct without pointer",
+			Struct: struct {
+				Username     string
+				Email        string
+				Password     string
+				WorkspaceID  string
+				LeaveMeAlone string
+			}{Username: "foo", Email: "[email protected]", Password: "foobar", WorkspaceID: "gitpodio-gitpod-uesaddev73c", LeaveMeAlone: "foo"},
+			Expectation: Expectation{
+				Result: struct {
+					Username     string
+					Email        string
+					Password     string
+					WorkspaceID  string
+					LeaveMeAlone string
+				}{Username: "[redacted:md5:acbd18db4cc2f85cedef654fccc4a4d8]", Email: "[redacted]", Password: "[redacted]", WorkspaceID: "[redacted:md5:a35538939333def8477b5c19ac694b35]", LeaveMeAlone: "foo"},
+			},
+		},
+		{
+			Name: "map field",
+			Struct: &struct {
+				WithMap map[string]interface{}
+			}{
+				WithMap: map[string]interface{}{
+					"email": "[email protected]",
+				},
+			},
+			Expectation: Expectation{
+				Result: &struct{ WithMap map[string]any }{WithMap: map[string]any{"email": string("[redacted]")}},
+			},
+		},
+		{
+			Name: "slices",
+			Struct: &struct {
+				Slice []string
+			}{Slice: []string{"foo", "bar", "[email protected]"}},
+			Expectation: Expectation{
+				Result: &struct {
+					Slice []string
+				}{Slice: []string{"foo", "bar", "[redacted:email]"}},
+			},
+		},
+		{
+			Name: "struct tags",
+			Struct: &struct {
+				Hashed   string `scrub:"hash"`
+				Redacted string `scrub:"redact"`
+				Email    string `scrub:"ignore"`
+			}{
+				Hashed:   "foo",
+				Redacted: "foo",
+				Email:    "foo",
+			},
+			Expectation: Expectation{
+				Result: &struct {
+					Hashed   string `scrub:"hash"`
+					Redacted string `scrub:"redact"`
+					Email    string `scrub:"ignore"`
+				}{
+					Hashed:   "[redacted:md5:acbd18db4cc2f85cedef654fccc4a4d8]",
+					Redacted: "[redacted]",
+					Email:    "foo",
+				},
+			},
+		},
+		{
+			Name: "trusted struct",
+			Struct: scrubStructToTest(&StructToTest{
+				Username: "foo",
+				Email:    "[email protected]",
+				Password: "foobar",
+			}),
+			Expectation: Expectation{
+				Result: &TrustedStructToTest{
+					StructToTest: StructToTest{
+						Username: "foo",
+						Email:    "trusted:[redacted:email]",
+						Password: "trusted:[redacted]",
+					},
+				},
+			},
+		},
+		{
+			Name: "trusted interface",
+			Struct: scrubStructToTestAsTrustedValue(&StructToTest{
+				Username: "foo",
+				Email:    "[email protected]",
+				Password: "foobar",
+			}),
+			Expectation: Expectation{
+				Result: &TrustedStructToTest{
+					StructToTest: StructToTest{
+						Username: "foo",
+						Email:    "trusted:[redacted:email]",
+						Password: "trusted:[redacted]",
+					},
+				},
+			},
+		},
+		{
+			Name: "contains unexported pointers",
+			Struct: UnexportedStructToTest{
+				Exported:      "foo",
+				unexportedPtr: nil,
+			},
+			Expectation: Expectation{
+				Result: UnexportedStructToTest{
+					Exported:      "foo",
+					unexportedPtr: nil,
+				},
+			},
+			CmpOpts: []cmp.Option{cmpopts.IgnoreUnexported(UnexportedStructToTest{})},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			var act Expectation
+			b, _ := json.Marshal(test.Struct)
+
+			act.Result = Default.DeepCopyStruct(test.Struct)
+			b2, _ := json.Marshal(test.Struct)
+
+			if diff := cmp.Diff(b, b2, test.CmpOpts...); diff != "" {
+				t.Errorf("DeepCopyStruct for origin struct modified (-want +got):\n%s", diff)
+			}
+
+			if diff := cmp.Diff(test.Expectation, act, test.CmpOpts...); diff != "" {
+				t.Errorf("DeepCopyStruct() mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
 func BenchmarkKeyValue(b *testing.B) {
 	key := HashedFieldNames[rand.Intn(len(HashedFieldNames))]
 

components/ws-manager-api/go/crd/v1/workspace_types.go

@@ -169,7 +169,7 @@ func (ps PortSpec) Equal(other PortSpec) bool {
 // WorkspaceStatus defines the observed state of Workspace
 type WorkspaceStatus struct {
 	PodStarts  int    `json:"podStarts"`
-	URL        string `json:"url,omitempty"`
+	URL        string `json:"url,omitempty" scrub:"redact"`
 	OwnerToken string `json:"ownerToken,omitempty" scrub:"redact"`
 
 	// +kubebuilder:default=Unknown