Watch UpstreamSettingsPolicies and translate into datplane config

Author: Kate Osborn

charts/nginx-gateway-fabric/templates/clusterrole.yaml

@@ -104,6 +104,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   {{- if .Values.nginxGateway.snippetsFilters.enable }}
   - snippetsfilters
   {{- end }}
@@ -116,6 +117,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   {{- if .Values.nginxGateway.snippetsFilters.enable }}
   - snippetsfilters/status
   {{- end }}

config/crd/kustomization.yaml

@@ -5,3 +5,5 @@ resources:
   - bases/gateway.nginx.org_nginxgateways.yaml
   - bases/gateway.nginx.org_nginxproxies.yaml
   - bases/gateway.nginx.org_observabilitypolicies.yaml
+  - bases/gateway.nginx.org_snippetsfilters.yaml
+  - bases/gateway.nginx.org_upstreamsettingspolicies.yaml

deploy/crds.yaml

@@ -0,0 +1,37 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: coffee
+spec:
+  parentRefs:
+  - name: gateway
+    sectionName: http
+  hostnames:
+  - "cafe.example.com"
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /coffee
+    backendRefs:
+    - name: coffee
+      port: 80
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: tea
+spec:
+  parentRefs:
+  - name: gateway
+    sectionName: http
+  hostnames:
+  - "cafe.example.com"
+  rules:
+  - matches:
+    - path:
+        type: Exact
+        value: /tea
+    backendRefs:
+    - name: tea
+      port: 80

examples/upstream-settings-policy/cafe-routes.yaml

@@ -0,0 +1,65 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coffee
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: coffee
+  template:
+    metadata:
+      labels:
+        app: coffee
+    spec:
+      containers:
+      - name: coffee
+        image: nginxdemos/nginx-hello:plain-text
+        ports:
+        - containerPort: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: coffee
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: coffee
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tea
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: tea
+  template:
+    metadata:
+      labels:
+        app: tea
+    spec:
+      containers:
+      - name: tea
+        image: nginxdemos/nginx-hello:plain-text
+        ports:
+        - containerPort: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tea
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: tea

examples/upstream-settings-policy/cafe.yaml

@@ -0,0 +1,11 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: gateway
+spec:
+  gatewayClassName: nginx
+  listeners:
+  - name: http
+    port: 80
+    protocol: HTTP
+    hostname: "*.example.com"

examples/upstream-settings-policy/gateway.yaml

@@ -7,7 +7,7 @@ spec:
   targetRefs:
     - group: core
       kind: Service
-      name: service
+      name: coffee
   keepAlive:
     connections: 32
     requests: 1001

examples/upstream-settings-policy/upstream-settings-policy.yaml

@@ -11,9 +11,9 @@ import (
 
 // Gateway API Kinds.
 const (
-	// Gateway is the Gateway Kind.
+	// Gateway is the Gateway kind.
 	Gateway = "Gateway"
-	// GatewayClass is the GatewayClass Kind.
+	// GatewayClass is the GatewayClass kind.
 	GatewayClass = "GatewayClass"
 	// HTTPRoute is the HTTPRoute kind.
 	HTTPRoute = "HTTPRoute"
@@ -23,6 +23,12 @@ const (
 	TLSRoute = "TLSRoute"
 )
 
+// Core API Kinds.
+const (
+	// Service is the Service kind.
+	Service = "Service"
+)
+
 // NGINX Gateway Fabric kinds.
 const (
 	// ClientSettingsPolicy is the ClientSettingsPolicy kind.
@@ -33,6 +39,8 @@ const (
 	NginxProxy = "NginxProxy"
 	// SnippetsFilter is the SnippetsFilter kind.
 	SnippetsFilter = "SnippetsFilter"
+	// UpstreamSettingsPolicy is the UpstreamSettingsPolicy kind.
+	UpstreamSettingsPolicy = "UpstreamSettingsPolicy"
 )
 
 // MustExtractGVK is a function that extracts the GroupVersionKind (GVK) of a client.object.

internal/framework/kinds/kinds.go

@@ -52,6 +52,7 @@ import (
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/clientsettings"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/observability"
+	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/upstreamsettings"
 	ngxvalidation "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/validation"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/file"
 	ngxruntime "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/runtime"
@@ -311,6 +312,10 @@ func createPolicyManager(
 			GVK:       mustExtractGVK(&ngfAPI.ObservabilityPolicy{}),
 			Validator: observability.NewValidator(validator),
 		},
+		{
+			GVK:       mustExtractGVK(&ngfAPI.UpstreamSettingsPolicy{}),
+			Validator: upstreamsettings.NewValidator(validator),
+		},
 	}
 
 	return policies.NewManager(mustExtractGVK, cfgs...)
@@ -492,6 +497,12 @@ func registerControllers(
 				controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
 			},
 		},
+		{
+			objectType: &ngfAPI.UpstreamSettingsPolicy{},
+			options: []controller.Option{
+				controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
+			},
+		},
 	}
 
 	if cfg.ExperimentalFeatures {
@@ -728,6 +739,7 @@ func prepareFirstEventBatchPreparerArgs(cfg config.Config) ([]client.Object, []c
 		&gatewayv1.GRPCRouteList{},
 		&ngfAPI.ClientSettingsPolicyList{},
 		&ngfAPI.ObservabilityPolicyList{},
+		&ngfAPI.UpstreamSettingsPolicyList{},
 		partialObjectMetadataList,
 	}
 

internal/mode/static/manager.go

@@ -67,6 +67,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				partialObjectMetadataList,
 				&ngfAPI.ClientSettingsPolicyList{},
 				&ngfAPI.ObservabilityPolicyList{},
+				&ngfAPI.UpstreamSettingsPolicyList{},
 			},
 		},
 		{
@@ -96,6 +97,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				partialObjectMetadataList,
 				&ngfAPI.ClientSettingsPolicyList{},
 				&ngfAPI.ObservabilityPolicyList{},
+				&ngfAPI.UpstreamSettingsPolicyList{},
 			},
 		},
 		{
@@ -128,6 +130,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				&gatewayv1.GRPCRouteList{},
 				&ngfAPI.ClientSettingsPolicyList{},
 				&ngfAPI.ObservabilityPolicyList{},
+				&ngfAPI.UpstreamSettingsPolicyList{},
 			},
 		},
 		{
@@ -158,6 +161,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				&ngfAPI.ClientSettingsPolicyList{},
 				&ngfAPI.ObservabilityPolicyList{},
 				&ngfAPI.SnippetsFilterList{},
+				&ngfAPI.UpstreamSettingsPolicyList{},
 			},
 		},
 		{
@@ -191,6 +195,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				&ngfAPI.ClientSettingsPolicyList{},
 				&ngfAPI.ObservabilityPolicyList{},
 				&ngfAPI.SnippetsFilterList{},
+				&ngfAPI.UpstreamSettingsPolicyList{},
 			},
 		},
 	}

internal/mode/static/manager_test.go

@@ -30,7 +30,9 @@ func (v *Validator) Validate(policy policies.Policy, _ *policies.GlobalSettings)
 
 	targetRefPath := field.NewPath("spec").Child("targetRef")
 	supportedKinds := []gatewayv1.Kind{kinds.Gateway, kinds.HTTPRoute, kinds.GRPCRoute}
-	if err := policies.ValidateTargetRef(csp.Spec.TargetRef, targetRefPath, supportedKinds); err != nil {
+	supportedGroups := []gatewayv1.Group{gatewayv1.GroupName}
+
+	if err := policies.ValidateTargetRef(csp.Spec.TargetRef, targetRefPath, supportedGroups, supportedKinds); err != nil {
 		return []conditions.Condition{staticConds.NewPolicyInvalid(err.Error())}
 	}
 

internal/mode/static/nginx/config/policies/clientsettings/validator.go

@@ -45,8 +45,10 @@ func (v *Validator) Validate(
 
 	targetRefPath := field.NewPath("spec").Child("targetRefs")
 	supportedKinds := []gatewayv1.Kind{kinds.HTTPRoute, kinds.GRPCRoute}
+	supportedGroups := []gatewayv1.Group{gatewayv1.GroupName}
+
 	for _, ref := range obs.Spec.TargetRefs {
-		if err := policies.ValidateTargetRef(ref, targetRefPath, supportedKinds); err != nil {
+		if err := policies.ValidateTargetRef(ref, targetRefPath, supportedGroups, supportedKinds); err != nil {
 			return []conditions.Condition{staticConds.NewPolicyInvalid(err.Error())}
 		}
 	}

internal/mode/static/nginx/config/policies/observability/validator.go

@@ -24,25 +24,26 @@ type Policy interface {
 // GlobalSettings contains global settings from the current state of the graph that may be
 // needed for policy validation or generation if certain policies rely on those global settings.
 type GlobalSettings struct {
-	// NginxProxyValid is whether or not the NginxProxy resource is valid.
+	// NginxProxyValid is whether the NginxProxy resource is valid.
 	NginxProxyValid bool
-	// TelemetryEnabled is whether or not telemetry is enabled in the NginxProxy resource.
+	// TelemetryEnabled is whether telemetry is enabled in the NginxProxy resource.
 	TelemetryEnabled bool
 }
 
 // ValidateTargetRef validates a policy's targetRef for the proper group and kind.
 func ValidateTargetRef(
 	ref v1alpha2.LocalPolicyTargetReference,
 	basePath *field.Path,
+	groups []gatewayv1.Group,
 	supportedKinds []gatewayv1.Kind,
 ) error {
-	if ref.Group != gatewayv1.GroupName {
+	if !slices.Contains(groups, ref.Group) {
 		path := basePath.Child("group")
 
 		return field.NotSupported(
 			path,
 			ref.Group,
-			[]string{gatewayv1.GroupName},
+			groups,
 		)
 	}