CP/DP Split: write configuration to agent (#2999)

This commit adds functionality to send nginx configuration to the agent. It also adds support for the single nginx Deployment to be scaled, and send configuration to all replicas. This requires tracking all Subscriptions for a particular deployment, and receiving all responses from those replicas to determine the status to write to the Gateway.

Author: sjberman

build/Dockerfile

@@ -14,7 +14,7 @@ FROM golang:1.23 AS ca-certs-provider
 FROM scratch AS common
 # CA certs are needed for telemetry report so that NGF can verify the server's certificate.
 COPY --from=ca-certs-provider --link /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-USER 102:1001
+USER 101:1001
 ARG BUILD_AGENT
 ENV BUILD_AGENT=${BUILD_AGENT}
 ENTRYPOINT [ "/usr/bin/gateway" ]

charts/nginx-gateway-fabric/README.md

@@ -268,7 +268,6 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.image.tag` |  | string | `"edge"` |
 | `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` |
 | `nginx.plus` | Is NGINX Plus image being used | bool | `false` |
-| `nginx.securityContext.allowPrivilegeEscalation` | Some environments may need this set to true in order for the control plane to successfully reload NGINX. | bool | `false` |
 | `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` |

charts/nginx-gateway-fabric/templates/clusterrole.yaml

@@ -11,35 +11,21 @@ rules:
   - namespaces
   - services
   - secrets
+  - pods
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - configmaps
 {{- end }}
   verbs:
   - get
   - list
   - watch
-{{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }}
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
 - apiGroups:
   - apps
   resources:
   - replicasets
   verbs:
   - get
-{{- end }}
-{{- if .Values.nginx.plus }}
-- apiGroups:
-  - apps
-  resources:
-  - replicasets
-  verbs:
   - list
-{{- end }}
 {{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }}
 - apiGroups:
   - ""

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -139,8 +139,9 @@ spec:
           capabilities:
             drop:
             - ALL
+          allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
-          runAsUser: 102
+          runAsUser: 101
           runAsGroup: 1001
         {{- with .Values.nginxGateway.extraVolumeMounts -}}
         {{ toYaml . | nindent 8 }}

charts/nginx-gateway-fabric/templates/scc.yaml

@@ -1,9 +1,10 @@
+# TODO(sberman): will need an SCC for nginx ServiceAccounts as well.
 {{- if .Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints" }}
 kind: SecurityContextConstraints
 apiVersion: security.openshift.io/v1
 metadata:
   name: {{ include "nginx-gateway.scc-name" . }}
-allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }}
+allowPrivilegeEscalation: false
 allowHostDirVolumePlugin: false
 allowHostIPC: false
 allowHostNetwork: false
@@ -14,7 +15,7 @@ readOnlyRootFilesystem: true
 runAsUser:
   type: MustRunAsRange
   uidRangeMin: 101
-  uidRangeMax: 102
+  uidRangeMax: 101
 fsGroup:
   type: MustRunAs
   ranges:
@@ -29,16 +30,8 @@ seLinuxContext:
   type: MustRunAs
 seccompProfiles:
 - runtime/default
-volumes:
-- emptyDir
-- secret
-- configMap
-- projected
 users:
 - {{ printf "system:serviceaccount:%s:%s" .Release.Namespace (include "nginx-gateway.serviceAccountName" .) }}
-allowedCapabilities:
-- NET_BIND_SERVICE
-- KILL
 requiredDropCapabilities:
 - ALL
 {{- end }}

charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml

@@ -15,5 +15,29 @@ data:
     - /var/run/nginx
     features:
     - connection
+    - configuration
+    - certificates
+    - metrics
+    {{- if .Values.nginx.plus }}
+    - api-action
+    {{- end }}
     log:
       level: debug
+    collector:
+      receivers:
+        host_metrics:
+          collection_interval: 1m0s
+          initial_delay: 1s
+          scrapers:
+            cpu: {}
+            memory: {}
+            disk: {}
+            network: {}
+            filesystem: {}
+      processors:
+        batch: {}
+      exporters:
+        prometheus_exporter:
+          server:
+            host: "0.0.0.0"
+            port: 9113

charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml

@@ -13,30 +13,37 @@ spec:
       labels:
         app.kubernetes.io/name: tmp-nginx-deployment
         app.kubernetes.io/instance: {{ .Release.Name }}
+      annotations:
+        {{- if .Values.metrics.enable }}
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ .Values.metrics.port }}"
+        {{- if .Values.metrics.secure }}
+        prometheus.io/scheme: "https"
+        {{- end }}
+        {{- end }}
     spec:
       initContainers:
-      - name: sleep # wait for a bit for control plane to be ready
-        image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }}
-        imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }}
-        command:
-        - /usr/bin/gateway
-        - sleep
-        - --duration=15s
       - name: init
         image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }}
         imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }}
         command:
         - /usr/bin/gateway
         - initialize
         - --source
+        - /agent/nginx-agent.conf
+        - --destination
+        - /etc/nginx-agent
+        - --source
         - /includes/main.conf
+        - --destination
+        - /etc/nginx/main-includes
         {{- if .Values.nginx.plus }}
         - --source
         - /includes/mgmt.conf
         - --nginx-plus
-        {{- end }}
         - --destination
         - /etc/nginx/main-includes
+        {{- end }}
         env:
           - name: POD_UID
             valueFrom:
@@ -49,9 +56,13 @@ spec:
             drop:
             - ALL
           readOnlyRootFilesystem: true
-          runAsUser: 102
+          runAsUser: 101
           runAsGroup: 1001
         volumeMounts:
+        - name: nginx-agent-config
+          mountPath: /agent
+        - name: nginx-agent
+          mountPath: /etc/nginx-agent
         - name: nginx-includes-bootstrap
           mountPath: /includes
         - name: nginx-main-includes
@@ -69,10 +80,11 @@ spec:
           name: http
         - containerPort: 443
           name: https
+        - name: metrics
+          containerPort: {{ .Values.metrics.port }}
         securityContext:
           seccompProfile:
             type: RuntimeDefault
-          allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }}
           capabilities:
             add:
             - NET_BIND_SERVICE
@@ -84,6 +96,8 @@ spec:
         volumeMounts:
         - name: nginx-agent
           mountPath: /etc/nginx-agent
+        - name: nginx-agent-log
+          mountPath: /var/log/nginx-agent
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
         - name: nginx-stream-conf
@@ -140,8 +154,12 @@ spec:
       {{- end }}
       volumes:
       - name: nginx-agent
+        emptyDir: {}
+      - name: nginx-agent-config
         configMap:
           name: nginx-agent-config
+      - name: nginx-agent-log
+        emptyDir: {}
       - name: nginx-conf
         emptyDir: {}
       - name: nginx-stream-conf

charts/nginx-gateway-fabric/values.schema.json

@@ -270,20 +270,6 @@
           "title": "plus",
           "type": "boolean"
         },
-        "securityContext": {
-          "properties": {
-            "allowPrivilegeEscalation": {
-              "default": false,
-              "description": "Some environments may need this set to true in order for the control plane to successfully reload NGINX.",
-              "required": [],
-              "title": "allowPrivilegeEscalation",
-              "type": "boolean"
-            }
-          },
-          "required": [],
-          "title": "securityContext",
-          "type": "object"
-        },
         "usage": {
           "description": "Configuration for NGINX Plus usage reporting.",
           "properties": {

charts/nginx-gateway-fabric/values.yaml

@@ -131,10 +131,6 @@ nginx:
     # @schema
     pullPolicy: Always
 
-  securityContext:
-    # -- Some environments may need this set to true in order for the control plane to successfully reload NGINX.
-    allowPrivilegeEscalation: false
-
   # -- Is NGINX Plus image being used
   plus: false
 

cmd/gateway/commands.go

@@ -519,14 +519,14 @@ func createInitializeCommand() *cobra.Command {
 
 	// flag values
 	var srcFiles []string
-	var dest string
+	var destDirs []string
 	var plus bool
 
 	cmd := &cobra.Command{
 		Use:   "initialize",
 		Short: "Write initial configuration files",
 		RunE: func(_ *cobra.Command, _ []string) error {
-			if err := validateCopyArgs(srcFiles, dest); err != nil {
+			if err := validateCopyArgs(srcFiles, destDirs); err != nil {
 				return err
 			}
 
@@ -546,7 +546,7 @@ func createInitializeCommand() *cobra.Command {
 			logger.Info(
 				"Starting init container",
 				"source filenames to copy", srcFiles,
-				"destination directory", dest,
+				"destination directories", destDirs,
 				"nginx-plus",
 				plus,
 			)
@@ -558,16 +558,21 @@ func createInitializeCommand() *cobra.Command {
 				Logger:          logger.WithName("deployCtxCollector"),
 			})
 
+			files := make([]fileToCopy, 0, len(srcFiles))
+			for i, src := range srcFiles {
+				files = append(files, fileToCopy{
+					destDirName: destDirs[i],
+					srcFileName: src,
+				})
+			}
+
 			return initialize(initializeConfig{
 				fileManager:   file.NewStdLibOSFileManager(),
 				fileGenerator: ngxConfig.NewGeneratorImpl(plus, nil, logger.WithName("generator")),
 				logger:        logger,
 				plus:          plus,
 				collector:     dcc,
-				copy: copyFiles{
-					srcFileNames: srcFiles,
-					destDirName:  dest,
-				},
+				copy:          files,
 			})
 		},
 	}
@@ -579,11 +584,11 @@ func createInitializeCommand() *cobra.Command {
 		"The source files to be copied",
 	)
 
-	cmd.Flags().StringVar(
-		&dest,
+	cmd.Flags().StringSliceVar(
+		&destDirs,
 		destFlag,
-		"",
-		"The destination directory for the source files to be copied to",
+		[]string{},
+		"The destination directories for the source files at the same array index to be copied to",
 	)
 
 	cmd.Flags().BoolVar(

cmd/gateway/initialize.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"time"
 
@@ -17,23 +18,23 @@ const (
 	collectDeployCtxTimeout = 10 * time.Second
 )
 
-type copyFiles struct {
-	destDirName  string
-	srcFileNames []string
+type fileToCopy struct {
+	destDirName string
+	srcFileName string
 }
 
 type initializeConfig struct {
 	collector     licensing.Collector
 	fileManager   file.OSFileManager
 	fileGenerator config.Generator
 	logger        logr.Logger
-	copy          copyFiles
+	copy          []fileToCopy
 	plus          bool
 }
 
 func initialize(cfg initializeConfig) error {
-	for _, src := range cfg.copy.srcFileNames {
-		if err := copyFile(cfg.fileManager, src, cfg.copy.destDirName); err != nil {
+	for _, f := range cfg.copy {
+		if err := copyFile(cfg.fileManager, f.srcFileName, f.destDirName); err != nil {
 			return err
 		}
 	}
@@ -58,7 +59,7 @@ func initialize(cfg initializeConfig) error {
 		return fmt.Errorf("failed to generate deployment context file: %w", err)
 	}
 
-	if err := file.Write(cfg.fileManager, depCtxFile); err != nil {
+	if err := file.Write(cfg.fileManager, file.Convert(depCtxFile)); err != nil {
 		return fmt.Errorf("failed to write deployment context file: %w", err)
 	}
 
@@ -84,5 +85,9 @@ func copyFile(osFileManager file.OSFileManager, src, dest string) error {
 		return fmt.Errorf("error copying file contents: %w", err)
 	}
 
+	if err := osFileManager.Chmod(destFile, os.FileMode(file.RegularFileModeInt)); err != nil {
+		return fmt.Errorf("error setting file permissions: %w", err)
+	}
+
 	return nil
 }