Add new feature for custom workspace network CIDR

components/ws-daemon/nsinsider/main.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/netip"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -268,22 +269,25 @@ func main() {
 						Name:     "target-pid",
 						Required: true,
 					},
+					&cli.StringFlag{
+						Name:     "workspace-cidr",
+						Required: true,
+					},
 				},
 				Action: func(c *cli.Context) error {
 					containerIf, vethIf, cethIf := "eth0", "veth0", "eth0"
-					mask := net.IPv4Mask(255, 255, 255, 0)
-					vethIp := net.IPNet{
-						IP:   net.IPv4(10, 0, 5, 1),
-						Mask: mask,
-					}
-					cethIp := net.IPNet{
-						IP:   net.IPv4(10, 0, 5, 2),
-						Mask: mask,
+					networkCIDR := c.String("workspace-cidr")
+
+					vethIp, cethIp, mask, err := processWorkspaceCIDR(networkCIDR)
+					if err != nil {
+						return xerrors.Errorf("parsing workspace CIDR (%v):%v", networkCIDR, err)
 					}
-					masqueradeAddr := net.IPNet{
-						IP:   vethIp.IP.Mask(mask),
-						Mask: mask,
+
+					vethIpNet := net.IPNet{
+						IP:   vethIp,
+						Mask: mask.Mask,
 					}
+
 					targetPid := c.Int("target-pid")
 
 					eth0, err := netlink.LinkByName(containerIf)
@@ -308,7 +312,7 @@ func main() {
 					if err != nil {
 						return xerrors.Errorf("cannot found %q netns failed: %v", vethIf, err)
 					}
-					if err := netlink.AddrAdd(vethLink, &netlink.Addr{IPNet: &vethIp}); err != nil {
+					if err := netlink.AddrAdd(vethLink, &netlink.Addr{IPNet: &vethIpNet}); err != nil {
 						return xerrors.Errorf("failed to add IP address to %q: %v", vethIf, err)
 					}
 					if err := netlink.LinkSetUp(vethLink); err != nil {
@@ -329,29 +333,10 @@ func main() {
 						Type:     nftables.ChainTypeNAT,
 					})
 
-					// ip saddr 10.0.5.0/24 oifname "eth0" masquerade
 					nc.AddRule(&nftables.Rule{
 						Table: nat,
 						Chain: postrouting,
 						Exprs: []expr.Any{
-							&expr.Payload{
-								DestRegister: 1,
-								Base:         expr.PayloadBaseNetworkHeader,
-								Offset:       12,
-								Len:          net.IPv4len,
-							},
-							&expr.Bitwise{
-								SourceRegister: 1,
-								DestRegister:   1,
-								Len:            net.IPv4len,
-								Mask:           masqueradeAddr.Mask,
-								Xor:            net.IPv4Mask(0, 0, 0, 0),
-							},
-							&expr.Cmp{
-								Op:       expr.CmpOpEq,
-								Register: 1,
-								Data:     masqueradeAddr.IP.To4(),
-							},
 							&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
 							&expr.Cmp{
 								Op:       expr.CmpOpEq,
@@ -408,7 +393,7 @@ func main() {
 
 							&expr.Immediate{
 								Register: 2,
-								Data:     cethIp.IP.To4(),
+								Data:     cethIp.To4(),
 							},
 							&expr.NAT{
 								Type:        expr.NATTypeDestNAT,
@@ -418,7 +403,6 @@ func main() {
 							},
 						},
 					})
-
 					if err := nc.Flush(); err != nil {
 						return xerrors.Errorf("failed to apply nftables: %v", err)
 					}
@@ -429,23 +413,31 @@ func main() {
 			{
 				Name:  "setup-peer-veth",
 				Usage: "set up a peer veth",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "workspace-cidr",
+						Required: true,
+					},
+				},
 				Action: func(c *cli.Context) error {
 					cethIf := "eth0"
-					mask := net.IPv4Mask(255, 255, 255, 0)
-					cethIp := net.IPNet{
-						IP:   net.IPv4(10, 0, 5, 2),
-						Mask: mask,
+
+					networkCIDR := c.String("workspace-cidr")
+					vethIp, cethIp, mask, err := processWorkspaceCIDR(networkCIDR)
+					if err != nil {
+						return xerrors.Errorf("parsing workspace CIDR (%v):%v", networkCIDR, err)
 					}
-					vethIp := net.IPNet{
-						IP:   net.IPv4(10, 0, 5, 1),
-						Mask: mask,
+
+					cethIpNet := net.IPNet{
+						IP:   cethIp,
+						Mask: mask.Mask,
 					}
 
 					cethLink, err := netlink.LinkByName(cethIf)
 					if err != nil {
 						return xerrors.Errorf("cannot found %q netns failed: %v", cethIf, err)
 					}
-					if err := netlink.AddrAdd(cethLink, &netlink.Addr{IPNet: &cethIp}); err != nil {
+					if err := netlink.AddrAdd(cethLink, &netlink.Addr{IPNet: &cethIpNet}); err != nil {
 						return xerrors.Errorf("failed to add IP address to %q: %v", cethIf, err)
 					}
 					if err := netlink.LinkSetUp(cethLink); err != nil {
@@ -462,7 +454,7 @@ func main() {
 
 					defaultGw := netlink.Route{
 						Scope: netlink.SCOPE_UNIVERSE,
-						Gw:    vethIp.IP,
+						Gw:    vethIp,
 					}
 					if err := netlink.RouteReplace(&defaultGw); err != nil {
 						return xerrors.Errorf("failed to set up deafult gw: %v", err)
@@ -687,3 +679,27 @@ const (
 	// FlagAtRecursive: Apply to the entire subtree: https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/fcntl.h#L112
 	flagAtRecursive = 0x8000
 )
+
+func processWorkspaceCIDR(networkCIDR string) (net.IP, net.IP, *net.IPNet, error) {
+	netIP, mask, err := net.ParseCIDR(networkCIDR)
+	if err != nil {
+		return nil, nil, nil, xerrors.Errorf("cannot configure workspace CIDR: %w", err)
+	}
+
+	addr, err := netip.ParseAddr(netIP.String())
+	if err != nil {
+		return nil, nil, nil, xerrors.Errorf("cannot configure workspace CIDR: %w", err)
+	}
+
+	vethIp := addr.Next()
+	if !vethIp.IsValid() {
+		return nil, nil, nil, xerrors.Errorf("workspace CIDR is not big enough (%v)", networkCIDR)
+	}
+
+	cethIp := vethIp.Next()
+	if !cethIp.IsValid() {
+		return nil, nil, nil, xerrors.Errorf("workspace CIDR is not big enough (%v)", networkCIDR)
+	}
+
+	return net.ParseIP(vethIp.String()), net.ParseIP(cethIp.String()), mask, nil
+}

components/ws-daemon/pkg/content/hooks.go

@@ -23,10 +23,10 @@ import (
 )
 
 // WorkspaceLifecycleHooks configures the lifecycle hooks for all workspaces
-func WorkspaceLifecycleHooks(cfg Config, workspaceExistenceCheck WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, xfs *quota.XFS, cgroupMountPoint string) map[session.WorkspaceState][]session.WorkspaceLivecycleHook {
+func WorkspaceLifecycleHooks(cfg Config, workspaceCIDR string, workspaceExistenceCheck WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, xfs *quota.XFS, cgroupMountPoint string) map[session.WorkspaceState][]session.WorkspaceLivecycleHook {
 	// startIWS starts the in-workspace service for a workspace. This lifecycle hook is idempotent, hence can - and must -
 	// be called on initialization and ready. The on-ready hook exists only to support ws-daemon restarts.
-	startIWS := iws.ServeWorkspace(uidmapper, api.FSShiftMethod(cfg.UserNamespaces.FSShift), cgroupMountPoint)
+	startIWS := iws.ServeWorkspace(uidmapper, api.FSShiftMethod(cfg.UserNamespaces.FSShift), cgroupMountPoint, workspaceCIDR)
 
 	return map[session.WorkspaceState][]session.WorkspaceLivecycleHook{
 		session.WorkspaceInitializing: {

components/ws-daemon/pkg/content/service.go

@@ -69,7 +69,7 @@ type WorkspaceService struct {
 type WorkspaceExistenceCheck func(instanceID string) bool
 
 // NewWorkspaceService creates a new workspce initialization service, starts housekeeping and the Prometheus integration
-func NewWorkspaceService(ctx context.Context, cfg Config, runtime container.Runtime, wec WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, cgroupMountPoint string, reg prometheus.Registerer) (res *WorkspaceService, err error) {
+func NewWorkspaceService(ctx context.Context, cfg Config, runtime container.Runtime, wec WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, cgroupMountPoint string, reg prometheus.Registerer, workspaceCIDR string) (res *WorkspaceService, err error) {
 	//nolint:ineffassign
 	span, ctx := opentracing.StartSpanFromContext(ctx, "NewWorkspaceService")
 	defer tracing.FinishSpan(span, &err)
@@ -86,7 +86,9 @@ func NewWorkspaceService(ctx context.Context, cfg Config, runtime container.Runt
 	}
 
 	// read all session json files
-	store, err := session.NewStore(ctx, cfg.WorkingArea, WorkspaceLifecycleHooks(cfg, wec, uidmapper, xfs, cgroupMountPoint))
+	store, err := session.NewStore(ctx, cfg.WorkingArea,
+		WorkspaceLifecycleHooks(cfg, workspaceCIDR, wec, uidmapper, xfs, cgroupMountPoint),
+	)
 	if err != nil {
 		return nil, xerrors.Errorf("cannot create session store: %w", err)
 	}

components/ws-daemon/pkg/daemon/config.go

@@ -43,6 +43,8 @@ type RuntimeConfig struct {
 	Kubeconfig          string            `json:"kubeconfig"`
 	KubernetesNamespace string            `json:"namespace"`
 	SecretsNamespace    string            `json:"secretsNamespace"`
+
+	WorkspaceCIDR string `json:"workspaceCIDR,omitempty"`
 }
 
 type IOLimitConfig struct {

components/ws-daemon/pkg/daemon/daemon.go

@@ -194,6 +194,7 @@ func NewDaemon(config Config) (*Daemon, error) {
 
 		hooks := content.WorkspaceLifecycleHooks(
 			contentCfg,
+			config.Runtime.WorkspaceCIDR,
 			func(instanceID string) bool { return true },
 			&iws.Uidmapper{Config: config.Uidmapper, Runtime: containerRuntime},
 			xfs,
@@ -239,6 +240,7 @@ func NewDaemon(config Config) (*Daemon, error) {
 		&iws.Uidmapper{Config: config.Uidmapper, Runtime: containerRuntime},
 		config.CPULimit.CGroupBasePath,
 		wrappedReg,
+		config.Runtime.WorkspaceCIDR,
 	)
 	if err != nil {
 		return nil, xerrors.Errorf("cannot create content service: %w", err)

components/ws-daemon/pkg/iws/iws.go

@@ -84,7 +84,7 @@ var (
 )
 
 // ServeWorkspace establishes the IWS server for a workspace
-func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMountPoint string) func(ctx context.Context, ws *session.Workspace) error {
+func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMountPoint string, workspaceCIDR string) func(ctx context.Context, ws *session.Workspace) error {
 	return func(ctx context.Context, ws *session.Workspace) (err error) {
 		span, _ := opentracing.StartSpanFromContext(ctx, "iws.ServeWorkspace")
 		defer tracing.FinishSpan(span, &err)
@@ -98,6 +98,7 @@ func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMount
 			Session:          ws,
 			FSShift:          fsshift,
 			CGroupMountPoint: cgroupMountPoint,
+			WorkspaceCIDR:    workspaceCIDR,
 		}
 		err = iws.Start()
 		if err != nil {
@@ -139,6 +140,8 @@ type InWorkspaceServiceServer struct {
 	FSShift          api.FSShiftMethod
 	CGroupMountPoint string
 
+	WorkspaceCIDR string
+
 	srv  *grpc.Server
 	sckt io.Closer
 
@@ -361,7 +364,10 @@ func (wbs *InWorkspaceServiceServer) SetupPairVeths(ctx context.Context, req *ap
 	}
 
 	err = nsi.Nsinsider(wbs.Session.InstanceID, int(containerPID), func(c *exec.Cmd) {
-		c.Args = append(c.Args, "setup-pair-veths", "--target-pid", strconv.Itoa(int(req.Pid)))
+		c.Args = append(c.Args, "setup-pair-veths",
+			"--target-pid", strconv.Itoa(int(req.Pid)),
+			fmt.Sprintf("--workspace-cidr=%v", wbs.WorkspaceCIDR),
+		)
 	}, nsi.EnterMountNS(true), nsi.EnterPidNS(true), nsi.EnterNetNS(true))
 	if err != nil {
 		log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a pair of veths")
@@ -373,7 +379,9 @@ func (wbs *InWorkspaceServiceServer) SetupPairVeths(ctx context.Context, req *ap
 		return nil, xerrors.Errorf("cannot map in-container PID %d (container PID: %d): %w", req.Pid, containerPID, err)
 	}
 	err = nsi.Nsinsider(wbs.Session.InstanceID, int(pid), func(c *exec.Cmd) {
-		c.Args = append(c.Args, "setup-peer-veth")
+		c.Args = append(c.Args, "setup-peer-veth",
+			fmt.Sprintf("--workspace-cidr=%v", wbs.WorkspaceCIDR),
+		)
 	}, nsi.EnterMountNS(true), nsi.EnterPidNS(true), nsi.EnterNetNS(true))
 	if err != nil {
 		log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a peer veths")

install/installer/pkg/cluster/checks.go

@@ -7,6 +7,8 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/netip"
 	"strings"
 
 	"github.com/Masterminds/semver"
@@ -291,3 +293,64 @@ func checkNamespaceExists(ctx context.Context, config *rest.Config, namespace st
 
 	return nil, nil
 }
+
+func CheckWorkspaceCIDR(networkCIDR string) ValidationCheck {
+	return ValidationCheck{
+		Name:        "workspace CIDR is present and valid",
+		Description: "ensures the workspace CIDR contains a valid network address range",
+		Check: func(ctx context.Context, config *rest.Config, namespace string) ([]ValidationError, error) {
+			netIP, ipNet, err := net.ParseCIDR(networkCIDR)
+			if err != nil {
+				return []ValidationError{
+					{
+						Message: fmt.Sprintf("invalid workspace CIDR: %v", err),
+						Type:    ValidationStatusError,
+					},
+				}, nil
+			}
+
+			ipNet.Mask.Size()
+			mask, _ := ipNet.Mask.Size()
+			if mask > 30 {
+				return []ValidationError{
+					{
+						Message: "the workspace CIDR does not have a mask less than or equal to /30",
+						Type:    ValidationStatusError,
+					},
+				}, nil
+			}
+
+			addr, err := netip.ParseAddr(netIP.String())
+			if err != nil {
+				return []ValidationError{
+					{
+						Message: fmt.Sprintf("invalid workspace CIDR: %v", err),
+						Type:    ValidationStatusError,
+					},
+				}, nil
+			}
+
+			vethIp := addr.Next()
+			if !vethIp.IsValid() {
+				return []ValidationError{
+					{
+						Message: fmt.Sprintf("workspace CIDR is not big enough (%v)", networkCIDR),
+						Type:    ValidationStatusError,
+					},
+				}, nil
+			}
+
+			cethIp := vethIp.Next()
+			if !cethIp.IsValid() {
+				return []ValidationError{
+					{
+						Message: fmt.Sprintf("workspace CIDR is not big enough (%v)", networkCIDR),
+						Type:    ValidationStatusError,
+					},
+				}, nil
+			}
+
+			return nil, nil
+		},
+	}
+}

install/installer/pkg/components/ws-daemon/configmap.go

@@ -67,6 +67,9 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 
 	var wscontroller daemon.WorkspaceControllerConfig
 
+	// default workspace network CIDR (and fallback)
+	workspaceCIDR := "10.0.5.0/30"
+
 	ctx.WithExperimental(func(ucfg *experimental.Config) error {
 		if ucfg.Workspace == nil {
 			return nil
@@ -105,6 +108,10 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 		wscontroller.WorkingAreaSuffix = "-mk2"
 		wscontroller.MaxConcurrentReconciles = 15
 
+		if ucfg.Workspace.WorkspaceCIDR != "" {
+			workspaceCIDR = ucfg.Workspace.WorkspaceCIDR
+		}
+
 		return nil
 	})
 
@@ -123,6 +130,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 						SocketPath: "/mnt/containerd/containerd.sock",
 					},
 				},
+				WorkspaceCIDR: workspaceCIDR,
 			},
 			Content: content.Config{
 				WorkingArea:     "/mnt/workingarea",

install/installer/pkg/config/v1/experimental/experimental.go

@@ -68,6 +68,8 @@ type WorkspaceConfig struct {
 	WorkspaceURLTemplate     string   `json:"workspaceURLTemplate,omitempty"`
 	WorkspacePortURLTemplate string   `json:"workspacePortURLTemplate,omitempty"`
 
+	WorkspaceCIDR string `json:"workspaceCIDR,omitempty"`
+
 	CPULimits struct {
 		Enabled          bool              `json:"enabled"`
 		NodeCPUBandwidth resource.Quantity `json:"nodeBandwidth"`