Skip to content

Commit 9cf0243

Browse files
aledbfaledbf
authored
Add new feature for custom workspace network CIDR (#17145)
* Add new feature for custom workspace network CIDR * Cleanup
1 parent 7f953e6 commit 9cf0243

File tree

10 files changed

+156
-49
lines changed

10 files changed

+156
-49
lines changed

‎components/ws-daemon/nsinsider/main.go

Lines changed: 58 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"fmt"
99
"io/ioutil"
1010
"net"
11+
"net/netip"
1112
"os"
1213
"os/exec"
1314
"path/filepath"
@@ -268,22 +269,25 @@ func main() {
268269
Name: "target-pid",
269270
Required: true,
270271
},
272+
&cli.StringFlag{
273+
Name: "workspace-cidr",
274+
Required: true,
275+
},
271276
},
272277
Action: func(c *cli.Context) error {
273278
containerIf, vethIf, cethIf := "eth0", "veth0", "eth0"
274-
mask := net.IPv4Mask(255, 255, 255, 0)
275-
vethIp := net.IPNet{
276-
IP: net.IPv4(10, 0, 5, 1),
277-
Mask: mask,
278-
}
279-
cethIp := net.IPNet{
280-
IP: net.IPv4(10, 0, 5, 2),
281-
Mask: mask,
279+
networkCIDR := c.String("workspace-cidr")
280+
281+
vethIp, cethIp, mask, err := processWorkspaceCIDR(networkCIDR)
282+
if err != nil {
283+
return xerrors.Errorf("parsing workspace CIDR (%v):%v", networkCIDR, err)
282284
}
283-
masqueradeAddr := net.IPNet{
284-
IP: vethIp.IP.Mask(mask),
285-
Mask: mask,
285+
286+
vethIpNet := net.IPNet{
287+
IP: vethIp,
288+
Mask: mask.Mask,
286289
}
290+
287291
targetPid := c.Int("target-pid")
288292

289293
eth0, err := netlink.LinkByName(containerIf)
@@ -308,7 +312,7 @@ func main() {
308312
if err != nil {
309313
return xerrors.Errorf("cannot found %q netns failed: %v", vethIf, err)
310314
}
311-
if err := netlink.AddrAdd(vethLink, &netlink.Addr{IPNet: &vethIp}); err != nil {
315+
if err := netlink.AddrAdd(vethLink, &netlink.Addr{IPNet: &vethIpNet}); err != nil {
312316
return xerrors.Errorf("failed to add IP address to %q: %v", vethIf, err)
313317
}
314318
if err := netlink.LinkSetUp(vethLink); err != nil {
@@ -329,29 +333,10 @@ func main() {
329333
Type: nftables.ChainTypeNAT,
330334
})
331335

332-
// ip saddr 10.0.5.0/24 oifname "eth0" masquerade
333336
nc.AddRule(&nftables.Rule{
334337
Table: nat,
335338
Chain: postrouting,
336339
Exprs: []expr.Any{
337-
&expr.Payload{
338-
DestRegister: 1,
339-
Base: expr.PayloadBaseNetworkHeader,
340-
Offset: 12,
341-
Len: net.IPv4len,
342-
},
343-
&expr.Bitwise{
344-
SourceRegister: 1,
345-
DestRegister: 1,
346-
Len: net.IPv4len,
347-
Mask: masqueradeAddr.Mask,
348-
Xor: net.IPv4Mask(0, 0, 0, 0),
349-
},
350-
&expr.Cmp{
351-
Op: expr.CmpOpEq,
352-
Register: 1,
353-
Data: masqueradeAddr.IP.To4(),
354-
},
355340
&expr.Meta{Key: expr.MetaKeyOIFNAME, Register: 1},
356341
&expr.Cmp{
357342
Op: expr.CmpOpEq,
@@ -408,7 +393,7 @@ func main() {
408393

409394
&expr.Immediate{
410395
Register: 2,
411-
Data: cethIp.IP.To4(),
396+
Data: cethIp.To4(),
412397
},
413398
&expr.NAT{
414399
Type: expr.NATTypeDestNAT,
@@ -418,7 +403,6 @@ func main() {
418403
},
419404
},
420405
})
421-
422406
if err := nc.Flush(); err != nil {
423407
return xerrors.Errorf("failed to apply nftables: %v", err)
424408
}
@@ -429,23 +413,31 @@ func main() {
429413
{
430414
Name: "setup-peer-veth",
431415
Usage: "set up a peer veth",
416+
Flags: []cli.Flag{
417+
&cli.StringFlag{
418+
Name: "workspace-cidr",
419+
Required: true,
420+
},
421+
},
432422
Action: func(c *cli.Context) error {
433423
cethIf := "eth0"
434-
mask := net.IPv4Mask(255, 255, 255, 0)
435-
cethIp := net.IPNet{
436-
IP: net.IPv4(10, 0, 5, 2),
437-
Mask: mask,
424+
425+
networkCIDR := c.String("workspace-cidr")
426+
vethIp, cethIp, mask, err := processWorkspaceCIDR(networkCIDR)
427+
if err != nil {
428+
return xerrors.Errorf("parsing workspace CIDR (%v):%v", networkCIDR, err)
438429
}
439-
vethIp := net.IPNet{
440-
IP: net.IPv4(10, 0, 5, 1),
441-
Mask: mask,
430+
431+
cethIpNet := net.IPNet{
432+
IP: cethIp,
433+
Mask: mask.Mask,
442434
}
443435

444436
cethLink, err := netlink.LinkByName(cethIf)
445437
if err != nil {
446438
return xerrors.Errorf("cannot found %q netns failed: %v", cethIf, err)
447439
}
448-
if err := netlink.AddrAdd(cethLink, &netlink.Addr{IPNet: &cethIp}); err != nil {
440+
if err := netlink.AddrAdd(cethLink, &netlink.Addr{IPNet: &cethIpNet}); err != nil {
449441
return xerrors.Errorf("failed to add IP address to %q: %v", cethIf, err)
450442
}
451443
if err := netlink.LinkSetUp(cethLink); err != nil {
@@ -462,7 +454,7 @@ func main() {
462454

463455
defaultGw := netlink.Route{
464456
Scope: netlink.SCOPE_UNIVERSE,
465-
Gw: vethIp.IP,
457+
Gw: vethIp,
466458
}
467459
if err := netlink.RouteReplace(&defaultGw); err != nil {
468460
return xerrors.Errorf("failed to set up deafult gw: %v", err)
@@ -687,3 +679,27 @@ const (
687679
// FlagAtRecursive: Apply to the entire subtree: https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/fcntl.h#L112
688680
flagAtRecursive = 0x8000
689681
)
682+
683+
func processWorkspaceCIDR(networkCIDR string) (net.IP, net.IP, *net.IPNet, error) {
684+
netIP, mask, err := net.ParseCIDR(networkCIDR)
685+
if err != nil {
686+
return nil, nil, nil, xerrors.Errorf("cannot configure workspace CIDR: %w", err)
687+
}
688+
689+
addr, err := netip.ParseAddr(netIP.String())
690+
if err != nil {
691+
return nil, nil, nil, xerrors.Errorf("cannot configure workspace CIDR: %w", err)
692+
}
693+
694+
vethIp := addr.Next()
695+
if !vethIp.IsValid() {
696+
return nil, nil, nil, xerrors.Errorf("workspace CIDR is not big enough (%v)", networkCIDR)
697+
}
698+
699+
cethIp := vethIp.Next()
700+
if !cethIp.IsValid() {
701+
return nil, nil, nil, xerrors.Errorf("workspace CIDR is not big enough (%v)", networkCIDR)
702+
}
703+
704+
return net.ParseIP(vethIp.String()), net.ParseIP(cethIp.String()), mask, nil
705+
}

‎components/ws-daemon/pkg/content/hooks.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ import (
2323
)
2424

2525
// WorkspaceLifecycleHooks configures the lifecycle hooks for all workspaces
26-
func WorkspaceLifecycleHooks(cfg Config, workspaceExistenceCheck WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, xfs *quota.XFS, cgroupMountPoint string) map[session.WorkspaceState][]session.WorkspaceLivecycleHook {
26+
func WorkspaceLifecycleHooks(cfg Config, workspaceCIDR string, workspaceExistenceCheck WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, xfs *quota.XFS, cgroupMountPoint string) map[session.WorkspaceState][]session.WorkspaceLivecycleHook {
2727
// startIWS starts the in-workspace service for a workspace. This lifecycle hook is idempotent, hence can - and must -
2828
// be called on initialization and ready. The on-ready hook exists only to support ws-daemon restarts.
29-
startIWS := iws.ServeWorkspace(uidmapper, api.FSShiftMethod(cfg.UserNamespaces.FSShift), cgroupMountPoint)
29+
startIWS := iws.ServeWorkspace(uidmapper, api.FSShiftMethod(cfg.UserNamespaces.FSShift), cgroupMountPoint, workspaceCIDR)
3030

3131
return map[session.WorkspaceState][]session.WorkspaceLivecycleHook{
3232
session.WorkspaceInitializing: {

‎components/ws-daemon/pkg/content/service.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ type WorkspaceService struct {
6969
type WorkspaceExistenceCheck func(instanceID string) bool
7070

7171
// NewWorkspaceService creates a new workspce initialization service, starts housekeeping and the Prometheus integration
72-
func NewWorkspaceService(ctx context.Context, cfg Config, runtime container.Runtime, wec WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, cgroupMountPoint string, reg prometheus.Registerer) (res *WorkspaceService, err error) {
72+
func NewWorkspaceService(ctx context.Context, cfg Config, runtime container.Runtime, wec WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, cgroupMountPoint string, reg prometheus.Registerer, workspaceCIDR string) (res *WorkspaceService, err error) {
7373
//nolint:ineffassign
7474
span, ctx := opentracing.StartSpanFromContext(ctx, "NewWorkspaceService")
7575
defer tracing.FinishSpan(span, &err)
@@ -86,7 +86,9 @@ func NewWorkspaceService(ctx context.Context, cfg Config, runtime container.Runt
8686
}
8787

8888
// read all session json files
89-
store, err := session.NewStore(ctx, cfg.WorkingArea, WorkspaceLifecycleHooks(cfg, wec, uidmapper, xfs, cgroupMountPoint))
89+
store, err := session.NewStore(ctx, cfg.WorkingArea,
90+
WorkspaceLifecycleHooks(cfg, workspaceCIDR, wec, uidmapper, xfs, cgroupMountPoint),
91+
)
9092
if err != nil {
9193
return nil, xerrors.Errorf("cannot create session store: %w", err)
9294
}

‎components/ws-daemon/pkg/daemon/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ type RuntimeConfig struct {
4343
Kubeconfig string `json:"kubeconfig"`
4444
KubernetesNamespace string `json:"namespace"`
4545
SecretsNamespace string `json:"secretsNamespace"`
46+
47+
WorkspaceCIDR string `json:"workspaceCIDR,omitempty"`
4648
}
4749

4850
type IOLimitConfig struct {

‎components/ws-daemon/pkg/daemon/daemon.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ func NewDaemon(config Config) (*Daemon, error) {
194194

195195
hooks := content.WorkspaceLifecycleHooks(
196196
contentCfg,
197+
config.Runtime.WorkspaceCIDR,
197198
func(instanceID string) bool { return true },
198199
&iws.Uidmapper{Config: config.Uidmapper, Runtime: containerRuntime},
199200
xfs,
@@ -239,6 +240,7 @@ func NewDaemon(config Config) (*Daemon, error) {
239240
&iws.Uidmapper{Config: config.Uidmapper, Runtime: containerRuntime},
240241
config.CPULimit.CGroupBasePath,
241242
wrappedReg,
243+
config.Runtime.WorkspaceCIDR,
242244
)
243245
if err != nil {
244246
return nil, xerrors.Errorf("cannot create content service: %w", err)

‎components/ws-daemon/pkg/iws/iws.go

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ var (
8484
)
8585

8686
// ServeWorkspace establishes the IWS server for a workspace
87-
func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMountPoint string) func(ctx context.Context, ws *session.Workspace) error {
87+
func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMountPoint string, workspaceCIDR string) func(ctx context.Context, ws *session.Workspace) error {
8888
return func(ctx context.Context, ws *session.Workspace) (err error) {
8989
span, _ := opentracing.StartSpanFromContext(ctx, "iws.ServeWorkspace")
9090
defer tracing.FinishSpan(span, &err)
@@ -98,6 +98,7 @@ func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMount
9898
Session: ws,
9999
FSShift: fsshift,
100100
CGroupMountPoint: cgroupMountPoint,
101+
WorkspaceCIDR: workspaceCIDR,
101102
}
102103
err = iws.Start()
103104
if err != nil {
@@ -139,6 +140,8 @@ type InWorkspaceServiceServer struct {
139140
FSShift api.FSShiftMethod
140141
CGroupMountPoint string
141142

143+
WorkspaceCIDR string
144+
142145
srv *grpc.Server
143146
sckt io.Closer
144147

@@ -361,7 +364,10 @@ func (wbs *InWorkspaceServiceServer) SetupPairVeths(ctx context.Context, req *ap
361364
}
362365

363366
err = nsi.Nsinsider(wbs.Session.InstanceID, int(containerPID), func(c *exec.Cmd) {
364-
c.Args = append(c.Args, "setup-pair-veths", "--target-pid", strconv.Itoa(int(req.Pid)))
367+
c.Args = append(c.Args, "setup-pair-veths",
368+
"--target-pid", strconv.Itoa(int(req.Pid)),
369+
fmt.Sprintf("--workspace-cidr=%v", wbs.WorkspaceCIDR),
370+
)
365371
}, nsi.EnterMountNS(true), nsi.EnterPidNS(true), nsi.EnterNetNS(true))
366372
if err != nil {
367373
log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a pair of veths")
@@ -373,7 +379,9 @@ func (wbs *InWorkspaceServiceServer) SetupPairVeths(ctx context.Context, req *ap
373379
return nil, xerrors.Errorf("cannot map in-container PID %d (container PID: %d): %w", req.Pid, containerPID, err)
374380
}
375381
err = nsi.Nsinsider(wbs.Session.InstanceID, int(pid), func(c *exec.Cmd) {
376-
c.Args = append(c.Args, "setup-peer-veth")
382+
c.Args = append(c.Args, "setup-peer-veth",
383+
fmt.Sprintf("--workspace-cidr=%v", wbs.WorkspaceCIDR),
384+
)
377385
}, nsi.EnterMountNS(true), nsi.EnterPidNS(true), nsi.EnterNetNS(true))
378386
if err != nil {
379387
log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a peer veths")

‎install/installer/pkg/cluster/checks.go

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ package cluster
77
import (
88
"context"
99
"fmt"
10+
"net"
11+
"net/netip"
1012
"strings"
1113

1214
"github.com/Masterminds/semver"
@@ -291,3 +293,64 @@ func checkNamespaceExists(ctx context.Context, config *rest.Config, namespace st
291293

292294
return nil, nil
293295
}
296+
297+
func CheckWorkspaceCIDR(networkCIDR string) ValidationCheck {
298+
return ValidationCheck{
299+
Name: "workspace CIDR is present and valid",
300+
Description: "ensures the workspace CIDR contains a valid network address range",
301+
Check: func(ctx context.Context, config *rest.Config, namespace string) ([]ValidationError, error) {
302+
netIP, ipNet, err := net.ParseCIDR(networkCIDR)
303+
if err != nil {
304+
return []ValidationError{
305+
{
306+
Message: fmt.Sprintf("invalid workspace CIDR: %v", err),
307+
Type: ValidationStatusError,
308+
},
309+
}, nil
310+
}
311+
312+
ipNet.Mask.Size()
313+
mask, _ := ipNet.Mask.Size()
314+
if mask > 30 {
315+
return []ValidationError{
316+
{
317+
Message: "the workspace CIDR does not have a mask less than or equal to /30",
318+
Type: ValidationStatusError,
319+
},
320+
}, nil
321+
}
322+
323+
addr, err := netip.ParseAddr(netIP.String())
324+
if err != nil {
325+
return []ValidationError{
326+
{
327+
Message: fmt.Sprintf("invalid workspace CIDR: %v", err),
328+
Type: ValidationStatusError,
329+
},
330+
}, nil
331+
}
332+
333+
vethIp := addr.Next()
334+
if !vethIp.IsValid() {
335+
return []ValidationError{
336+
{
337+
Message: fmt.Sprintf("workspace CIDR is not big enough (%v)", networkCIDR),
338+
Type: ValidationStatusError,
339+
},
340+
}, nil
341+
}
342+
343+
cethIp := vethIp.Next()
344+
if !cethIp.IsValid() {
345+
return []ValidationError{
346+
{
347+
Message: fmt.Sprintf("workspace CIDR is not big enough (%v)", networkCIDR),
348+
Type: ValidationStatusError,
349+
},
350+
}, nil
351+
}
352+
353+
return nil, nil
354+
},
355+
}
356+
}

‎install/installer/pkg/components/ws-daemon/configmap.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
6767

6868
var wscontroller daemon.WorkspaceControllerConfig
6969

70+
// default workspace network CIDR (and fallback)
71+
workspaceCIDR := "10.0.5.0/30"
72+
7073
ctx.WithExperimental(func(ucfg *experimental.Config) error {
7174
if ucfg.Workspace == nil {
7275
return nil
@@ -105,6 +108,10 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
105108
wscontroller.WorkingAreaSuffix = "-mk2"
106109
wscontroller.MaxConcurrentReconciles = 15
107110

111+
if ucfg.Workspace.WorkspaceCIDR != "" {
112+
workspaceCIDR = ucfg.Workspace.WorkspaceCIDR
113+
}
114+
108115
return nil
109116
})
110117

@@ -123,6 +130,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
123130
SocketPath: "/mnt/containerd/containerd.sock",
124131
},
125132
},
133+
WorkspaceCIDR: workspaceCIDR,
126134
},
127135
Content: content.Config{
128136
WorkingArea: "/mnt/workingarea",

‎install/installer/pkg/config/v1/experimental/experimental.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ type WorkspaceConfig struct {
6868
WorkspaceURLTemplate string `json:"workspaceURLTemplate,omitempty"`
6969
WorkspacePortURLTemplate string `json:"workspacePortURLTemplate,omitempty"`
7070

71+
WorkspaceCIDR string `json:"workspaceCIDR,omitempty"`
72+
7173
CPULimits struct {
7274
Enabled bool `json:"enabled"`
7375
NodeCPUBandwidth resource.Quantity `json:"nodeBandwidth"`

0 commit comments

Comments
 (0)