Skip to content

Commit 820f283

Browse files
aledbfaledbf
committed
Add new feature for custom workspace network CIDR
1 parent 2401a18 commit 820f283

File tree

8 files changed

+133
-25
lines changed

8 files changed

+133
-25
lines changed

‎components/ws-daemon/nsinsider/main.go

Lines changed: 55 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"fmt"
99
"io/ioutil"
1010
"net"
11+
"net/netip"
1112
"os"
1213
"os/exec"
1314
"path/filepath"
@@ -268,22 +269,30 @@ func main() {
268269
Name: "target-pid",
269270
Required: true,
270271
},
272+
&cli.IntFlag{
273+
Name: "workspace-cidr",
274+
Required: true,
275+
},
271276
},
272277
Action: func(c *cli.Context) error {
273278
containerIf, vethIf, cethIf := "eth0", "veth0", "eth0"
274-
mask := net.IPv4Mask(255, 255, 255, 0)
275-
vethIp := net.IPNet{
276-
IP: net.IPv4(10, 0, 5, 1),
277-
Mask: mask,
279+
networkCIDR := c.String("workspace-cidr")
280+
281+
vethIp, cethIp, mask, err := processWorkspaceCIDR(networkCIDR)
282+
if err != nil {
283+
return xerrors.Errorf("parsing workspace CIDR (%v):%v", networkCIDR, err)
278284
}
279-
cethIp := net.IPNet{
280-
IP: net.IPv4(10, 0, 5, 2),
281-
Mask: mask,
285+
286+
vethIpNet := net.IPNet{
287+
IP: net.ParseIP(vethIp.String()),
288+
Mask: mask.Mask,
282289
}
290+
283291
masqueradeAddr := net.IPNet{
284-
IP: vethIp.IP.Mask(mask),
285-
Mask: mask,
292+
IP: vethIpNet.IP.Mask(mask.Mask),
293+
Mask: mask.Mask,
286294
}
295+
287296
targetPid := c.Int("target-pid")
288297

289298
eth0, err := netlink.LinkByName(containerIf)
@@ -308,7 +317,7 @@ func main() {
308317
if err != nil {
309318
return xerrors.Errorf("cannot found %q netns failed: %v", vethIf, err)
310319
}
311-
if err := netlink.AddrAdd(vethLink, &netlink.Addr{IPNet: &vethIp}); err != nil {
320+
if err := netlink.AddrAdd(vethLink, &netlink.Addr{IPNet: &vethIpNet}); err != nil {
312321
return xerrors.Errorf("failed to add IP address to %q: %v", vethIf, err)
313322
}
314323
if err := netlink.LinkSetUp(vethLink); err != nil {
@@ -329,7 +338,6 @@ func main() {
329338
Type: nftables.ChainTypeNAT,
330339
})
331340

332-
// ip saddr 10.0.5.0/24 oifname "eth0" masquerade
333341
nc.AddRule(&nftables.Rule{
334342
Table: nat,
335343
Chain: postrouting,
@@ -408,7 +416,7 @@ func main() {
408416

409417
&expr.Immediate{
410418
Register: 2,
411-
Data: cethIp.IP.To4(),
419+
Data: cethIp,
412420
},
413421
&expr.NAT{
414422
Type: expr.NATTypeDestNAT,
@@ -431,21 +439,23 @@ func main() {
431439
Usage: "set up a peer veth",
432440
Action: func(c *cli.Context) error {
433441
cethIf := "eth0"
434-
mask := net.IPv4Mask(255, 255, 255, 0)
435-
cethIp := net.IPNet{
436-
IP: net.IPv4(10, 0, 5, 2),
437-
Mask: mask,
442+
443+
networkCIDR := c.String("workspace-cidr")
444+
vethIp, cethIp, mask, err := processWorkspaceCIDR(networkCIDR)
445+
if err != nil {
446+
return xerrors.Errorf("parsing workspace CIDR (%v):%v", networkCIDR, err)
438447
}
439-
vethIp := net.IPNet{
440-
IP: net.IPv4(10, 0, 5, 1),
441-
Mask: mask,
448+
449+
cethIpNet := net.IPNet{
450+
IP: net.ParseIP(cethIp.String()),
451+
Mask: mask.Mask,
442452
}
443453

444454
cethLink, err := netlink.LinkByName(cethIf)
445455
if err != nil {
446456
return xerrors.Errorf("cannot found %q netns failed: %v", cethIf, err)
447457
}
448-
if err := netlink.AddrAdd(cethLink, &netlink.Addr{IPNet: &cethIp}); err != nil {
458+
if err := netlink.AddrAdd(cethLink, &netlink.Addr{IPNet: &cethIpNet}); err != nil {
449459
return xerrors.Errorf("failed to add IP address to %q: %v", cethIf, err)
450460
}
451461
if err := netlink.LinkSetUp(cethLink); err != nil {
@@ -462,7 +472,7 @@ func main() {
462472

463473
defaultGw := netlink.Route{
464474
Scope: netlink.SCOPE_UNIVERSE,
465-
Gw: vethIp.IP,
475+
Gw: vethIp,
466476
}
467477
if err := netlink.RouteReplace(&defaultGw); err != nil {
468478
return xerrors.Errorf("failed to set up deafult gw: %v", err)
@@ -687,3 +697,27 @@ const (
687697
// FlagAtRecursive: Apply to the entire subtree: https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/fcntl.h#L112
688698
flagAtRecursive = 0x8000
689699
)
700+
701+
func processWorkspaceCIDR(networkCIDR string) (net.IP, net.IP, *net.IPNet, error) {
702+
netIP, mask, err := net.ParseCIDR(networkCIDR)
703+
if err != nil {
704+
return nil, nil, nil, xerrors.Errorf("cannot configure workspace CIDR: %w", err)
705+
}
706+
707+
addr, err := netip.ParseAddr(netIP.String())
708+
if err != nil {
709+
return nil, nil, nil, xerrors.Errorf("cannot configure workspace CIDR: %w", err)
710+
}
711+
712+
vethIp := addr.Next()
713+
if !vethIp.IsValid() {
714+
return nil, nil, nil, xerrors.Errorf("workspace CIDR is not big enough (%v)", networkCIDR)
715+
}
716+
717+
cethIp := vethIp.Next()
718+
if !cethIp.IsValid() {
719+
return nil, nil, nil, xerrors.Errorf("workspace CIDR is not big enough (%v)", networkCIDR)
720+
}
721+
722+
return net.ParseIP(vethIp.String()), net.ParseIP(cethIp.String()), mask, nil
723+
}

‎components/ws-daemon/pkg/content/hooks.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import (
2626
func WorkspaceLifecycleHooks(cfg Config, workspaceExistenceCheck WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, xfs *quota.XFS, cgroupMountPoint string) map[session.WorkspaceState][]session.WorkspaceLivecycleHook {
2727
// startIWS starts the in-workspace service for a workspace. This lifecycle hook is idempotent, hence can - and must -
2828
// be called on initialization and ready. The on-ready hook exists only to support ws-daemon restarts.
29-
startIWS := iws.ServeWorkspace(uidmapper, api.FSShiftMethod(cfg.UserNamespaces.FSShift), cgroupMountPoint)
29+
startIWS := iws.ServeWorkspace(uidmapper, api.FSShiftMethod(cfg.UserNamespaces.FSShift), cgroupMountPoint, cfg.WorkspaceCIDR)
3030

3131
return map[session.WorkspaceState][]session.WorkspaceLivecycleHook{
3232
session.WorkspaceInitializing: {

‎components/ws-daemon/pkg/daemon/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ type RuntimeConfig struct {
4343
Kubeconfig string `json:"kubeconfig"`
4444
KubernetesNamespace string `json:"namespace"`
4545
SecretsNamespace string `json:"secretsNamespace"`
46+
47+
WorkspaceNetworkCIDR string `json:"workspaceNetworkCIDR,omitempty"`
4648
}
4749

4850
type IOLimitConfig struct {

‎components/ws-daemon/pkg/iws/iws.go

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ var (
8484
)
8585

8686
// ServeWorkspace establishes the IWS server for a workspace
87-
func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMountPoint string) func(ctx context.Context, ws *session.Workspace) error {
87+
func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMountPoint string, workspaceCIDR string) func(ctx context.Context, ws *session.Workspace) error {
8888
return func(ctx context.Context, ws *session.Workspace) (err error) {
8989
span, _ := opentracing.StartSpanFromContext(ctx, "iws.ServeWorkspace")
9090
defer tracing.FinishSpan(span, &err)
@@ -139,6 +139,8 @@ type InWorkspaceServiceServer struct {
139139
FSShift api.FSShiftMethod
140140
CGroupMountPoint string
141141

142+
WorkspaceCIDR string
143+
142144
srv *grpc.Server
143145
sckt io.Closer
144146

@@ -361,7 +363,10 @@ func (wbs *InWorkspaceServiceServer) SetupPairVeths(ctx context.Context, req *ap
361363
}
362364

363365
err = nsi.Nsinsider(wbs.Session.InstanceID, int(containerPID), func(c *exec.Cmd) {
364-
c.Args = append(c.Args, "setup-pair-veths", "--target-pid", strconv.Itoa(int(req.Pid)))
366+
c.Args = append(c.Args, "setup-pair-veths",
367+
"--target-pid", strconv.Itoa(int(req.Pid)),
368+
fmt.Sprintf("--workspace-cidr=%v", wbs.WorkspaceCIDR),
369+
)
365370
}, nsi.EnterMountNS(true), nsi.EnterPidNS(true), nsi.EnterNetNS(true))
366371
if err != nil {
367372
log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a pair of veths")
@@ -373,7 +378,9 @@ func (wbs *InWorkspaceServiceServer) SetupPairVeths(ctx context.Context, req *ap
373378
return nil, xerrors.Errorf("cannot map in-container PID %d (container PID: %d): %w", req.Pid, containerPID, err)
374379
}
375380
err = nsi.Nsinsider(wbs.Session.InstanceID, int(pid), func(c *exec.Cmd) {
376-
c.Args = append(c.Args, "setup-peer-veth")
381+
c.Args = append(c.Args, "setup-peer-veth",
382+
fmt.Sprintf("--workspace-cidr=%v", wbs.WorkspaceCIDR),
383+
)
377384
}, nsi.EnterMountNS(true), nsi.EnterPidNS(true), nsi.EnterNetNS(true))
378385
if err != nil {
379386
log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a peer veths")

‎install/installer/pkg/cluster/checks.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ package cluster
77
import (
88
"context"
99
"fmt"
10+
"net"
11+
"net/netip"
1012
"strings"
1113

1214
"github.com/Masterminds/semver"
@@ -291,3 +293,53 @@ func checkNamespaceExists(ctx context.Context, config *rest.Config, namespace st
291293

292294
return nil, nil
293295
}
296+
297+
func CheckWorkspaceNetworkCIDR(networkCIDR string) ValidationCheck {
298+
return ValidationCheck{
299+
Name: "workspace CIDR is present and valid",
300+
Description: "ensures the workspace CIDR contains a valid network address range",
301+
Check: func(ctx context.Context, config *rest.Config, namespace string) ([]ValidationError, error) {
302+
netIP, _, err := net.ParseCIDR(networkCIDR)
303+
if err != nil {
304+
return []ValidationError{
305+
{
306+
Message: fmt.Sprintf("invalid workspace CIDR: %v", err),
307+
Type: ValidationStatusError,
308+
},
309+
}, nil
310+
}
311+
312+
addr, err := netip.ParseAddr(netIP.String())
313+
if err != nil {
314+
return []ValidationError{
315+
{
316+
Message: fmt.Sprintf("invalid workspace CIDR: %v", err),
317+
Type: ValidationStatusError,
318+
},
319+
}, nil
320+
}
321+
322+
vethIp := addr.Next()
323+
if !vethIp.IsValid() {
324+
return []ValidationError{
325+
{
326+
Message: fmt.Sprintf("workspace CIDR is not big enough (%v)", networkCIDR),
327+
Type: ValidationStatusError,
328+
},
329+
}, nil
330+
}
331+
332+
cethIp := vethIp.Next()
333+
if !cethIp.IsValid() {
334+
return []ValidationError{
335+
{
336+
Message: fmt.Sprintf("workspace CIDR is not big enough (%v)", networkCIDR),
337+
Type: ValidationStatusError,
338+
},
339+
}, nil
340+
}
341+
342+
return nil, nil
343+
},
344+
}
345+
}

‎install/installer/pkg/components/ws-daemon/configmap.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
6666
runtimeMapping[ctx.Config.Workspace.Runtime.ContainerDRuntimeDir] = "/mnt/node0"
6767

6868
var wscontroller daemon.WorkspaceControllerConfig
69+
var workspaceNetworkCIDR string
6970

7071
ctx.WithExperimental(func(ucfg *experimental.Config) error {
7172
if ucfg.Workspace == nil {
@@ -105,6 +106,8 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
105106
wscontroller.WorkingAreaSuffix = "-mk2"
106107
wscontroller.MaxConcurrentReconciles = 15
107108

109+
workspaceNetworkCIDR = ucfg.Workspace.WorkspaceNetworkCIDR
110+
108111
return nil
109112
})
110113

@@ -123,6 +126,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
123126
SocketPath: "/mnt/containerd/containerd.sock",
124127
},
125128
},
129+
WorkspaceNetworkCIDR: workspaceNetworkCIDR,
126130
},
127131
Content: content.Config{
128132
WorkingArea: "/mnt/workingarea",

‎install/installer/pkg/config/v1/experimental/experimental.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ type WorkspaceConfig struct {
6868
WorkspaceURLTemplate string `json:"workspaceURLTemplate,omitempty"`
6969
WorkspacePortURLTemplate string `json:"workspacePortURLTemplate,omitempty"`
7070

71+
WorkspaceNetworkCIDR string `json:"workspaceNetworkCIDR,omitempty"`
72+
7173
CPULimits struct {
7274
Enabled bool `json:"enabled"`
7375
NodeCPUBandwidth resource.Quantity `json:"nodeBandwidth"`

‎install/installer/pkg/config/v1/experimental/validation.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,13 @@ func ClusterValidation(cfg *Config) cluster.ValidationChecks {
4646
if scr := cfg.Workspace.RegistryFacade.RedisCache.PasswordSecret; scr != "" {
4747
res = append(res, cluster.CheckSecret(scr, cluster.CheckSecretRequiredData("password")))
4848
}
49+
50+
if cfg.Workspace.WorkspaceNetworkCIDR == "" {
51+
// default workspace network CIDR fallback
52+
cfg.Workspace.WorkspaceNetworkCIDR = "10.0.5.0/31"
53+
} else {
54+
res = append(res, cluster.CheckWorkspaceNetworkCIDR(cfg.Workspace.WorkspaceNetworkCIDR))
55+
}
4956
}
5057

5158
return res

0 commit comments

Comments
 (0)