Skip to content

Commit eed0983

Browse files
aledbfaledbf
committed
Add new feature for custom workspace network CIDR
1 parent 2401a18 commit eed0983

File tree

10 files changed

+148
-28
lines changed

10 files changed

+148
-28
lines changed

‎components/ws-daemon/nsinsider/main.go

Lines changed: 61 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"fmt"
99
"io/ioutil"
1010
"net"
11+
"net/netip"
1112
"os"
1213
"os/exec"
1314
"path/filepath"
@@ -268,22 +269,30 @@ func main() {
268269
Name: "target-pid",
269270
Required: true,
270271
},
272+
&cli.StringFlag{
273+
Name: "workspace-cidr",
274+
Required: true,
275+
},
271276
},
272277
Action: func(c *cli.Context) error {
273278
containerIf, vethIf, cethIf := "eth0", "veth0", "eth0"
274-
mask := net.IPv4Mask(255, 255, 255, 0)
275-
vethIp := net.IPNet{
276-
IP: net.IPv4(10, 0, 5, 1),
277-
Mask: mask,
279+
networkCIDR := c.String("workspace-cidr")
280+
281+
vethIp, cethIp, mask, err := processWorkspaceCIDR(networkCIDR)
282+
if err != nil {
283+
return xerrors.Errorf("parsing workspace CIDR (%v):%v", networkCIDR, err)
278284
}
279-
cethIp := net.IPNet{
280-
IP: net.IPv4(10, 0, 5, 2),
281-
Mask: mask,
285+
286+
vethIpNet := net.IPNet{
287+
IP: net.ParseIP(vethIp.String()),
288+
Mask: mask.Mask,
282289
}
290+
283291
masqueradeAddr := net.IPNet{
284-
IP: vethIp.IP.Mask(mask),
285-
Mask: mask,
292+
IP: vethIpNet.IP.Mask(mask.Mask),
293+
Mask: mask.Mask,
286294
}
295+
287296
targetPid := c.Int("target-pid")
288297

289298
eth0, err := netlink.LinkByName(containerIf)
@@ -308,7 +317,7 @@ func main() {
308317
if err != nil {
309318
return xerrors.Errorf("cannot found %q netns failed: %v", vethIf, err)
310319
}
311-
if err := netlink.AddrAdd(vethLink, &netlink.Addr{IPNet: &vethIp}); err != nil {
320+
if err := netlink.AddrAdd(vethLink, &netlink.Addr{IPNet: &vethIpNet}); err != nil {
312321
return xerrors.Errorf("failed to add IP address to %q: %v", vethIf, err)
313322
}
314323
if err := netlink.LinkSetUp(vethLink); err != nil {
@@ -329,7 +338,6 @@ func main() {
329338
Type: nftables.ChainTypeNAT,
330339
})
331340

332-
// ip saddr 10.0.5.0/24 oifname "eth0" masquerade
333341
nc.AddRule(&nftables.Rule{
334342
Table: nat,
335343
Chain: postrouting,
@@ -408,7 +416,7 @@ func main() {
408416

409417
&expr.Immediate{
410418
Register: 2,
411-
Data: cethIp.IP.To4(),
419+
Data: cethIp,
412420
},
413421
&expr.NAT{
414422
Type: expr.NATTypeDestNAT,
@@ -429,23 +437,31 @@ func main() {
429437
{
430438
Name: "setup-peer-veth",
431439
Usage: "set up a peer veth",
440+
Flags: []cli.Flag{
441+
&cli.StringFlag{
442+
Name: "workspace-cidr",
443+
Required: true,
444+
},
445+
},
432446
Action: func(c *cli.Context) error {
433447
cethIf := "eth0"
434-
mask := net.IPv4Mask(255, 255, 255, 0)
435-
cethIp := net.IPNet{
436-
IP: net.IPv4(10, 0, 5, 2),
437-
Mask: mask,
448+
449+
networkCIDR := c.String("workspace-cidr")
450+
vethIp, cethIp, mask, err := processWorkspaceCIDR(networkCIDR)
451+
if err != nil {
452+
return xerrors.Errorf("parsing workspace CIDR (%v):%v", networkCIDR, err)
438453
}
439-
vethIp := net.IPNet{
440-
IP: net.IPv4(10, 0, 5, 1),
441-
Mask: mask,
454+
455+
cethIpNet := net.IPNet{
456+
IP: net.ParseIP(cethIp.String()),
457+
Mask: mask.Mask,
442458
}
443459

444460
cethLink, err := netlink.LinkByName(cethIf)
445461
if err != nil {
446462
return xerrors.Errorf("cannot found %q netns failed: %v", cethIf, err)
447463
}
448-
if err := netlink.AddrAdd(cethLink, &netlink.Addr{IPNet: &cethIp}); err != nil {
464+
if err := netlink.AddrAdd(cethLink, &netlink.Addr{IPNet: &cethIpNet}); err != nil {
449465
return xerrors.Errorf("failed to add IP address to %q: %v", cethIf, err)
450466
}
451467
if err := netlink.LinkSetUp(cethLink); err != nil {
@@ -462,7 +478,7 @@ func main() {
462478

463479
defaultGw := netlink.Route{
464480
Scope: netlink.SCOPE_UNIVERSE,
465-
Gw: vethIp.IP,
481+
Gw: vethIp,
466482
}
467483
if err := netlink.RouteReplace(&defaultGw); err != nil {
468484
return xerrors.Errorf("failed to set up deafult gw: %v", err)
@@ -687,3 +703,27 @@ const (
687703
// FlagAtRecursive: Apply to the entire subtree: https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/fcntl.h#L112
688704
flagAtRecursive = 0x8000
689705
)
706+
707+
func processWorkspaceCIDR(networkCIDR string) (net.IP, net.IP, *net.IPNet, error) {
708+
netIP, mask, err := net.ParseCIDR(networkCIDR)
709+
if err != nil {
710+
return nil, nil, nil, xerrors.Errorf("cannot configure workspace CIDR: %w", err)
711+
}
712+
713+
addr, err := netip.ParseAddr(netIP.String())
714+
if err != nil {
715+
return nil, nil, nil, xerrors.Errorf("cannot configure workspace CIDR: %w", err)
716+
}
717+
718+
vethIp := addr.Next()
719+
if !vethIp.IsValid() {
720+
return nil, nil, nil, xerrors.Errorf("workspace CIDR is not big enough (%v)", networkCIDR)
721+
}
722+
723+
cethIp := vethIp.Next()
724+
if !cethIp.IsValid() {
725+
return nil, nil, nil, xerrors.Errorf("workspace CIDR is not big enough (%v)", networkCIDR)
726+
}
727+
728+
return net.ParseIP(vethIp.String()), net.ParseIP(cethIp.String()), mask, nil
729+
}

‎components/ws-daemon/pkg/content/hooks.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ import (
2323
)
2424

2525
// WorkspaceLifecycleHooks configures the lifecycle hooks for all workspaces
26-
func WorkspaceLifecycleHooks(cfg Config, workspaceExistenceCheck WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, xfs *quota.XFS, cgroupMountPoint string) map[session.WorkspaceState][]session.WorkspaceLivecycleHook {
26+
func WorkspaceLifecycleHooks(cfg Config, workspaceCIDR string, workspaceExistenceCheck WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, xfs *quota.XFS, cgroupMountPoint string) map[session.WorkspaceState][]session.WorkspaceLivecycleHook {
2727
// startIWS starts the in-workspace service for a workspace. This lifecycle hook is idempotent, hence can - and must -
2828
// be called on initialization and ready. The on-ready hook exists only to support ws-daemon restarts.
29-
startIWS := iws.ServeWorkspace(uidmapper, api.FSShiftMethod(cfg.UserNamespaces.FSShift), cgroupMountPoint)
29+
startIWS := iws.ServeWorkspace(uidmapper, api.FSShiftMethod(cfg.UserNamespaces.FSShift), cgroupMountPoint, workspaceCIDR)
3030

3131
return map[session.WorkspaceState][]session.WorkspaceLivecycleHook{
3232
session.WorkspaceInitializing: {

‎components/ws-daemon/pkg/content/service.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ type WorkspaceService struct {
6969
type WorkspaceExistenceCheck func(instanceID string) bool
7070

7171
// NewWorkspaceService creates a new workspce initialization service, starts housekeeping and the Prometheus integration
72-
func NewWorkspaceService(ctx context.Context, cfg Config, runtime container.Runtime, wec WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, cgroupMountPoint string, reg prometheus.Registerer) (res *WorkspaceService, err error) {
72+
func NewWorkspaceService(ctx context.Context, cfg Config, runtime container.Runtime, wec WorkspaceExistenceCheck, uidmapper *iws.Uidmapper, cgroupMountPoint string, reg prometheus.Registerer, workspaceCIDR string) (res *WorkspaceService, err error) {
7373
//nolint:ineffassign
7474
span, ctx := opentracing.StartSpanFromContext(ctx, "NewWorkspaceService")
7575
defer tracing.FinishSpan(span, &err)
@@ -86,7 +86,9 @@ func NewWorkspaceService(ctx context.Context, cfg Config, runtime container.Runt
8686
}
8787

8888
// read all session json files
89-
store, err := session.NewStore(ctx, cfg.WorkingArea, WorkspaceLifecycleHooks(cfg, wec, uidmapper, xfs, cgroupMountPoint))
89+
store, err := session.NewStore(ctx, cfg.WorkingArea,
90+
WorkspaceLifecycleHooks(cfg, workspaceCIDR, wec, uidmapper, xfs, cgroupMountPoint),
91+
)
9092
if err != nil {
9193
return nil, xerrors.Errorf("cannot create session store: %w", err)
9294
}

‎components/ws-daemon/pkg/daemon/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ type RuntimeConfig struct {
4343
Kubeconfig string `json:"kubeconfig"`
4444
KubernetesNamespace string `json:"namespace"`
4545
SecretsNamespace string `json:"secretsNamespace"`
46+
47+
WorkspaceCIDR string `json:"workspaceCIDR,omitempty"`
4648
}
4749

4850
type IOLimitConfig struct {

‎components/ws-daemon/pkg/daemon/daemon.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ func NewDaemon(config Config) (*Daemon, error) {
194194

195195
hooks := content.WorkspaceLifecycleHooks(
196196
contentCfg,
197+
config.Runtime.WorkspaceCIDR,
197198
func(instanceID string) bool { return true },
198199
&iws.Uidmapper{Config: config.Uidmapper, Runtime: containerRuntime},
199200
xfs,
@@ -239,6 +240,7 @@ func NewDaemon(config Config) (*Daemon, error) {
239240
&iws.Uidmapper{Config: config.Uidmapper, Runtime: containerRuntime},
240241
config.CPULimit.CGroupBasePath,
241242
wrappedReg,
243+
config.Runtime.WorkspaceCIDR,
242244
)
243245
if err != nil {
244246
return nil, xerrors.Errorf("cannot create content service: %w", err)

‎components/ws-daemon/pkg/iws/iws.go

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ var (
8484
)
8585

8686
// ServeWorkspace establishes the IWS server for a workspace
87-
func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMountPoint string) func(ctx context.Context, ws *session.Workspace) error {
87+
func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMountPoint string, workspaceCIDR string) func(ctx context.Context, ws *session.Workspace) error {
8888
return func(ctx context.Context, ws *session.Workspace) (err error) {
8989
span, _ := opentracing.StartSpanFromContext(ctx, "iws.ServeWorkspace")
9090
defer tracing.FinishSpan(span, &err)
@@ -98,6 +98,7 @@ func ServeWorkspace(uidmapper *Uidmapper, fsshift api.FSShiftMethod, cgroupMount
9898
Session: ws,
9999
FSShift: fsshift,
100100
CGroupMountPoint: cgroupMountPoint,
101+
WorkspaceCIDR: workspaceCIDR,
101102
}
102103
err = iws.Start()
103104
if err != nil {
@@ -139,6 +140,8 @@ type InWorkspaceServiceServer struct {
139140
FSShift api.FSShiftMethod
140141
CGroupMountPoint string
141142

143+
WorkspaceCIDR string
144+
142145
srv *grpc.Server
143146
sckt io.Closer
144147

@@ -361,7 +364,10 @@ func (wbs *InWorkspaceServiceServer) SetupPairVeths(ctx context.Context, req *ap
361364
}
362365

363366
err = nsi.Nsinsider(wbs.Session.InstanceID, int(containerPID), func(c *exec.Cmd) {
364-
c.Args = append(c.Args, "setup-pair-veths", "--target-pid", strconv.Itoa(int(req.Pid)))
367+
c.Args = append(c.Args, "setup-pair-veths",
368+
"--target-pid", strconv.Itoa(int(req.Pid)),
369+
fmt.Sprintf("--workspace-cidr=%v", wbs.WorkspaceCIDR),
370+
)
365371
}, nsi.EnterMountNS(true), nsi.EnterPidNS(true), nsi.EnterNetNS(true))
366372
if err != nil {
367373
log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a pair of veths")
@@ -373,7 +379,9 @@ func (wbs *InWorkspaceServiceServer) SetupPairVeths(ctx context.Context, req *ap
373379
return nil, xerrors.Errorf("cannot map in-container PID %d (container PID: %d): %w", req.Pid, containerPID, err)
374380
}
375381
err = nsi.Nsinsider(wbs.Session.InstanceID, int(pid), func(c *exec.Cmd) {
376-
c.Args = append(c.Args, "setup-peer-veth")
382+
c.Args = append(c.Args, "setup-peer-veth",
383+
fmt.Sprintf("--workspace-cidr=%v", wbs.WorkspaceCIDR),
384+
)
377385
}, nsi.EnterMountNS(true), nsi.EnterPidNS(true), nsi.EnterNetNS(true))
378386
if err != nil {
379387
log.WithError(err).WithFields(wbs.Session.OWI()).Error("SetupPairVeths: cannot setup a peer veths")

‎install/installer/pkg/cluster/checks.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ package cluster
77
import (
88
"context"
99
"fmt"
10+
"net"
11+
"net/netip"
1012
"strings"
1113

1214
"github.com/Masterminds/semver"
@@ -291,3 +293,53 @@ func checkNamespaceExists(ctx context.Context, config *rest.Config, namespace st
291293

292294
return nil, nil
293295
}
296+
297+
func CheckWorkspaceCIDR(networkCIDR string) ValidationCheck {
298+
return ValidationCheck{
299+
Name: "workspace CIDR is present and valid",
300+
Description: "ensures the workspace CIDR contains a valid network address range",
301+
Check: func(ctx context.Context, config *rest.Config, namespace string) ([]ValidationError, error) {
302+
netIP, _, err := net.ParseCIDR(networkCIDR)
303+
if err != nil {
304+
return []ValidationError{
305+
{
306+
Message: fmt.Sprintf("invalid workspace CIDR: %v", err),
307+
Type: ValidationStatusError,
308+
},
309+
}, nil
310+
}
311+
312+
addr, err := netip.ParseAddr(netIP.String())
313+
if err != nil {
314+
return []ValidationError{
315+
{
316+
Message: fmt.Sprintf("invalid workspace CIDR: %v", err),
317+
Type: ValidationStatusError,
318+
},
319+
}, nil
320+
}
321+
322+
vethIp := addr.Next()
323+
if !vethIp.IsValid() {
324+
return []ValidationError{
325+
{
326+
Message: fmt.Sprintf("workspace CIDR is not big enough (%v)", networkCIDR),
327+
Type: ValidationStatusError,
328+
},
329+
}, nil
330+
}
331+
332+
cethIp := vethIp.Next()
333+
if !cethIp.IsValid() {
334+
return []ValidationError{
335+
{
336+
Message: fmt.Sprintf("workspace CIDR is not big enough (%v)", networkCIDR),
337+
Type: ValidationStatusError,
338+
},
339+
}, nil
340+
}
341+
342+
return nil, nil
343+
},
344+
}
345+
}

‎install/installer/pkg/components/ws-daemon/configmap.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
6767

6868
var wscontroller daemon.WorkspaceControllerConfig
6969

70+
// default workspace network CIDR (and fallback)
71+
workspaceCIDR := "10.0.5.0/31"
72+
7073
ctx.WithExperimental(func(ucfg *experimental.Config) error {
7174
if ucfg.Workspace == nil {
7275
return nil
@@ -105,6 +108,10 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
105108
wscontroller.WorkingAreaSuffix = "-mk2"
106109
wscontroller.MaxConcurrentReconciles = 15
107110

111+
if ucfg.Workspace.WorkspaceCIDR != "" {
112+
workspaceCIDR = ucfg.Workspace.WorkspaceCIDR
113+
}
114+
108115
return nil
109116
})
110117

@@ -123,6 +130,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
123130
SocketPath: "/mnt/containerd/containerd.sock",
124131
},
125132
},
133+
WorkspaceCIDR: workspaceCIDR,
126134
},
127135
Content: content.Config{
128136
WorkingArea: "/mnt/workingarea",

‎install/installer/pkg/config/v1/experimental/experimental.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ type WorkspaceConfig struct {
6868
WorkspaceURLTemplate string `json:"workspaceURLTemplate,omitempty"`
6969
WorkspacePortURLTemplate string `json:"workspacePortURLTemplate,omitempty"`
7070

71+
WorkspaceCIDR string `json:"workspaceCIDR,omitempty"`
72+
7173
CPULimits struct {
7274
Enabled bool `json:"enabled"`
7375
NodeCPUBandwidth resource.Quantity `json:"nodeBandwidth"`

‎install/installer/pkg/config/v1/experimental/validation.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ func ClusterValidation(cfg *Config) cluster.ValidationChecks {
4646
if scr := cfg.Workspace.RegistryFacade.RedisCache.PasswordSecret; scr != "" {
4747
res = append(res, cluster.CheckSecret(scr, cluster.CheckSecretRequiredData("password")))
4848
}
49+
50+
if cfg.Workspace.WorkspaceCIDR != "" {
51+
res = append(res, cluster.CheckWorkspaceCIDR(cfg.Workspace.WorkspaceCIDR))
52+
}
4953
}
5054

5155
return res

0 commit comments

Comments
 (0)