Skip to content

Commit 67ce7d0

Browse files
csweichelroboquat
csweichel
authored and
roboquat
committed
[ws-daemon] Use baseserver to run services
1 parent 7b68fb4 commit 67ce7d0

File tree

6 files changed

+46
-161
lines changed

6 files changed

+46
-161
lines changed

‎components/ws-daemon/cmd/run.go

Lines changed: 29 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -7,33 +7,26 @@ package cmd
77
import (
88
"context"
99
"fmt"
10-
"net"
11-
"net/http"
12-
"os"
13-
"os/signal"
14-
"syscall"
10+
"google.golang.org/grpc/credentials/insecure"
1511
"time"
1612

17-
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
1813
"github.com/heptiolabs/healthcheck"
1914
"github.com/prometheus/client_golang/prometheus"
20-
"github.com/prometheus/client_golang/prometheus/collectors"
21-
"github.com/prometheus/client_golang/prometheus/promhttp"
2215
"github.com/spf13/cobra"
23-
"golang.org/x/xerrors"
2416
"google.golang.org/grpc"
2517
"google.golang.org/grpc/credentials"
26-
"google.golang.org/grpc/health"
2718
"google.golang.org/grpc/health/grpc_health_v1"
2819

20+
"github.com/gitpod-io/gitpod/common-go/baseserver"
2921
common_grpc "github.com/gitpod-io/gitpod/common-go/grpc"
3022
"github.com/gitpod-io/gitpod/common-go/log"
31-
"github.com/gitpod-io/gitpod/common-go/pprof"
3223
"github.com/gitpod-io/gitpod/common-go/watch"
3324
"github.com/gitpod-io/gitpod/ws-daemon/pkg/config"
3425
"github.com/gitpod-io/gitpod/ws-daemon/pkg/daemon"
3526
)
3627

28+
const grpcServerName = "wsdaemon"
29+
3730
// serveCmd represents the serve command
3831
var runCmd = &cobra.Command{
3932
Use: "run",
@@ -42,91 +35,31 @@ var runCmd = &cobra.Command{
4235
Run: func(cmd *cobra.Command, args []string) {
4336
cfg, err := config.Read(configFile)
4437
if err != nil {
45-
log.WithError(err).Fatal("cannot read configuration. Maybe missing --config?")
46-
}
47-
reg := prometheus.NewRegistry()
48-
dmn, err := daemon.NewDaemon(cfg.Daemon, prometheus.WrapRegistererWithPrefix("gitpod_ws_daemon_", reg))
49-
if err != nil {
50-
log.WithError(err).Fatal("cannot create daemon")
38+
log.WithError(err).Fatal("Cannot read configuration. Maybe missing --config?")
5139
}
5240

53-
common_grpc.SetupLogging()
54-
55-
grpcMetrics := grpc_prometheus.NewServerMetrics()
56-
grpcMetrics.EnableHandlingTimeHistogram()
57-
reg.MustRegister(grpcMetrics)
58-
59-
grpcOpts := common_grpc.ServerOptionsWithInterceptors(
60-
[]grpc.StreamServerInterceptor{grpcMetrics.StreamServerInterceptor()},
61-
[]grpc.UnaryServerInterceptor{grpcMetrics.UnaryServerInterceptor()},
41+
health := healthcheck.NewHandler()
42+
srv, err := baseserver.New(grpcServerName,
43+
baseserver.WithGRPC(&cfg.Service),
44+
baseserver.WithHealthHandler(health),
6245
)
63-
tlsOpt, err := cfg.Service.TLS.ServerOption()
6446
if err != nil {
65-
log.WithError(err).Fatal("cannot use TLS config")
47+
log.WithError(err).Fatal("Cannot set up server.")
6648
}
67-
if tlsOpt != nil {
68-
log.WithField("crt", cfg.Service.TLS.Certificate).WithField("key", cfg.Service.TLS.PrivateKey).Debug("securing gRPC server with TLS")
69-
grpcOpts = append(grpcOpts, tlsOpt)
70-
} else {
71-
log.Warn("no TLS configured - gRPC server will be unsecured")
72-
}
73-
74-
healthServer := health.NewServer()
7549

76-
server := grpc.NewServer(grpcOpts...)
77-
server.RegisterService(&grpc_health_v1.Health_ServiceDesc, healthServer)
78-
79-
dmn.Register(server)
80-
lis, err := net.Listen("tcp", cfg.Service.Addr)
50+
dmn, err := daemon.NewDaemon(cfg.Daemon, prometheus.WrapRegistererWithPrefix("gitpod_ws_daemon_", srv.MetricsRegistry()))
8151
if err != nil {
82-
log.WithError(err).Fatalf("cannot listen on %s", cfg.Service.Addr)
52+
log.WithError(err).Fatal("Cannot create daemon.")
8353
}
84-
go func() {
85-
err := server.Serve(lis)
86-
if err != nil {
87-
log.WithError(err).Fatal("cannot start server")
88-
}
89-
}()
90-
log.WithField("addr", cfg.Service.Addr).Info("started gRPC server")
91-
92-
if cfg.Prometheus.Addr != "" {
93-
reg.MustRegister(
94-
collectors.NewGoCollector(),
95-
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
96-
)
97-
98-
handler := http.NewServeMux()
99-
handler.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
10054

101-
go func() {
102-
err := http.ListenAndServe(cfg.Prometheus.Addr, handler)
103-
if err != nil {
104-
log.WithError(err).Error("Prometheus metrics server failed")
105-
}
106-
}()
107-
log.WithField("addr", cfg.Prometheus.Addr).Info("started Prometheus metrics server")
108-
}
109-
110-
if cfg.PProf.Addr != "" {
111-
go pprof.Serve(cfg.PProf.Addr)
112-
}
55+
health.AddReadinessCheck("grpc-server", grpcProbe(cfg.Service))
56+
health.AddReadinessCheck("ws-daemon", dmn.ReadinessProbe())
11357

114-
if cfg.ReadinessProbeAddr != "" {
115-
// Ensure we can access the GRPC server is healthy, the etc hosts file was updated and containerd is available.
116-
health := healthcheck.NewHandler()
117-
health.AddReadinessCheck("grpc-server", grpcProbe(cfg.Service))
118-
health.AddReadinessCheck("ws-daemon", dmn.ReadinessProbe())
119-
120-
go func() {
121-
if err := http.ListenAndServe(cfg.ReadinessProbeAddr, health); err != nil && err != http.ErrServerClosed {
122-
log.WithError(err).Panic("error starting HTTP server")
123-
}
124-
}()
125-
}
58+
dmn.Register(srv.GRPC())
12659

12760
err = dmn.Start()
12861
if err != nil {
129-
log.WithError(err).Fatal("cannot start daemon")
62+
log.WithError(err).Fatal("Cannot start daemon.")
13063
}
13164

13265
ctx, cancel := context.WithCancel(context.Background())
@@ -138,57 +71,50 @@ var runCmd = &cobra.Command{
13871

13972
cfg, err := config.Read(configFile)
14073
if err != nil {
141-
log.WithError(err).Warn("cannot reload configuration")
74+
log.WithError(err).Warn("Cannot reload configuration.")
14275
return
14376
}
14477

14578
err = dmn.ReloadConfig(ctx, &cfg.Daemon)
14679
if err != nil {
147-
log.WithError(err).Warn("cannot reload configuration")
80+
log.WithError(err).Warn("Cannot reload configuration.")
14881
}
14982
})
15083
if err != nil {
151-
log.WithError(err).Fatal("cannot start watch of configuration file")
84+
log.WithError(err).Fatal("Cannot start watch of configuration file.")
15285
}
15386

154-
// run until we're told to stop
155-
sigChan := make(chan os.Signal, 1)
156-
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
157-
log.Info("🧫 ws-daemon is up and running. Stop with SIGINT or CTRL+C")
158-
<-sigChan
159-
server.Stop()
160-
err = dmn.Stop()
87+
err = srv.ListenAndServe()
16188
if err != nil {
162-
log.WithError(err).Error("cannot shut down gracefully")
89+
log.WithError(err).Fatal("Failed to listen and serve.")
16390
}
164-
log.Info("Received SIGINT - shutting down")
16591
},
16692
}
16793

16894
func init() {
16995
rootCmd.AddCommand(runCmd)
17096
}
17197

172-
func grpcProbe(tlsConfig config.AddrTLS) func() error {
98+
func grpcProbe(cfg baseserver.ServerConfiguration) func() error {
17399
return func() error {
174-
secopt := grpc.WithInsecure()
175-
if tlsConfig.TLS != nil && tlsConfig.TLS.Certificate != "" {
100+
creds := insecure.NewCredentials()
101+
if cfg.TLS != nil && cfg.TLS.CertPath != "" {
176102
tlsConfig, err := common_grpc.ClientAuthTLSConfig(
177-
tlsConfig.TLS.Authority, tlsConfig.TLS.Certificate, tlsConfig.TLS.PrivateKey,
103+
cfg.TLS.CAPath, cfg.TLS.CertPath, cfg.TLS.KeyPath,
178104
common_grpc.WithSetRootCAs(true),
179-
common_grpc.WithServerName("wsdaemon"),
105+
common_grpc.WithServerName(grpcServerName),
180106
)
181107
if err != nil {
182-
return xerrors.Errorf("cannot load ws-daemon certificate: %w", err)
108+
return fmt.Errorf("cannot load ws-daemon certificate: %w", err)
183109
}
184110

185-
secopt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
111+
creds = credentials.NewTLS(tlsConfig)
186112
}
187113

188114
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
189115
defer cancel()
190116

191-
conn, err := grpc.DialContext(ctx, tlsConfig.Addr, secopt)
117+
conn, err := grpc.DialContext(ctx, cfg.Address, grpc.WithTransportCredentials(creds))
192118
if err != nil {
193119
return err
194120
}

‎components/ws-daemon/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ require (
1414
github.com/gitpod-io/gitpod/ws-daemon/api v0.0.0-00010101000000-000000000000
1515
github.com/google/go-cmp v0.5.8
1616
github.com/google/uuid v1.3.0
17-
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
17+
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
1818
github.com/opencontainers/go-digest v1.0.0
1919
github.com/opencontainers/image-spec v1.0.2
2020
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417

‎components/ws-daemon/pkg/config/config.go

Lines changed: 3 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,12 @@ package config
66

77
import (
88
"bytes"
9-
"crypto/tls"
109
"encoding/json"
1110
"os"
1211

1312
"golang.org/x/xerrors"
14-
"google.golang.org/grpc"
15-
"google.golang.org/grpc/credentials"
1613

17-
common_grpc "github.com/gitpod-io/gitpod/common-go/grpc"
14+
"github.com/gitpod-io/gitpod/common-go/baseserver"
1815
"github.com/gitpod-io/gitpod/ws-daemon/pkg/daemon"
1916
)
2017

@@ -36,42 +33,6 @@ func Read(fn string) (*Config, error) {
3633
}
3734

3835
type Config struct {
39-
Daemon daemon.Config `json:"daemon"`
40-
Service AddrTLS `json:"service"`
41-
Prometheus Addr `json:"prometheus"`
42-
PProf Addr `json:"pprof"`
43-
ReadinessProbeAddr string `json:"readinessProbeAddr"`
44-
}
45-
46-
type Addr struct {
47-
Addr string `json:"address"`
48-
}
49-
50-
type AddrTLS struct {
51-
Addr string `json:"address"`
52-
TLS *TLS `json:"tls,omitempty"`
53-
}
54-
type TLS struct {
55-
Authority string `json:"ca"`
56-
Certificate string `json:"crt"`
57-
PrivateKey string `json:"key"`
58-
}
59-
60-
// ServerOption produces the GRPC option that configures a server to use this TLS configuration
61-
func (c *TLS) ServerOption() (grpc.ServerOption, error) {
62-
if c.Authority == "" || c.Certificate == "" || c.PrivateKey == "" {
63-
return nil, nil
64-
}
65-
66-
tlsConfig, err := common_grpc.ClientAuthTLSConfig(
67-
c.Authority, c.Certificate, c.PrivateKey,
68-
common_grpc.WithClientAuth(tls.RequireAndVerifyClientCert),
69-
common_grpc.WithSetClientCAs(true),
70-
common_grpc.WithServerName("ws-manager"),
71-
)
72-
if err != nil {
73-
return nil, xerrors.Errorf("cannot load certs: %w", err)
74-
}
75-
76-
return grpc.Creds(credentials.NewTLS(tlsConfig)), nil
36+
Daemon daemon.Config `json:"daemon"`
37+
Service baseserver.ServerConfiguration `json:"service"`
7738
}

‎install/installer/pkg/components/ws-daemon/configmap.go

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"fmt"
99
"time"
1010

11+
"github.com/gitpod-io/gitpod/common-go/baseserver"
1112
"github.com/gitpod-io/gitpod/common-go/util"
1213
"github.com/gitpod-io/gitpod/installer/pkg/common"
1314
config "github.com/gitpod-io/gitpod/installer/pkg/config/v1"
@@ -127,21 +128,14 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
127128
}},
128129
},
129130
},
130-
Service: wsdconfig.AddrTLS{
131-
Addr: fmt.Sprintf(":%d", ServicePort),
132-
TLS: &wsdconfig.TLS{
133-
Authority: "/certs/ca.crt",
134-
Certificate: "/certs/tls.crt",
135-
PrivateKey: "/certs/tls.key",
131+
Service: baseserver.ServerConfiguration{
132+
Address: fmt.Sprintf(":%d", ServicePort),
133+
TLS: &baseserver.TLSConfiguration{
134+
CAPath: "/certs/ca.crt",
135+
CertPath: "/certs/tls.crt",
136+
KeyPath: "/certs/tls.key",
136137
},
137138
},
138-
Prometheus: wsdconfig.Addr{
139-
Addr: "localhost:9500",
140-
},
141-
PProf: wsdconfig.Addr{
142-
Addr: "localhost:6060",
143-
},
144-
ReadinessProbeAddr: fmt.Sprintf(":%v", ReadinessPort),
145139
}
146140
fc, err := common.ToJSONString(wsdcfg)
147141
if err != nil {

‎install/installer/pkg/components/ws-daemon/constants.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
package wsdaemon
66

7+
import "github.com/gitpod-io/gitpod/common-go/baseserver"
8+
79
const (
810
Component = "ws-daemon"
911
ServicePort = 8080
@@ -12,5 +14,5 @@ const (
1214
HostBackupPath = "/var/gitpod/tmp/backup"
1315
TLSSecretName = "ws-daemon-tls"
1416
VolumeTLSCerts = "ws-daemon-tls-certs"
15-
ReadinessPort = 8086
17+
ReadinessPort = baseserver.BuiltinHealthPort
1618
)

‎install/installer/pkg/components/ws-manager/deployment.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
package wsmanager
66

77
import (
8+
"fmt"
9+
"github.com/gitpod-io/gitpod/common-go/baseserver"
810
"github.com/gitpod-io/gitpod/installer/pkg/cluster"
911
"github.com/gitpod-io/gitpod/installer/pkg/common"
1012
wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
@@ -77,7 +79,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
7779
},
7880
},
7981
},
80-
*common.KubeRBACProxyContainer(ctx),
82+
*common.KubeRBACProxyContainerWithConfig(ctx, 9500, fmt.Sprintf("http://127.0.0.1:%d/", baseserver.BuiltinMetricsPort)),
8183
},
8284
Volumes: []corev1.Volume{
8385
{

0 commit comments

Comments
 (0)