[baseserver] Emit metrics for logs produced by level ENG-349 (#19063)

* [baseserver] Emit metrics for logs produced by level

* fix

* fix

* fix

* fix

* Fix

* Fix

* fix

* retest

* fix

Author: easyCZ

components/common-go/baseserver/server.go

@@ -16,6 +16,7 @@ import (
 	"syscall"
 
 	common_grpc "github.com/gitpod-io/gitpod/common-go/grpc"
+	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/pprof"
 	"github.com/gitpod-io/gitpod/common-go/tracing"
 	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
@@ -348,6 +349,10 @@ func (s *Server) initializeMetrics() error {
 		return fmt.Errorf("failed to register baseserver metrics: %w", err)
 	}
 
+	if err := s.MetricsRegistry().Register(log.DefaultMetrics); err != nil {
+		return fmt.Errorf("failed to register log metrics: %w", err)
+	}
+
 	reportServerVersion(s.options.version)
 
 	return nil

components/common-go/log/log.go

@@ -54,6 +54,8 @@ func Init(service, version string, json, verbose bool) {
 	Log = log.WithFields(ServiceContext(service, version))
 	log.SetReportCaller(true)
 
+	log.AddHook(NewLogHook(DefaultMetrics))
+
 	if json {
 		Log.Logger.SetFormatter(newGcpFormatter(false))
 	} else {

components/common-go/log/metrics.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License.AGPL.txt in the project root for license information.
+
+package log
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	DefaultMetrics = NewMetrics()
+)
+
+type Metrics struct {
+	logEmitedCounter *prometheus.CounterVec
+}
+
+func (m *Metrics) ReportLog(level logrus.Level) {
+	m.logEmitedCounter.WithLabelValues(level.String()).Inc()
+}
+
+func NewMetrics() *Metrics {
+	return &Metrics{
+		logEmitedCounter: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "gitpod_logs_total",
+			Help: "Total number of logs produced by level",
+		}, []string{"level"}),
+	}
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector to the provided channel and returns once
+// the last descriptor has been sent.
+func (m *Metrics) Describe(ch chan<- *prometheus.Desc) {
+	m.logEmitedCounter.Describe(ch)
+}
+
+// Collect is called by the Prometheus registry when collecting
+// metrics. The implementation sends each collected metric via the
+// provided channel and returns once the last metric has been sent.
+func (m *Metrics) Collect(ch chan<- prometheus.Metric) {
+	m.logEmitedCounter.Collect(ch)
+}
+
+func NewLogHook(metrics *Metrics) *LogHook {
+	return &LogHook{metrics: metrics}
+}
+
+type LogHook struct {
+	metrics *Metrics
+}
+
+func (h *LogHook) Levels() []logrus.Level {
+	return logrus.AllLevels
+}
+
+func (h *LogHook) Fire(entry *logrus.Entry) error {
+	h.metrics.ReportLog(entry.Level)
+	return nil
+}

components/gitpod-cli/go.mod

@@ -30,15 +30,21 @@ require (
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/gitpod-io/gitpod/components/scrubber v0.0.0-00010101000000-000000000000 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/mattn/go-colorable v0.0.9 // indirect
 	github.com/mattn/go-isatty v0.0.3 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.24.2 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
 	golang.org/x/sys v0.11.0 // indirect
 	google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect

components/gitpod-cli/go.sum

@@ -20,6 +20,8 @@ require (
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gitpod-io/gitpod/components/scrubber v0.0.0-00010101000000-000000000000 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
@@ -29,8 +31,13 @@ require (
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
 	go.opentelemetry.io/otel v1.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
 	go.opentelemetry.io/otel/trace v1.16.0 // indirect

components/gitpod-db/go/go.mod

@@ -14,7 +14,9 @@ require (
 require golang.org/x/sys v0.11.0 // indirect
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/gitpod-io/gitpod/components/scrubber v0.0.0-00010101000000-000000000000 // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -23,7 +25,13 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/sourcegraph/jsonrpc2 v0.0.0-20200429184054-15c2290dcb37 // indirect
 	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/text v0.8.0 // indirect