Skip to content

Update getMetricsFromPort to infer port number #2279

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 21, 2021
Merged

Update getMetricsFromPort to infer port number #2279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ spec:
imagePullPolicy: {{ .Values.olm.image.pullPolicy }}
ports:
- containerPort: {{ .Values.olm.service.internalPort }}
name: metrics
livenessProbe:
httpGet:
path: /healthz
Expand Down
3 changes: 2 additions & 1 deletion deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ spec:
image: {{ .Values.catalog.image.ref }}
imagePullPolicy: {{ .Values.catalog.image.pullPolicy }}
ports:
- containerPort: {{ .Values.catalog.service.internalPort }}
- containerPort: {{ .Values.olm.service.internalPort }}
name: metrics
livenessProbe:
httpGet:
path: /healthz
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/installplan_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ var _ = Describe("Install Plan", func() {

BeforeEach(func() {
counter = 0
for _, metric := range getMetricsFromPod(ctx.Ctx().KubeClient(), getPodWithLabel(ctx.Ctx().KubeClient(), "app=catalog-operator"), "8080") {
for _, metric := range getMetricsFromPod(ctx.Ctx().KubeClient(), getPodWithLabel(ctx.Ctx().KubeClient(), "app=catalog-operator")) {
if metric.Family == "installplan_warnings_total" {
counter = metric.Value
}
Expand Down Expand Up @@ -182,7 +182,7 @@ var _ = Describe("Install Plan", func() {

It("increments a metric counting the warning", func() {
Eventually(func() []Metric {
return getMetricsFromPod(ctx.Ctx().KubeClient(), getPodWithLabel(ctx.Ctx().KubeClient(), "app=catalog-operator"), "8080")
return getMetricsFromPod(ctx.Ctx().KubeClient(), getPodWithLabel(ctx.Ctx().KubeClient(), "app=catalog-operator"))
}).Should(ContainElement(LikeMetric(
WithFamily("installplan_warnings_total"),
WithValueGreaterThan(counter),
Expand Down
43 changes: 27 additions & 16 deletions test/e2e/metrics_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"
"fmt"
"regexp"
"strconv"
"strings"
"sync"

Expand Down Expand Up @@ -80,7 +81,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {

It("generates csv_abnormal metric for OLM pod", func() {

Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"), "8080")).To(And(
Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"))).To(And(
ContainElement(LikeMetric(
WithFamily("csv_abnormal"),
WithName(failingCSV.Name),
Expand Down Expand Up @@ -108,7 +109,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {

It("deletes its associated CSV metrics", func() {
// Verify that when the csv has been deleted, it deletes the corresponding CSV metrics
Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"), "8080")).ToNot(And(
Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"))).ToNot(And(
ContainElement(LikeMetric(WithFamily("csv_abnormal"), WithName(failingCSV.Name))),
ContainElement(LikeMetric(WithFamily("csv_succeeded"), WithName(failingCSV.Name))),
))
Expand Down Expand Up @@ -138,7 +139,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {

// Verify metrics have been emitted for subscription
Eventually(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}).Should(ContainElement(LikeMetric(
WithFamily("subscription_sync_total"),
WithName("metric-subscription-for-create"),
Expand All @@ -153,7 +154,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
// Verify metrics have been emitted for dependency resolution
Eventually(func() bool {
return Eventually(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}).Should(ContainElement(LikeMetric(
WithFamily("olm_resolution_duration_seconds"),
WithLabel("outcome", "failed"),
Expand All @@ -168,7 +169,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
BeforeEach(func() {
subscriptionCleanup, subscription = createSubscription(GinkgoT(), crc, testNamespace, "metric-subscription-for-update", testPackageName, stableChannel, v1alpha1.ApprovalManual)
Eventually(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}).Should(ContainElement(LikeMetric(WithFamily("subscription_sync_total"), WithLabel("name", "metric-subscription-for-update"))))
Eventually(func() error {
s, err := crc.OperatorsV1alpha1().Subscriptions(subscription.GetNamespace()).Get(context.TODO(), subscription.GetName(), metav1.GetOptions{})
Expand All @@ -189,7 +190,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {

It("deletes the old Subscription metric and emits the new metric", func() {
Eventually(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}).Should(And(
Not(ContainElement(LikeMetric(
WithFamily("subscription_sync_total"),
Expand Down Expand Up @@ -223,7 +224,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {

It("deletes the old subscription metric and emits the new metric(there is only one metric for the subscription)", func() {
Eventually(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}).Should(And(
Not(ContainElement(LikeMetric(
WithFamily("subscription_sync_total"),
Expand Down Expand Up @@ -253,7 +254,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
BeforeEach(func() {
subscriptionCleanup, subscription = createSubscription(GinkgoT(), crc, testNamespace, "metric-subscription-for-delete", testPackageName, stableChannel, v1alpha1.ApprovalManual)
Eventually(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}).Should(ContainElement(LikeMetric(WithFamily("subscription_sync_total"), WithLabel("name", "metric-subscription-for-delete"))))
if subscriptionCleanup != nil {
subscriptionCleanup()
Expand All @@ -269,7 +270,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {

It("deletes the Subscription metric", func() {
Eventually(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}).ShouldNot(ContainElement(LikeMetric(WithFamily("subscription_sync_total"), WithName("metric-subscription-for-delete"))))
})
})
Expand Down Expand Up @@ -312,7 +313,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
})
It("emits metrics for the catalogSource", func() {
Eventually(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}).Should(And(
ContainElement(LikeMetric(
WithFamily("catalog_source_count"),
Expand All @@ -332,7 +333,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
})
It("deletes the metrics for the CatalogSource", func() {
Eventually(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}).Should(And(
Not(ContainElement(LikeMetric(
WithFamily("catalogsource_ready"),
Expand All @@ -356,7 +357,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
})
It("emits metrics for the CatlogSource with a Value greater than 0", func() {
Eventually(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}).Should(And(
ContainElement(LikeMetric(
WithFamily("catalogsource_ready"),
Expand All @@ -366,7 +367,7 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
)),
))
Consistently(func() []Metric {
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"), "8080")
return getMetricsFromPod(c, getPodWithLabel(c, "app=catalog-operator"))
}, "3m").Should(And(
ContainElement(LikeMetric(
WithFamily("catalogsource_ready"),
Expand Down Expand Up @@ -395,7 +396,18 @@ func getPodWithLabel(client operatorclient.ClientInterface, label string) *corev
return &podList.Items[0]
}

func getMetricsFromPod(client operatorclient.ClientInterface, pod *corev1.Pod, port string) []Metric {
func extractMetricPortFromPod(pod *corev1.Pod) string {
for _, container := range pod.Spec.Containers {
for _, port := range container.Ports {
if port.Name == "metrics" {
return strconv.Itoa(int(port.ContainerPort))
}
}
}
return "-1"
}

func getMetricsFromPod(client operatorclient.ClientInterface, pod *corev1.Pod) []Metric {
ctx.Ctx().Logf("querying pod %s/%s\n", pod.GetNamespace(), pod.GetName())

// assuming -tls-cert and -tls-key aren't used anywhere else as a parameter value
Expand All @@ -417,14 +429,13 @@ func getMetricsFromPod(client operatorclient.ClientInterface, pod *corev1.Pod, p
scheme = "http"
}
ctx.Ctx().Logf("Retrieving metrics using scheme %v\n", scheme)

mfs := make(map[string]*io_prometheus_client.MetricFamily)
EventuallyWithOffset(1, func() error {
raw, err := client.KubernetesInterface().CoreV1().RESTClient().Get().
Namespace(pod.GetNamespace()).
Resource("pods").
SubResource("proxy").
Name(net.JoinSchemeNamePort(scheme, pod.GetName(), port)).
Name(net.JoinSchemeNamePort(scheme, pod.GetName(), extractMetricPortFromPod(pod))).
Suffix("metrics").
Do(context.Background()).Raw()
if err != nil {
Expand Down