Skip to content

Commit 52d6dbf

Browse files
DirectXMan12DirectXMan12
committed
Refactor health probes
This refactors the health probes, bringing the style a bit more in line with CR. The health probes themselves are now their own handler, and you add checks simply by using a map from check name to checker (which is just a function). The internal structure should now be more condusive to JSON output as well.
1 parent 9bc08d8 commit 52d6dbf

File tree

6 files changed

+324
-239
lines changed

6 files changed

+324
-239
lines changed

‎pkg/healthz/doc.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,15 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
// This package is almost exact copy of apiserver's healthz package:
18-
// https://github.com/kubernetes/apiserver/tree/master/pkg/server/healthz
17+
// Package healthz contains helpers from supporting liveness and readiness endpoints.
18+
// (often referred to as healthz and readyz, respectively).
1919
//
20-
// Except that LogHealthz checker removed
21-
// and some style fixes is made to satisfy linters
20+
// This package draws heavily from the apiserver's healthz package
21+
// ( https://github.com/kubernetes/apiserver/tree/master/pkg/server/healthz )
22+
// but has some changes to bring it in line with controller-runtime's style.
23+
//
24+
// The main entrypoint is the Handler -- this serves both aggregated health status
25+
// and individual health check endpoints.
2226
package healthz
2327

2428
import (

‎pkg/healthz/healthz.go

Lines changed: 138 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -17,98 +17,176 @@ limitations under the License.
1717
package healthz
1818

1919
import (
20-
"bytes"
2120
"fmt"
2221
"net/http"
22+
"path"
23+
"sort"
2324
"strings"
2425

2526
"k8s.io/apimachinery/pkg/util/sets"
2627
)
2728

29+
// Handler is an http.Handler that aggregates the results of the given
30+
// checkers to the root path, and supports calling individual checkers on
31+
// subpaths of the name of the checker.
32+
//
33+
// Adding checks on the fly is *not* threadsafe -- use a wrapper.
2834
type Handler struct {
29-
checks []Checker
35+
Checks map[string]Checker
3036
}
3137

32-
// GetChecks returns all health checks.
33-
func (h *Handler) GetChecks() []Checker {
34-
return h.checks
38+
// checkStatus holds the output of a particular check
39+
type checkStatus struct {
40+
name string
41+
healthy bool
42+
excluded bool
3543
}
3644

37-
// AddCheck adds new health check to handler.
38-
func (h *Handler) AddCheck(check Checker) {
39-
h.checks = append(h.checks, check)
40-
}
45+
func (h *Handler) serveAggregated(resp http.ResponseWriter, req *http.Request) {
46+
failed := false
47+
excluded := getExcludedChecks(req)
4148

42-
// Build creates http.Handler that serves checks.
43-
func (h *Handler) Build() http.Handler {
44-
return handleRootHealthz(h.checks...)
45-
}
49+
parts := make([]checkStatus, 0, len(h.Checks))
4650

47-
// HealthzChecker is a named healthz checker.
48-
type Checker interface {
49-
Name() string
50-
Check(req *http.Request) error
51-
}
51+
// calculate the results...
52+
for checkName, check := range h.Checks {
53+
// no-op the check if we've specified we want to exclude the check
54+
if excluded.Has(checkName) {
55+
excluded.Delete(checkName)
56+
parts = append(parts, checkStatus{name: checkName, healthy: true, excluded: true})
57+
continue
58+
}
59+
if err := check(req); err != nil {
60+
log.V(1).Info("healthz check failed", "checker", checkName, "error", err)
61+
parts = append(parts, checkStatus{name: checkName, healthy: false})
62+
failed = true
63+
} else {
64+
parts = append(parts, checkStatus{name: checkName, healthy: true})
65+
}
66+
}
5267

53-
// PingHealthz returns true automatically when checked
54-
var PingHealthz Checker = ping{}
68+
// ...default a check if none is present...
69+
if len(h.Checks) == 0 {
70+
parts = append(parts, checkStatus{name: "ping", healthy: true})
71+
}
5572

56-
// ping implements the simplest possible healthz checker.
57-
type ping struct{}
73+
for _, c := range excluded.List() {
74+
log.V(1).Info("cannot exclude health check, no matches for it", "checker", c)
75+
}
5876

59-
func (ping) Name() string {
60-
return "ping"
61-
}
77+
// ...sort to be consistent...
78+
sort.Slice(parts, func(i, j int) bool { return parts[i].name < parts[j].name })
79+
80+
// ...and write out the result
81+
// TODO(directxman12): this should also accept a request for JSON content (via a accept header)
82+
_, forceVerbose := req.URL.Query()["verbose"]
83+
writeStatusesAsText(resp, parts, excluded, failed, forceVerbose)
84+
}
85+
86+
// writeStatusAsText writes out the given check statuses in some semi-arbitrary
87+
// bespoke text format that we copied from Kubernetes. unknownExcludes lists
88+
// any checks that the user requested to have excluded, but weren't actually
89+
// known checks. writeStatusAsText is always verbose on failure, and can be
90+
// forced to be verbose on success using the given argument.
91+
func writeStatusesAsText(resp http.ResponseWriter, parts []checkStatus, unknownExcludes sets.String, failed, forceVerbose bool) {
92+
resp.Header().Set("Content-Type", "text/plain; charset=utf-8")
93+
resp.Header().Set("X-Content-Type-Options", "nosniff")
94+
95+
// always write status code first
96+
if failed {
97+
resp.WriteHeader(http.StatusInternalServerError)
98+
} else {
99+
resp.WriteHeader(http.StatusOK)
100+
}
62101

63-
// PingHealthz is a health check that returns true.
64-
func (ping) Check(_ *http.Request) error {
65-
return nil
66-
}
102+
// shortcut for easy non-verbose success
103+
if !failed && !forceVerbose {
104+
fmt.Fprint(resp, "ok")
105+
return
106+
}
67107

68-
// NamedCheck returns a healthz checker for the given name and function.
69-
func NamedCheck(name string, check func(r *http.Request) error) Checker {
70-
return &healthzCheck{name, check}
71-
}
108+
// we're always verbose on failure, so from this point on we're guaranteed to be verbose
109+
110+
for _, checkOut := range parts {
111+
switch {
112+
case checkOut.excluded:
113+
fmt.Fprintf(resp, "[+]%s excluded: ok\n", checkOut.name)
114+
case checkOut.healthy:
115+
fmt.Fprintf(resp, "[+]%s ok\n", checkOut.name)
116+
default:
117+
// don't include the error since this endpoint is public. If someone wants more detail
118+
// they should have explicit permission to the detailed checks.
119+
fmt.Fprintf(resp, "[-]%s failed: reason withheld\n", checkOut.name)
120+
}
121+
}
72122

73-
// InstallPathHandler registers handlers for health checking on
74-
// a specific path to mux. *All handlers* for the path must be
75-
// specified in exactly one call to InstallPathHandler. Calling
76-
// InstallPathHandler more than once for the same path and mux will
77-
// result in a panic.
78-
func InstallPathHandler(mux mux, path string, handler *Handler) {
79-
if len(handler.GetChecks()) == 0 {
80-
log.V(1).Info("No default health checks specified. Installing the ping handler.")
81-
handler.AddCheck(PingHealthz)
123+
if unknownExcludes.Len() > 0 {
124+
fmt.Fprintf(resp, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(unknownExcludes.List()...))
82125
}
83126

84-
mux.Handle(path, handler.Build())
85-
for _, check := range handler.GetChecks() {
86-
log.V(1).Info("installing healthz checker", "checker", check.Name())
87-
mux.Handle(fmt.Sprintf("%s/%s", path, check.Name()), adaptCheckToHandler(check.Check))
127+
if failed {
128+
log.Info("healthz check failed", "statuses", parts)
129+
fmt.Fprintf(resp, "healthz check failed\n")
130+
} else {
131+
fmt.Fprint(resp, "healthz check passed\n")
88132
}
89133
}
90134

91-
// mux is an interface describing the methods InstallHandler requires.
92-
type mux interface {
93-
Handle(pattern string, handler http.Handler)
94-
}
135+
func (h *Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
136+
// clean up the request (duplicating the internal logic of http.ServeMux a bit)
137+
// clean up the path a bit
138+
reqPath := req.URL.Path
139+
if reqPath == "" || reqPath[0] != '/' {
140+
reqPath = "/" + reqPath
141+
}
142+
// path.Clean removes the trailing slash except for root for us
143+
// (which is fine, since we're only serving one layer of sub-paths)
144+
reqPath = path.Clean(reqPath)
145+
146+
// either serve the root endpoint...
147+
if reqPath == "/" {
148+
h.serveAggregated(resp, req)
149+
return
150+
}
95151

96-
// healthzCheck implements HealthzChecker on an arbitrary name and check function.
97-
type healthzCheck struct {
98-
name string
99-
check func(r *http.Request) error
100-
}
152+
// ...the default check (if nothing else is present)...
153+
if len(h.Checks) == 0 && reqPath[1:] == "ping" {
154+
CheckHandler{Checker: Ping}.ServeHTTP(resp, req)
155+
return
156+
}
101157

102-
var _ Checker = &healthzCheck{}
158+
// ...or an individual checker
159+
checkName := reqPath[1:] // ignore the leading slash
160+
checker, known := h.Checks[checkName]
161+
if !known {
162+
http.NotFoundHandler().ServeHTTP(resp, req)
163+
return
164+
}
103165

104-
func (c *healthzCheck) Name() string {
105-
return c.name
166+
CheckHandler{Checker: checker}.ServeHTTP(resp, req)
106167
}
107168

108-
func (c *healthzCheck) Check(r *http.Request) error {
109-
return c.check(r)
169+
// CheckHandler is an http.Handler that serves a health check endpoint at the root path,
170+
// based on its checker.
171+
type CheckHandler struct {
172+
Checker
173+
}
174+
175+
func (h CheckHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
176+
err := h.Checker(req)
177+
if err != nil {
178+
http.Error(resp, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError)
179+
} else {
180+
fmt.Fprint(resp, "ok")
181+
}
110182
}
111183

184+
// Checker knows how to perform a health check.
185+
type Checker func(req *http.Request) error
186+
187+
// Ping returns true automatically when checked
188+
var Ping Checker = func(_ *http.Request) error { return nil }
189+
112190
// getExcludedChecks extracts the health check names to be excluded from the query param
113191
func getExcludedChecks(r *http.Request) sets.String {
114192
checks, found := r.URL.Query()["exclude"]
@@ -118,71 +196,6 @@ func getExcludedChecks(r *http.Request) sets.String {
118196
return sets.NewString()
119197
}
120198

121-
// handleRootHealthz returns an http.HandlerFunc that serves the provided checks.
122-
func handleRootHealthz(checks ...Checker) http.HandlerFunc {
123-
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
124-
failed := false
125-
excluded := getExcludedChecks(r)
126-
var verboseOut bytes.Buffer
127-
for _, check := range checks {
128-
// no-op the check if we've specified we want to exclude the check
129-
if excluded.Has(check.Name()) {
130-
excluded.Delete(check.Name())
131-
fmt.Fprintf(&verboseOut, "[+]%s excluded: ok\n", check.Name())
132-
continue
133-
}
134-
if err := check.Check(r); err != nil {
135-
// don't include the error since this endpoint is public. If someone wants more detail
136-
// they should have explicit permission to the detailed checks.
137-
log.V(1).Info("healthz check failed", "checker", check.Name(), "error", err)
138-
fmt.Fprintf(&verboseOut, "[-]%s failed: reason withheld\n", check.Name())
139-
failed = true
140-
} else {
141-
fmt.Fprintf(&verboseOut, "[+]%s ok\n", check.Name())
142-
}
143-
}
144-
if excluded.Len() > 0 {
145-
fmt.Fprintf(&verboseOut, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(excluded.List()...))
146-
for _, c := range excluded.List() {
147-
log.Info("cannot exclude health check, no matches for it", "checker", c)
148-
}
149-
}
150-
// always be verbose on failure
151-
if failed {
152-
log.V(1).Info("healthz check failed", "message", verboseOut.String())
153-
http.Error(w, fmt.Sprintf("%shealthz check failed", verboseOut.String()), http.StatusInternalServerError)
154-
return
155-
}
156-
157-
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
158-
w.Header().Set("X-Content-Type-Options", "nosniff")
159-
if _, found := r.URL.Query()["verbose"]; !found {
160-
fmt.Fprint(w, "ok")
161-
return
162-
}
163-
164-
_, err := verboseOut.WriteTo(w)
165-
if err != nil {
166-
log.V(1).Info("healthz check failed", "message", verboseOut.String())
167-
http.Error(w, fmt.Sprintf("%shealthz check failed", verboseOut.String()), http.StatusInternalServerError)
168-
return
169-
}
170-
fmt.Fprint(w, "healthz check passed\n")
171-
})
172-
}
173-
174-
// adaptCheckToHandler returns an http.HandlerFunc that serves the provided checks.
175-
func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc {
176-
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
177-
err := c(r)
178-
if err != nil {
179-
http.Error(w, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError)
180-
} else {
181-
fmt.Fprint(w, "ok")
182-
}
183-
})
184-
}
185-
186199
// formatQuoted returns a formatted string of the health check names,
187200
// preserving the order passed in.
188201
func formatQuoted(names ...string) string {

0 commit comments

Comments
 (0)