Skip to content

Commit 0085b01

Browse files
DirectXMan12DirectXMan12
committed
Refactor health probes
This refactors the health probes, bringing the style a bit more in line with CR. The health probes themselves are now their own handler, and you add checks simply by using a map from check name to checker (which is just a function). The internal structure should now be more condusive to JSON output as well.
1 parent 9bc08d8 commit 0085b01

File tree

6 files changed

+316
-236
lines changed

6 files changed

+316
-236
lines changed

‎pkg/healthz/doc.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,15 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
// This package is almost exact copy of apiserver's healthz package:
18-
// https://github.com/kubernetes/apiserver/tree/master/pkg/server/healthz
17+
// Package healthz contains helpers from supporting liveness and readiness endpoints.
18+
// (often referred to as healthz and readyz, respectively).
1919
//
20-
// Except that LogHealthz checker removed
21-
// and some style fixes is made to satisfy linters
20+
// This package draws heavily from the apiserver's healthz package
21+
// ( https://github.com/kubernetes/apiserver/tree/master/pkg/server/healthz )
22+
// but has some changes to bring it in line with controller-runtime's style.
23+
//
24+
// The main entrypoint is the Handler -- this serves both aggregated health status
25+
// and individual health check endpoints.
2226
package healthz
2327

2428
import (

‎pkg/healthz/healthz.go

Lines changed: 132 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -17,98 +17,171 @@ limitations under the License.
1717
package healthz
1818

1919
import (
20-
"bytes"
2120
"fmt"
2221
"net/http"
22+
"path"
23+
"sort"
2324
"strings"
2425

2526
"k8s.io/apimachinery/pkg/util/sets"
2627
)
2728

29+
// Handler is an http.Handler that aggregates the results of the given
30+
// checkers to the root path, and supports calling individual checkers on
31+
// subpaths of the name of the checker.
32+
//
33+
// Adding checks on the fly is *not* threadsafe -- use a wrapper.
2834
type Handler struct {
29-
checks []Checker
35+
Checks map[string]Checker
3036
}
3137

32-
// GetChecks returns all health checks.
33-
func (h *Handler) GetChecks() []Checker {
34-
return h.checks
38+
// checkStatus holds the output of a particular check
39+
type checkStatus struct {
40+
name string
41+
healthy bool
42+
excluded bool
3543
}
3644

37-
// AddCheck adds new health check to handler.
38-
func (h *Handler) AddCheck(check Checker) {
39-
h.checks = append(h.checks, check)
40-
}
45+
func (h *Handler) serveAggregated(resp http.ResponseWriter, req *http.Request) {
46+
failed := false
47+
excluded := getExcludedChecks(req)
4148

42-
// Build creates http.Handler that serves checks.
43-
func (h *Handler) Build() http.Handler {
44-
return handleRootHealthz(h.checks...)
45-
}
49+
parts := make([]checkStatus, 0, len(h.Checks))
4650

47-
// HealthzChecker is a named healthz checker.
48-
type Checker interface {
49-
Name() string
50-
Check(req *http.Request) error
51-
}
51+
// calculate the results...
52+
for checkName, check := range h.Checks {
53+
// no-op the check if we've specified we want to exclude the check
54+
if excluded.Has(checkName) {
55+
excluded.Delete(checkName)
56+
parts = append(parts, checkStatus{name: checkName, healthy: true, excluded: true})
57+
continue
58+
}
59+
if err := check(req); err != nil {
60+
log.V(1).Info("healthz check failed", "checker", checkName, "error", err)
61+
parts = append(parts, checkStatus{name: checkName, healthy: false})
62+
failed = true
63+
} else {
64+
parts = append(parts, checkStatus{name: checkName, healthy: true})
65+
}
66+
}
5267

53-
// PingHealthz returns true automatically when checked
54-
var PingHealthz Checker = ping{}
68+
// ...default a check if none is present...
69+
if len(h.Checks) == 0 {
70+
parts = append(parts, checkStatus{name: "ping", healthy: true})
71+
}
5572

56-
// ping implements the simplest possible healthz checker.
57-
type ping struct{}
73+
for _, c := range excluded.List() {
74+
log.V(1).Info("cannot exclude health check, no matches for it", "checker", c)
75+
}
5876

59-
func (ping) Name() string {
60-
return "ping"
61-
}
77+
// ...sort to be consistent...
78+
sort.Slice(parts, func(i, j int) bool { return parts[i].name < parts[j].name })
6279

63-
// PingHealthz is a health check that returns true.
64-
func (ping) Check(_ *http.Request) error {
65-
return nil
80+
// ...and write out the result
81+
// TODO(directxman12): this should also accept a request for JSON content (via a accept header)
82+
_, forceVerbose := req.URL.Query()["verbose"]
83+
writeStatusesAsText(resp, parts, excluded, failed, forceVerbose)
6684
}
6785

68-
// NamedCheck returns a healthz checker for the given name and function.
69-
func NamedCheck(name string, check func(r *http.Request) error) Checker {
70-
return &healthzCheck{name, check}
71-
}
86+
func writeStatusesAsText(resp http.ResponseWriter, parts []checkStatus, excluded sets.String, failed, forceVerbose bool) {
87+
resp.Header().Set("Content-Type", "text/plain; charset=utf-8")
88+
resp.Header().Set("X-Content-Type-Options", "nosniff")
7289

73-
// InstallPathHandler registers handlers for health checking on
74-
// a specific path to mux. *All handlers* for the path must be
75-
// specified in exactly one call to InstallPathHandler. Calling
76-
// InstallPathHandler more than once for the same path and mux will
77-
// result in a panic.
78-
func InstallPathHandler(mux mux, path string, handler *Handler) {
79-
if len(handler.GetChecks()) == 0 {
80-
log.V(1).Info("No default health checks specified. Installing the ping handler.")
81-
handler.AddCheck(PingHealthz)
90+
// always write status code first
91+
if failed {
92+
resp.WriteHeader(http.StatusInternalServerError)
93+
} else {
94+
resp.WriteHeader(http.StatusOK)
8295
}
8396

84-
mux.Handle(path, handler.Build())
85-
for _, check := range handler.GetChecks() {
86-
log.V(1).Info("installing healthz checker", "checker", check.Name())
87-
mux.Handle(fmt.Sprintf("%s/%s", path, check.Name()), adaptCheckToHandler(check.Check))
97+
// shortcut for easy non-verbose success
98+
if !failed && !forceVerbose {
99+
fmt.Fprint(resp, "ok")
100+
return
88101
}
89-
}
90102

91-
// mux is an interface describing the methods InstallHandler requires.
92-
type mux interface {
93-
Handle(pattern string, handler http.Handler)
94-
}
103+
// we're always verbose on failure, so from this point on we're guaranteed to be verbose
104+
105+
for _, checkOut := range parts {
106+
switch {
107+
case checkOut.excluded:
108+
fmt.Fprintf(resp, "[+]%s excluded: ok\n", checkOut.name)
109+
case checkOut.healthy:
110+
fmt.Fprintf(resp, "[+]%s ok\n", checkOut.name)
111+
default:
112+
// don't include the error since this endpoint is public. If someone wants more detail
113+
// they should have explicit permission to the detailed checks.
114+
fmt.Fprintf(resp, "[-]%s failed: reason withheld\n", checkOut.name)
115+
}
116+
}
117+
118+
if excluded.Len() > 0 {
119+
fmt.Fprintf(resp, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(excluded.List()...))
120+
}
95121

96-
// healthzCheck implements HealthzChecker on an arbitrary name and check function.
97-
type healthzCheck struct {
98-
name string
99-
check func(r *http.Request) error
122+
if failed {
123+
log.Info("healthz check failed", "statuses", parts)
124+
fmt.Fprintf(resp, "healthz check failed\n")
125+
} else {
126+
fmt.Fprint(resp, "healthz check passed\n")
127+
}
100128
}
101129

102-
var _ Checker = &healthzCheck{}
130+
func (h *Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
131+
// clean up the request (duplicating the internal logic of http.ServeMux a bit)
132+
// clean up the path a bit
133+
reqPath := req.URL.Path
134+
if reqPath == "" || reqPath[0] != '/' {
135+
reqPath = "/" + reqPath
136+
}
137+
// path.Clean removes the trailing slash except for root for us
138+
// (which is fine, since we're only serving one layer of sub-paths)
139+
reqPath = path.Clean(reqPath)
140+
141+
// either serve the root endpoint...
142+
if reqPath == "/" {
143+
h.serveAggregated(resp, req)
144+
return
145+
}
146+
147+
// ...the default check (if nothing else is present)...
148+
if len(h.Checks) == 0 && reqPath[1:] == "ping" {
149+
CheckHandler{Checker: Ping}.ServeHTTP(resp, req)
150+
return
151+
}
152+
153+
// ...or an individual checker
154+
checkName := reqPath[1:] // ignore the leading slash
155+
checker, known := h.Checks[checkName]
156+
if !known {
157+
http.NotFoundHandler().ServeHTTP(resp, req)
158+
return
159+
}
103160

104-
func (c *healthzCheck) Name() string {
105-
return c.name
161+
CheckHandler{Checker: checker}.ServeHTTP(resp, req)
106162
}
107163

108-
func (c *healthzCheck) Check(r *http.Request) error {
109-
return c.check(r)
164+
// CheckHandler is an http.Handler that serves a health check endpoint at the root path,
165+
// based on its checker.
166+
type CheckHandler struct {
167+
Checker
168+
}
169+
170+
func (h CheckHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
171+
err := h.Checker(req)
172+
if err != nil {
173+
http.Error(resp, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError)
174+
} else {
175+
fmt.Fprint(resp, "ok")
176+
}
110177
}
111178

179+
// Checker knows how to perform a health check.
180+
type Checker func(req *http.Request) error
181+
182+
// Ping returns true automatically when checked
183+
var Ping Checker = func(_ *http.Request) error { return nil }
184+
112185
// getExcludedChecks extracts the health check names to be excluded from the query param
113186
func getExcludedChecks(r *http.Request) sets.String {
114187
checks, found := r.URL.Query()["exclude"]
@@ -118,71 +191,6 @@ func getExcludedChecks(r *http.Request) sets.String {
118191
return sets.NewString()
119192
}
120193

121-
// handleRootHealthz returns an http.HandlerFunc that serves the provided checks.
122-
func handleRootHealthz(checks ...Checker) http.HandlerFunc {
123-
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
124-
failed := false
125-
excluded := getExcludedChecks(r)
126-
var verboseOut bytes.Buffer
127-
for _, check := range checks {
128-
// no-op the check if we've specified we want to exclude the check
129-
if excluded.Has(check.Name()) {
130-
excluded.Delete(check.Name())
131-
fmt.Fprintf(&verboseOut, "[+]%s excluded: ok\n", check.Name())
132-
continue
133-
}
134-
if err := check.Check(r); err != nil {
135-
// don't include the error since this endpoint is public. If someone wants more detail
136-
// they should have explicit permission to the detailed checks.
137-
log.V(1).Info("healthz check failed", "checker", check.Name(), "error", err)
138-
fmt.Fprintf(&verboseOut, "[-]%s failed: reason withheld\n", check.Name())
139-
failed = true
140-
} else {
141-
fmt.Fprintf(&verboseOut, "[+]%s ok\n", check.Name())
142-
}
143-
}
144-
if excluded.Len() > 0 {
145-
fmt.Fprintf(&verboseOut, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(excluded.List()...))
146-
for _, c := range excluded.List() {
147-
log.Info("cannot exclude health check, no matches for it", "checker", c)
148-
}
149-
}
150-
// always be verbose on failure
151-
if failed {
152-
log.V(1).Info("healthz check failed", "message", verboseOut.String())
153-
http.Error(w, fmt.Sprintf("%shealthz check failed", verboseOut.String()), http.StatusInternalServerError)
154-
return
155-
}
156-
157-
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
158-
w.Header().Set("X-Content-Type-Options", "nosniff")
159-
if _, found := r.URL.Query()["verbose"]; !found {
160-
fmt.Fprint(w, "ok")
161-
return
162-
}
163-
164-
_, err := verboseOut.WriteTo(w)
165-
if err != nil {
166-
log.V(1).Info("healthz check failed", "message", verboseOut.String())
167-
http.Error(w, fmt.Sprintf("%shealthz check failed", verboseOut.String()), http.StatusInternalServerError)
168-
return
169-
}
170-
fmt.Fprint(w, "healthz check passed\n")
171-
})
172-
}
173-
174-
// adaptCheckToHandler returns an http.HandlerFunc that serves the provided checks.
175-
func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc {
176-
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
177-
err := c(r)
178-
if err != nil {
179-
http.Error(w, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError)
180-
} else {
181-
fmt.Fprint(w, "ok")
182-
}
183-
})
184-
}
185-
186194
// formatQuoted returns a formatted string of the health check names,
187195
// preserving the order passed in.
188196
func formatQuoted(names ...string) string {

0 commit comments

Comments
 (0)