@@ -2,14 +2,14 @@ package tg
22
33import (
44 "context"
5- "fmt"
65 "time"
76
87 "github.com/aws/aws-sdk-go/service/elbv2"
98 "github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/albctx"
109 "github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/aws"
1110 "github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/ingress/annotations/healthcheck"
1211 "github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/ingress/backend"
12+ backendpkg "github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/ingress/backend"
1313 "github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/ingress/controller/store"
1414 "github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/k8s"
1515 api "k8s.io/api/core/v1"
@@ -75,9 +75,11 @@ type targetHealthController struct {
7575
7676// SyncTargetsForReconciliation starts a go routine for reconciling pod condition statuses for the given targets in the background until they are healthy in the target group
7777func (c * targetHealthController ) SyncTargetsForReconciliation (ctx context.Context , t * Targets , desiredTargets []* elbv2.TargetDescription ) error {
78- conditionType := podConditionTypeForIngressBackend (t .Ingress , t .Backend )
79-
80- targetsToReconcile , err := c .filterTargetsNeedingReconciliation (conditionType , t , desiredTargets )
78+ readinessConditionTypes := []api.PodConditionType {
79+ backendpkg .PodReadinessGateConditionType (t .Ingress , t .Backend ),
80+ backendpkg .AnyLBTGReadyConditionType ,
81+ }
82+ targetsToReconcile , err := c .filterTargetsNeedingReconciliation (t , desiredTargets , readinessConditionTypes ... )
8183 if err != nil {
8284 return err
8385 }
@@ -102,7 +104,7 @@ func (c *targetHealthController) SyncTargetsForReconciliation(ctx context.Contex
102104 c .tgWatches [t .TgArn ] = tgWatch
103105
104106 // start watching target health in target group and updating pod condition status
105- go c .reconcilePodConditionsLoop (ctx , t .TgArn , conditionType , tgWatch )
107+ go c .reconcilePodConditionsLoop (ctx , t .TgArn , tgWatch , readinessConditionTypes ... )
106108 }
107109 tgWatch .interval <- c .ingressTargetHealthReconciliationInterval (t .Backend .ServiceName , t .Ingress )
108110 tgWatch .targetsToReconcile <- targetsToReconcile
@@ -126,16 +128,25 @@ func (c *targetHealthController) RemovePodConditions(ctx context.Context, t *Tar
126128 return err
127129 }
128130
129- conditionType := podConditionTypeForIngressBackend (t .Ingress , t .Backend )
130-
131+ readinessConditionTypes := []api.PodConditionType {
132+ backendpkg .PodReadinessGateConditionType (t .Ingress , t .Backend ),
133+ backendpkg .AnyLBTGReadyConditionType ,
134+ }
131135 for _ , pod := range pods {
132- if pod != nil {
133- if i , cond := podConditionForReadinessGate (pod , conditionType ); cond != nil {
136+ if pod == nil {
137+ continue
138+ }
139+ needUpdates := false
140+ for _ , conditionType := range readinessConditionTypes {
141+ i , cond := backendpkg .PodConditionForReadinessGate (pod , conditionType )
142+ if cond != nil {
134143 pod .Status .Conditions = append (pod .Status .Conditions [:i ], pod .Status .Conditions [i + 1 :]... )
135- err := c .client .Status ().Update (ctx , pod )
136- if err != nil && ! k8serrors .IsNotFound (err ) {
137- return err
138- }
144+ needUpdates = true
145+ }
146+ }
147+ if needUpdates {
148+ if err := c .client .Status ().Update (ctx , pod ); err != nil && ! k8serrors .IsNotFound (err ) {
149+ return err
139150 }
140151 }
141152 }
@@ -144,7 +155,7 @@ func (c *targetHealthController) RemovePodConditions(ctx context.Context, t *Tar
144155}
145156
146157// Background loop which keeps reconciling pod condition statuses for the given target groups until the given context is cancelled.
147- func (c * targetHealthController ) reconcilePodConditionsLoop (ctx context.Context , tgArn string , conditionType api. PodConditionType , tgWatch * targetGroupWatch ) {
158+ func (c * targetHealthController ) reconcilePodConditionsLoop (ctx context.Context , tgArn string , tgWatch * targetGroupWatch , readinessConditionTypes ... api. PodConditionType ) {
148159 logger := albctx .GetLogger (ctx )
149160 logger .Infof ("Starting reconciliation of pod condition status for target group: %v" , tgArn )
150161
@@ -162,7 +173,7 @@ func (c *targetHealthController) reconcilePodConditionsLoop(ctx context.Context,
162173 case targetsToReconcile = <- tgWatch .targetsToReconcile : // update targets
163174
164175 case <- reconcile :
165- notReadyTargets , err := c .reconcilePodConditions (ctx , tgArn , conditionType , tgWatch .ingress , tgWatch .backend , targetsToReconcile )
176+ notReadyTargets , err := c .reconcilePodConditions (ctx , tgArn , tgWatch .ingress , tgWatch .backend , targetsToReconcile , readinessConditionTypes ... )
166177 if err == nil {
167178 targetsToReconcile = notReadyTargets
168179 } else {
@@ -175,11 +186,10 @@ func (c *targetHealthController) reconcilePodConditionsLoop(ctx context.Context,
175186 return
176187 }
177188 }
178-
179189}
180190
181191// For each given pod, checks for the health status of the corresponding target in the target group and adds/updates a pod condition that can be used for pod readiness gates.
182- func (c * targetHealthController ) reconcilePodConditions (ctx context.Context , tgArn string , conditionType api. PodConditionType , ingress * extensions.Ingress , backend * extensions.IngressBackend , targetsToReconcile []* elbv2.TargetDescription ) ([]* elbv2.TargetDescription , error ) {
192+ func (c * targetHealthController ) reconcilePodConditions (ctx context.Context , tgArn string , ingress * extensions.Ingress , backend * extensions.IngressBackend , targetsToReconcile []* elbv2.TargetDescription , readinessConditionTypes ... api. PodConditionType ) ([]* elbv2.TargetDescription , error ) {
183193 var notReadyTargets []* elbv2.TargetDescription
184194
185195 in := & elbv2.DescribeTargetHealthInput {
@@ -206,85 +216,64 @@ func (c *targetHealthController) reconcilePodConditions(ctx context.Context, tgA
206216 continue
207217 }
208218 targetHealth , ok := targetsHealth [pod .Status .PodIP ]
209- if ok && podHasReadinessGate (pod , conditionType ) {
210- if aws .StringValue (targetHealth .State ) != elbv2 .TargetHealthStateEnumHealthy {
211- notReadyTargets = append (notReadyTargets , target )
212- }
213- if err := c .reconcilePodCondition (ctx , conditionType , pod , targetHealth , true ); err != nil {
214- return notReadyTargets , err
215- }
219+ if ! ok {
220+ continue
216221 }
217- }
218- return notReadyTargets , nil
219- }
220-
221- // Creates or updates the condition status for the given pod with the given target health.
222- func (c * targetHealthController ) reconcilePodCondition (ctx context.Context , conditionType api.PodConditionType , pod * api.Pod , targetHealth * elbv2.TargetHealth , updateTimes bool ) error {
223- conditionStatus := podConditionStatusFromTargetHealth (targetHealth )
224222
225- // check if condition already exists
226- now := metav1 .Now ()
227- i , cond := podConditionForReadinessGate (pod , conditionType )
228- if cond == nil {
229- // new condition
230- targetHealthCondition := api.PodCondition {
231- Type : conditionType ,
232- Status : conditionStatus ,
233- Reason : aws .StringValue (targetHealth .Reason ),
234- Message : aws .StringValue (targetHealth .Description ),
223+ needsContinueProbe := false
224+ needsUpdate := false
225+ for _ , conditionType := range readinessConditionTypes {
226+ if backendpkg .PodHasReadinessGate (pod , conditionType ) {
227+ if readinessConditionBecomeHealthy := updatePodReadinessCondition (pod , targetHealth , conditionType ); ! readinessConditionBecomeHealthy {
228+ needsContinueProbe = true
229+ }
230+ needsUpdate = true
231+ }
235232 }
236- if updateTimes {
237- targetHealthCondition .LastProbeTime = now
238- targetHealthCondition .LastTransitionTime = now
233+ if needsContinueProbe {
234+ notReadyTargets = append (notReadyTargets , target )
239235 }
240- pod .Status .Conditions = append (pod .Status .Conditions , targetHealthCondition )
241- } else {
242- // update condition
243- if updateTimes {
244- cond .LastProbeTime = now
245- if cond .Status != conditionStatus {
246- cond .LastTransitionTime = now
236+ if needsUpdate {
237+ if err := c .client .Status ().Update (ctx , pod ); err != nil && ! k8serrors .IsNotFound (err ) {
238+ return nil , err
247239 }
248240 }
249- cond .Status = conditionStatus
250- cond .Reason = aws .StringValue (targetHealth .Reason )
251- cond .Message = aws .StringValue (targetHealth .Description )
252- pod .Status .Conditions [i ] = * cond
253241 }
254-
255- // pod will always be updated (at least to update `LastProbeTime`);
256- // this will trigger another invocation of `Reconcile`, which will remove this pod from the list of pods to reconcile if its health status is ok
257- err := c .client .Status ().Update (ctx , pod )
258- if err != nil {
259- return err
260- }
261-
262- return nil
242+ return notReadyTargets , nil
263243}
264244
265245// From the given targets, only returns the ones that have a readiness gate for the given ingress/service and whose pod conditions actually need to be reconciled.
266- func (c * targetHealthController ) filterTargetsNeedingReconciliation (conditionType api.PodConditionType , t * Targets , desiredTargets []* elbv2.TargetDescription ) ([]* elbv2.TargetDescription , error ) {
267- targetsToReconcile := []* elbv2.TargetDescription {}
246+ func (c * targetHealthController ) filterTargetsNeedingReconciliation (t * Targets , desiredTargets []* elbv2.TargetDescription , readinessConditionTypes ... api.PodConditionType ) ([]* elbv2.TargetDescription , error ) {
268247 if len (desiredTargets ) == 0 {
269- return targetsToReconcile , nil
248+ return nil , nil
270249 }
271-
272250 // find the pods that correspond to the targets
273251 pods , err := c .endpointResolver .ReverseResolve (t .Ingress , t .Backend , desiredTargets )
274252 if err != nil {
275- return targetsToReconcile , err
253+ return nil , err
276254 }
277255
256+ var targetsToReconcile []* elbv2.TargetDescription
278257 // filter out targets whose pods don't have the `readinessGate` for this target group or whose pod condition status is already `True`
279258 for i , target := range desiredTargets {
280259 pod := pods [i ]
281- if pod != nil && podHasReadinessGate (pod , conditionType ) {
282- if _ , cond := podConditionForReadinessGate (pod , conditionType ); cond == nil || cond .Status != api .ConditionTrue {
283- targetsToReconcile = append (targetsToReconcile , target )
260+ if pod == nil {
261+ continue
262+ }
263+
264+ foundAnyUnreadyReadinessGate := false
265+ for _ , conditionType := range readinessConditionTypes {
266+ if backendpkg .PodHasReadinessGate (pod , conditionType ) {
267+ if _ , cond := backendpkg .PodConditionForReadinessGate (pod , conditionType ); cond == nil || cond .Status != api .ConditionTrue {
268+ foundAnyUnreadyReadinessGate = true
269+ break
270+ }
284271 }
285272 }
273+ if foundAnyUnreadyReadinessGate {
274+ targetsToReconcile = append (targetsToReconcile , target )
275+ }
286276 }
287-
288277 return targetsToReconcile , nil
289278}
290279
@@ -301,33 +290,37 @@ func (c *targetHealthController) ingressTargetHealthReconciliationInterval(servi
301290 return healthcheck .DefaultIntervalSeconds
302291}
303292
304- // PodConditionTypeForIngressBackend returns the PodConditionType that is associated with the given ingress and backend
305- func podConditionTypeForIngressBackend (ingress * extensions.Ingress , backend * extensions.IngressBackend ) api.PodConditionType {
306- return api .PodConditionType (fmt .Sprintf (
307- "target-health.alb.ingress.k8s.aws/%s_%s_%s" ,
308- ingress .Name ,
309- backend .ServiceName ,
310- backend .ServicePort .String (),
311- ))
312- }
293+ // updatePodReadinessCondition will creates or updates the condition status for the given pod with the given target health
294+ // returns whether pod readinessGate becomes healthy
295+ func updatePodReadinessCondition (pod * api.Pod , targetHealth * elbv2.TargetHealth , conditionType api.PodConditionType ) bool {
296+ conditionStatus := podConditionStatusFromTargetHealth (targetHealth )
313297
314- // PodHasReadinessGate returns true if the given pod has a readinessGate with the given conditionType
315- func podHasReadinessGate (pod * api.Pod , conditionType api.PodConditionType ) bool {
316- for _ , rg := range pod .Spec .ReadinessGates {
317- if rg .ConditionType == conditionType {
318- return true
298+ // check if condition already exists
299+ now := metav1 .Now ()
300+ i , cond := backendpkg .PodConditionForReadinessGate (pod , conditionType )
301+ if cond == nil {
302+ // new condition
303+ targetHealthCondition := api.PodCondition {
304+ Type : conditionType ,
305+ Status : conditionStatus ,
306+ Reason : aws .StringValue (targetHealth .Reason ),
307+ Message : aws .StringValue (targetHealth .Description ),
319308 }
320- }
321- return false
322- }
323-
324- func podConditionForReadinessGate (pod * api.Pod , conditionType api.PodConditionType ) (int , * api.PodCondition ) {
325- for i , condition := range pod .Status .Conditions {
326- if condition .Type == conditionType {
327- return i , & condition
309+ targetHealthCondition .LastProbeTime = now
310+ targetHealthCondition .LastTransitionTime = now
311+ pod .Status .Conditions = append (pod .Status .Conditions , targetHealthCondition )
312+ } else {
313+ cond .LastProbeTime = now
314+ if cond .Status != conditionStatus {
315+ cond .LastTransitionTime = now
328316 }
317+ cond .Status = conditionStatus
318+ cond .Reason = aws .StringValue (targetHealth .Reason )
319+ cond .Message = aws .StringValue (targetHealth .Description )
320+ pod .Status .Conditions [i ] = * cond
329321 }
330- return - 1 , nil
322+
323+ return conditionStatus == api .ConditionTrue
331324}
332325
333326func podConditionStatusFromTargetHealth (targetHealth * elbv2.TargetHealth ) api.ConditionStatus {
0 commit comments