Skip to content

Commit 5e9d74a

Browse files
jshum2479rjeberhard
jshum2479
authored and
rjeberhard
committed
Fix domain delayed response
1 parent 2340f3f commit 5e9d74a

File tree

7 files changed

+117
-29
lines changed

7 files changed

+117
-29
lines changed

‎operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -601,14 +601,20 @@ private void processServerPodWatch(V1Pod pod, String watchType) {
601601
// fall through
602602
case MODIFIED:
603603
boolean podPreviouslyEvicted = info.setServerPodFromEvent(serverName, pod, PodHelper::isEvicted);
604-
if (PodHelper.isEvicted(pod) && !podPreviouslyEvicted) {
604+
boolean isEvicted = PodHelper.isEvicted(pod);
605+
if (isEvicted && !podPreviouslyEvicted) {
605606
if (PodHelper.shouldRestartEvictedPod(pod)) {
606607
LOGGER.info(MessageKeys.POD_EVICTED, getPodName(pod), getPodStatusMessage(pod));
607608
createMakeRightOperation(info).interrupt().withExplicitRecheck().execute();
608609
} else {
609610
LOGGER.info(MessageKeys.POD_EVICTED_NO_RESTART, getPodName(pod), getPodStatusMessage(pod));
610611
}
611612
}
613+
boolean isReady = PodHelper.isReady(pod);
614+
boolean isLabeledForShutdown = PodHelper.isPodAlreadyAnnotatedForShutdown(pod);
615+
if ((isEvicted || isReady != isLabeledForShutdown || PodHelper.isFailed(pod)) && !PodHelper.isDeleting(pod)) {
616+
createMakeRightOperation(info).interrupt().withExplicitRecheck().execute();
617+
}
612618
boolean isUnschedulable = PodHelper.hasUnSchedulableCondition(pod);
613619
if (isUnschedulable) {
614620
LOGGER.info(POD_UNSCHEDULABLE, getPodName(pod), getUnSchedulableConditionMessage(pod));

‎operator/src/main/java/oracle/kubernetes/operator/LabelConstants.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ public interface LabelConstants {
2525
String MII_UPDATED_RESTART_REQUIRED_LABEL = "weblogic.configChangesPendingRestart";
2626
String INTROSPECTION_DOMAIN_SPEC_GENERATION = "weblogic.domainSpecGeneration";
2727
String TO_BE_ROLLED_LABEL = "weblogic.awaitingPodRoll";
28+
String TO_BE_SHUTDOWN_LABEL = "weblogic.awaitingShutdown";
2829
String DOMAIN_OBSERVED_GENERATION_LABEL = "weblogic.domainObservedGeneration";
2930
String CLUSTER_OBSERVED_GENERATION_LABEL = "weblogic.clusterObservedGeneration";
3031
String SERVICE_TYPE_LABEL = "serviceType";

‎operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import java.util.Collections;
88
import java.util.List;
99
import java.util.Map;
10+
import java.util.Objects;
1011
import java.util.Optional;
1112
import javax.annotation.Nonnull;
1213
import javax.annotation.Nullable;
@@ -48,6 +49,7 @@
4849
import oracle.kubernetes.weblogic.domain.model.ServerStatus;
4950
import oracle.kubernetes.weblogic.domain.model.Shutdown;
5051

52+
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
5153
import static oracle.kubernetes.operator.KubernetesConstants.EVICTED_REASON;
5254
import static oracle.kubernetes.operator.KubernetesConstants.POD_SCHEDULED;
5355
import static oracle.kubernetes.operator.KubernetesConstants.UNSCHEDULABLE_REASON;
@@ -56,6 +58,7 @@
5658
import static oracle.kubernetes.operator.ProcessingConstants.SERVERS_TO_ROLL;
5759
import static oracle.kubernetes.operator.WebLogicConstants.SHUTDOWN_STATE;
5860
import static oracle.kubernetes.operator.WebLogicConstants.UNKNOWN_STATE;
61+
import static oracle.kubernetes.operator.helpers.PodDisruptionBudgetHelper.getDomainUid;
5962

6063
@SuppressWarnings("ConstantConditions")
6164
public class PodHelper {
@@ -128,6 +131,7 @@ public static boolean isWaitingToRoll(V1Pod pod) {
128131
.orElse(false);
129132
}
130133

134+
131135
static boolean hasReadyServer(V1Pod pod) {
132136
return Optional.ofNullable(pod).map(PodHelper::hasReadyStatus).orElse(false);
133137
}
@@ -393,6 +397,89 @@ public static String getPodLabel(V1Pod pod, String labelName) {
393397
.orElse(null);
394398
}
395399

400+
401+
/**
402+
* get pod's annotation value for a annotation name.
403+
* @param pod pod
404+
* @param annotationName annotation name
405+
* @return annotation value
406+
*/
407+
public static String getPodAnnotation(V1Pod pod, String annotationName) {
408+
return Optional.ofNullable(pod)
409+
.map(V1Pod::getMetadata)
410+
.map(V1ObjectMeta::getAnnotations)
411+
.map(m -> m.get(annotationName))
412+
.orElse(null);
413+
}
414+
415+
private static boolean hasAnnotation(V1Pod pod, String annotation) {
416+
return Optional.ofNullable(pod).map(V1Pod::getMetadata).map(V1ObjectMeta::getAnnotations)
417+
.map(l -> l.containsKey(annotation)).orElse(false);
418+
}
419+
420+
private static Step patchPodAnnotation(V1Pod pod, String annotation, String value, Step next) {
421+
422+
if (!hasAnnotation(pod, annotation)) {
423+
JsonPatchBuilder patchBuilder = Json.createPatchBuilder();
424+
patchBuilder.add("/metadata/annotations/" + annotation, value);
425+
new CallBuilder()
426+
.patchPodAsync(pod.getMetadata().getName(), pod.getMetadata().getNamespace(),
427+
pod.getMetadata().getLabels().get(LabelConstants.DOMAINUID_LABEL),
428+
new V1Patch(patchBuilder.build().toString()),
429+
patchResponse(next));
430+
}
431+
return next;
432+
}
433+
434+
private static ResponseStep<V1Pod> patchResponse(Step next) {
435+
return new PatchPodResponseStep(next);
436+
}
437+
438+
private static class PatchPodResponseStep extends ResponseStep<V1Pod> {
439+
PatchPodResponseStep(Step next) {
440+
super(next);
441+
}
442+
443+
@Override
444+
public NextAction onSuccess(Packet packet, CallResponse<V1Pod> callResponse) {
445+
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
446+
V1Pod pod = callResponse.getResult();
447+
info.setServerPod(getPodServerName(pod), pod);
448+
return doNext(packet);
449+
}
450+
451+
@Override
452+
public NextAction onFailure(Packet packet, CallResponse<V1Pod> callResponse) {
453+
if (callResponse.getStatusCode() == HTTP_NOT_FOUND) {
454+
return doNext(packet);
455+
}
456+
return super.onFailure(packet, callResponse);
457+
}
458+
}
459+
460+
/**
461+
* Annotate pod as needing to shut down.
462+
* @param pod Pod
463+
* @param next Next step
464+
* @return Step that will check for existing annotation and add if it is missing
465+
*/
466+
public static Step annotatePodAsNeedingToShutdown(V1Pod pod, String value, Step next) {
467+
return patchPodAnnotation(pod, LabelConstants.TO_BE_SHUTDOWN_LABEL, value, next);
468+
}
469+
470+
/**
471+
* Check if the pod is already annotated for shut down.
472+
* @param pod Pod
473+
* @return true, if the pod is already annotated.
474+
*/
475+
public static boolean isPodAlreadyAnnotatedForShutdown(V1Pod pod) {
476+
return !Objects.isNull(getPodShutdownAnnotation(pod));
477+
}
478+
479+
public static String getPodShutdownAnnotation(V1Pod pod) {
480+
return getPodAnnotation(pod, LabelConstants.TO_BE_SHUTDOWN_LABEL);
481+
}
482+
396483
/**
397484
* Get the message from the pod's status.
398485
* @param pod pod

‎operator/src/main/java/oracle/kubernetes/operator/steps/ShutdownManagedServerStep.java

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import java.net.http.HttpRequest;
77
import java.net.http.HttpResponse;
8+
import java.time.OffsetDateTime;
89
import java.util.Collections;
910
import java.util.List;
1011
import java.util.Objects;
@@ -66,9 +67,9 @@ private ShutdownManagedServerStep(Step next, String serverName, V1Pod pod) {
6667
/**
6768
* Creates asynchronous {@link Step}.
6869
*
69-
* @param next Next processing step
70+
* @param next Next processing step
7071
* @param serverName name of server
71-
* @param pod server pod
72+
* @param pod server pod
7273
* @return asynchronous step
7374
*/
7475
static Step createShutdownManagedServerStep(Step next, String serverName, V1Pod pod) {
@@ -80,15 +81,15 @@ public NextAction apply(Packet packet) {
8081
LOGGER.fine(MessageKeys.BEGIN_SERVER_SHUTDOWN_REST, serverName);
8182
V1Service service = getDomainPresenceInfo(packet).getServerService(serverName);
8283

83-
if (service == null) {
84-
return doNext(packet);
85-
} else {
86-
return doNext(
87-
Step.chain(
88-
SecretHelper.createAuthorizationSourceStep(),
89-
new ShutdownManagedServerWithHttpStep(service, pod, getNext())),
90-
packet);
84+
String now = OffsetDateTime.now().toString();
85+
if (service == null || !PodHelper.isReady(pod) || PodHelper.isFailed(pod) || PodHelper.isWaitingToRoll(pod)) {
86+
return doNext(PodHelper.annotatePodAsNeedingToShutdown(pod, now, getNext()), packet);
9187
}
88+
return doNext(
89+
Step.chain(
90+
SecretHelper.createAuthorizationSourceStep(),
91+
PodHelper.annotatePodAsNeedingToShutdown(pod, now,
92+
new ShutdownManagedServerWithHttpStep(service, pod, getNext()))), packet);
9293
}
9394

9495
static final class ShutdownManagedServerProcessing extends HttpRequestProcessing {

‎operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -728,6 +728,7 @@ void afterMakeRightAndChangeServerToNever_serverPodsWaitForShutdownWithHttpToCom
728728
domainConfigurator.withDefaultServerStartPolicy(ServerStartPolicy.NEVER);
729729
DomainStatus status = newInfo.getDomain().getStatus();
730730
defineServerShutdownWithHttpOkResponse();
731+
makePodsReady();
731732
setAdminServerStatus(status, SUSPENDING_STATE);
732733
setManagedServerState(status, SUSPENDING_STATE);
733734

‎operator/src/test/java/oracle/kubernetes/operator/helpers/PodPresenceTest.java

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -517,23 +517,7 @@ void onModifyEventWithEvictedServerPod_cycleServerPod() {
517517
assertThat(numPodsDeleted, is(1));
518518
assertThat(createdPodNames, hasItem(SERVER));
519519
}
520-
521-
@Test
522-
void onModifyEventWithEvictedServerPod_dontCycleAlreadyEvictedServerPod() {
523-
V1Pod currentPod = withEvictedStatus(createServerPod());
524-
V1Pod modifiedPod = withEvictedStatus(createServerPod());
525-
List<String> createdPodNames = new ArrayList<>();
526-
testSupport.doOnCreate(POD, p -> recordPodCreation((V1Pod) p, createdPodNames));
527-
528-
info.setServerPod(SERVER, currentPod);
529-
Watch.Response<V1Pod> event = WatchEvent.createModifiedEvent(modifiedPod).toWatchResponse();
530-
531-
processor.dispatchPodWatch(event);
532-
533-
assertThat(numPodsDeleted, is(0));
534-
assertThat(createdPodNames, not(hasItem(SERVER)));
535-
}
536-
520+
537521
@Test
538522
void onModifyEventWithEvictedServerPod_notCycleServerPod_ifConfiguredNotTo() {
539523
TuningParametersStub.setParameter("restartEvictedPods", "false");

‎operator/src/test/java/oracle/kubernetes/operator/steps/ShutdownManagedServerStepTest.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818
import io.kubernetes.client.openapi.models.V1EnvVar;
1919
import io.kubernetes.client.openapi.models.V1ObjectMeta;
2020
import io.kubernetes.client.openapi.models.V1Pod;
21+
import io.kubernetes.client.openapi.models.V1PodCondition;
2122
import io.kubernetes.client.openapi.models.V1PodSpec;
23+
import io.kubernetes.client.openapi.models.V1PodStatus;
2224
import io.kubernetes.client.openapi.models.V1Service;
2325
import oracle.kubernetes.operator.DomainProcessorTestSetup;
2426
import oracle.kubernetes.operator.KubernetesConstants;
@@ -198,7 +200,13 @@ private V1Pod createPod(String serverName) {
198200
List<V1EnvVar> env = addShutdownEnvVars();
199201
List<V1Container> containers = addEnvToWLSContainer(env);
200202
V1PodSpec podSpec = new V1PodSpec().containers(containers);
201-
return new V1Pod().metadata(createManagedPodMetadata(serverName)).spec(podSpec);
203+
return new V1Pod().metadata(createManagedPodMetadata(serverName)).spec(podSpec).status(createPodReadyStatus());
204+
}
205+
206+
private V1PodStatus createPodReadyStatus() {
207+
return new V1PodStatus()
208+
.phase("Running")
209+
.addConditionsItem(new V1PodCondition().status("True").type("Ready"));
202210
}
203211

204212
@Nonnull

0 commit comments

Comments
 (0)