diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java index d8e6b1e529..3bc63e3e15 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java @@ -27,6 +27,7 @@ import edu.umd.cs.findbugs.annotations.CheckForNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import hudson.Functions; import hudson.model.TaskListener; import hudson.slaves.JNLPLauncher; import hudson.slaves.SlaveComputer; @@ -112,7 +113,6 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) { String cloudName = node.getCloudName(); final PodTemplate template = node.getTemplate(); - TaskListener runListener = TaskListener.NULL; try { KubernetesCloud cloud = node.getKubernetesCloud(); KubernetesClient client = cloud.connect(); @@ -128,8 +128,6 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) { node.setNamespace(namespace); - runListener = template.getListener(); - LOGGER.log(FINE, () -> "Creating Pod: " + cloudName + " " + namespace + "/" + podName); try { pod = client.pods().inNamespace(namespace).create(pod); @@ -138,16 +136,16 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) { int httpCode = e.getCode(); if (400 <= httpCode && httpCode < 500) { // 4xx if (httpCode == 403 && e.getMessage().contains("is forbidden: exceeded quota")) { - runListener.getLogger().printf("WARNING: Unable to create pod: %s %s/%s because kubernetes resource quota exceeded. %n%s%nRetrying...%n%n", + node.getRunListener().getLogger().printf("WARNING: Unable to create pod: %s %s/%s because kubernetes resource quota exceeded. %n%s%nRetrying...%n%n", cloudName, namespace, pod.getMetadata().getName(), e.getMessage()); } else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfilled on resourcequotas")) { // See: https://github.com/kubernetes/kubernetes/issues/67761 ; A retry usually works. - runListener.getLogger().printf("WARNING: Unable to create pod: %s %s/%s because kubernetes resource quota update conflict. %n%s%nRetrying...%n%n", + node.getRunListener().getLogger().printf("WARNING: Unable to create pod: %s %s/%s because kubernetes resource quota update conflict. %n%s%nRetrying...%n%n", cloudName, namespace, pod.getMetadata().getName(), e.getMessage()); } else { - runListener.getLogger().printf("ERROR: Unable to create pod %s %s/%s.%n%s%n", cloudName, namespace, pod.getMetadata().getName(), e.getMessage()); + node.getRunListener().getLogger().printf("ERROR: Unable to create pod %s %s/%s.%n%s%n", cloudName, namespace, pod.getMetadata().getName(), e.getMessage()); PodUtils.cancelQueueItemFor(pod, e.getMessage()); } } else if (500 <= httpCode && httpCode < 600) { // 5xx @@ -161,7 +159,7 @@ else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfill listener.getLogger().printf("Created Pod: %s %s/%s%n", cloudName, namespace, podName); Metrics.metricRegistry().counter(MetricNames.PODS_CREATED).inc(); - runListener.getLogger().printf("Created Pod: %s %s/%s%n", cloudName, namespace, podName); + node.getRunListener().getLogger().printf("Created Pod: %s %s/%s%n", cloudName, namespace, podName); kubernetesComputer.setLaunching(true); ObjectMeta podMetadata = pod.getMetadata(); @@ -253,6 +251,7 @@ else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfill Metrics.metricRegistry().counter(MetricNames.PODS_LAUNCHED).inc(); } catch (Throwable ex) { setProblem(ex); + Functions.printStackTrace(ex, node.getRunListener().error("Failed to launch " + node.getPodName())); LOGGER.log(Level.WARNING, String.format("Error in provisioning; agent=%s, template=%s", node, template), ex); LOGGER.log(Level.FINER, "Removing Jenkins node: {0}", node.getNodeName()); try { diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java index 90a5d12369..b5ca892148 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java @@ -1,10 +1,7 @@ package org.csanchez.jenkins.plugins.kubernetes; -import edu.umd.cs.findbugs.annotations.NonNull; -import io.fabric8.kubernetes.api.model.StatusDetails; import java.io.IOException; import java.util.HashSet; -import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -29,6 +26,7 @@ import org.csanchez.jenkins.plugins.kubernetes.pod.retention.PodRetention; import org.jenkinsci.plugins.durabletask.executors.OnceRetentionStrategy; import org.jenkinsci.plugins.kubernetes.auth.KubernetesAuthException; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; import org.jvnet.localizer.ResourceBundleHolder; import org.kohsuke.stapler.DataBoundConstructor; @@ -108,6 +106,42 @@ public PodTemplate getTemplateOrNull() { return template; } + /** + * Makes a best effort to find the build log corresponding to this agent. + */ + @NonNull + public TaskListener getRunListener() { + PodTemplate podTemplate = getTemplateOrNull(); + if (podTemplate != null) { + TaskListener listener = podTemplate.getListenerOrNull(); + if (listener != null) { + return listener; + } + } + Computer c = toComputer(); + if (c != null) { + for (Executor executor : c.getExecutors()) { + Queue.Executable executable = executor.getCurrentExecutable(); + // If this executor hosts a PlaceholderExecutable, send to the owning build log. + if (executable != null) { + Queue.Executable parentExecutable = executable.getParentExecutable(); + if (parentExecutable instanceof FlowExecutionOwner.Executable) { + FlowExecutionOwner flowExecutionOwner = ((FlowExecutionOwner.Executable) parentExecutable).asFlowExecutionOwner(); + if (flowExecutionOwner != null) { + try { + return flowExecutionOwner.getListener(); + } catch (IOException x) { + LOGGER.log(Level.WARNING, null, x); + } + } + } + } + // TODO handle freestyle and similar if executable instanceof Run, by capturing a TaskListener from RunListener.onStarted + } + } + return TaskListener.NULL; + } + /** * @deprecated Use {@link Builder} instead. */ diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java index 2c37c9d245..096a2b83db 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java @@ -818,11 +818,18 @@ public void setPodRetention(PodRetention podRetention) { this.podRetention = PodRetention.getPodTemplateDefault().equals(podRetention) ? null : podRetention; } + /** @see KubernetesSlave#getRunListener */ @NonNull public TaskListener getListener() { return listener == null ? TaskListener.NULL : listener; } + /** @see KubernetesSlave#getRunListener */ + @CheckForNull + public TaskListener getListenerOrNull() { + return listener; + } + public void setListener(@CheckForNull TaskListener listener) { this.listener = listener; } diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/pod/retention/Reaper.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/pod/retention/Reaper.java index 33bae4575b..a562896c7b 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/pod/retention/Reaper.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/pod/retention/Reaper.java @@ -387,10 +387,7 @@ public void onEvent(@NonNull Watcher.Action action, @NonNull KubernetesSlave nod String ns = pod.getMetadata().getNamespace(); String name = pod.getMetadata().getName(); LOGGER.info(() -> ns + "/" + name + " was just deleted, so removing corresponding Jenkins agent"); - PodTemplate template = node.getTemplateOrNull(); - if (template != null) { - template.getListener().getLogger().printf("Pod %s/%s was just deleted%n", ns, name); - } + node.getRunListener().getLogger().printf("Pod %s/%s was just deleted%n", ns, name); Jenkins.get().removeNode(node); } } @@ -407,8 +404,7 @@ public void onEvent(@NonNull Watcher.Action action, @NonNull KubernetesSlave nod if (!terminatedContainers.isEmpty()) { String ns = pod.getMetadata().getNamespace(); String name = pod.getMetadata().getName(); - PodTemplate template = node.getTemplateOrNull(); - TaskListener runListener = template != null ? template.getListener() : TaskListener.NULL; + TaskListener runListener = node.getRunListener(); terminatedContainers.forEach(c -> { ContainerStateTerminated t = c.getState().getTerminated(); LOGGER.info(() -> ns + "/" + name + " Container " + c.getName() + " was just terminated, so removing the corresponding Jenkins agent"); @@ -434,8 +430,7 @@ public void onEvent(@NonNull Watcher.Action action, @NonNull KubernetesSlave nod if ("Failed".equals(pod.getStatus().getPhase())) { String ns = pod.getMetadata().getNamespace(); String name = pod.getMetadata().getName(); - PodTemplate template = node.getTemplateOrNull(); - TaskListener runListener = template != null ? template.getListener() : TaskListener.NULL; + TaskListener runListener = node.getRunListener(); String reason = pod.getStatus().getReason(); LOGGER.info(() -> ns + "/" + name + " Pod just failed. Removing the corresponding Jenkins agent. Reason: " + reason + ", Message: " + pod.getStatus().getMessage()); runListener.getLogger().printf("%s/%s Pod just failed (Reason: %s, Message: %s)%n", ns, name, reason, pod.getStatus().getMessage()); @@ -476,12 +471,7 @@ public void onEvent(@NonNull Watcher.Action action, @NonNull KubernetesSlave nod if (backOffContainers.isEmpty()) { return; } - backOffContainers.forEach(cs -> { - PodTemplate template = node.getTemplateOrNull(); - if (template != null) { - template.getListener().error("Unable to pull Docker image \"" + cs.getImage() + "\". Check if image tag name is spelled correctly."); - } - }); + backOffContainers.forEach(cs -> node.getRunListener().error("Unable to pull Docker image \"" + cs.getImage() + "\". Check if image tag name is spelled correctly.")); terminationReasons.add("ImagePullBackOff"); PodUtils.cancelQueueItemFor(pod, "ImagePullBackOff"); node.terminate(); diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java index d1f9674e05..f9ffdaced2 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java @@ -515,6 +515,17 @@ public void podDeadlineExceeded() throws Exception { r.waitForMessage("Pod just failed (Reason: DeadlineExceeded, Message: Pod was active on the node longer than the specified deadline)", b); } + @Test + public void podDeadlineExceededGlobalTemplate() throws Exception { + PodTemplate podTemplate = new PodTemplate("podDeadlineExceededGlobalTemplate"); + podTemplate.setLabel("podDeadlineExceededGlobalTemplate"); + podTemplate.setActiveDeadlineSeconds(30); + cloud.addTemplate(podTemplate); + r.assertBuildStatus(Result.ABORTED, r.waitForCompletion(b)); + r.waitForMessage("Pod just failed (Reason: DeadlineExceeded, Message: Pod was active on the node longer than the specified deadline)", b); + r.waitForMessage("---Logs---", b); + } + @Test public void interruptedPod() throws Exception { r.waitForMessage("starting to sleep", b); diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pod/retention/ReaperTest.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pod/retention/ReaperTest.java index 44ea24e5de..37100f2eb5 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pod/retention/ReaperTest.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pod/retention/ReaperTest.java @@ -41,6 +41,7 @@ import hudson.Extension; import hudson.model.TaskListener; import hudson.slaves.ComputerLauncher; +import hudson.util.StreamTaskListener; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.server.mock.KubernetesServer; @@ -649,6 +650,7 @@ private KubernetesSlave addNode(KubernetesCloud cld, String podName, String node when(node.getNumExecutors()).thenReturn(1); PodTemplate podTemplate = new PodTemplate(); when(node.getTemplate()).thenReturn(podTemplate); + when(node.getRunListener()).thenReturn(StreamTaskListener.fromStderr()); ComputerLauncher launcher = mock(ComputerLauncher.class); when(node.getLauncher()).thenReturn(launcher); j.jenkins.addNode(node); diff --git a/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/podDeadlineExceededGlobalTemplate.groovy b/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/podDeadlineExceededGlobalTemplate.groovy new file mode 100644 index 0000000000..7314cdca47 --- /dev/null +++ b/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/podDeadlineExceededGlobalTemplate.groovy @@ -0,0 +1,3 @@ +node('podDeadlineExceededGlobalTemplate') { + sh 'sleep 120' +} diff --git a/test-in-k8s.sh b/test-in-k8s.sh index 5dbff43564..b3f22f74a4 100755 --- a/test-in-k8s.sh +++ b/test-in-k8s.sh @@ -14,8 +14,11 @@ tcp_port=$((2001 + port_offset)) kubectl delete --ignore-not-found --now pod jenkins sed "s/@HTTP_PORT@/$http_port/g; s/@TCP_PORT@/$tcp_port/g" < test-in-k8s.yaml | kubectl apply -f - kubectl wait --for=condition=Ready --timeout=15m pod/jenkins -# Copy temporary split files -tar cf - "$WORKSPACE_TMP" | kubectl exec -i jenkins -- tar xf - +if [[ -v WORKSPACE_TMP ]] +then + # Copy temporary split files + tar cf - "$WORKSPACE_TMP" | kubectl exec -i jenkins -- tar xf - +fi # Copy plugin files kubectl exec jenkins -- mkdir /checkout tar cf - pom.xml .mvn src | kubectl exec -i jenkins -- tar xf - -C /checkout