feat: New liveness probe to check for deadlocked threads (micronaut-p…

…rojects#11388)

core/src/main/java/io/micronaut/core/util/NativeImageUtils.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2017-2024 original authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.micronaut.core.util;
+
+import io.micronaut.core.annotation.Internal;
+
+/**
+ * Utility class to retrieve information about the context in which code gets executed.
+ * Partial fork of {@code org.graalvm.nativeimage.ImageInfo} to avoid a dependency on {@code org.graalvm.sdk:nativeimage}.
+ *
+ * @since 4.8.0
+ */
+@Internal
+public final class NativeImageUtils {
+    /**
+     * Holds the string that is the name of the system property providing information about the
+     * context in which code is currently executing. If the property returns the string given by
+     * {@link #PROPERTY_IMAGE_CODE_VALUE_BUILDTIME} the code is executing in the context of image
+     * building (e.g. in a static initializer of a class that will be contained in the image). If
+     * the property returns the string given by {@link #PROPERTY_IMAGE_CODE_VALUE_RUNTIME} the code
+     * is executing at image runtime. Otherwise, the property is not set.
+     */
+    public static final String PROPERTY_IMAGE_CODE_KEY = "org.graalvm.nativeimage.imagecode";
+
+    /**
+     * Holds the string that will be returned by the system property for
+     * {@link NativeImageUtils#PROPERTY_IMAGE_CODE_KEY} if code is executing in the context of image
+     * building (e.g. in a static initializer of class that will be contained in the image).
+     */
+    public static final String PROPERTY_IMAGE_CODE_VALUE_BUILDTIME = "buildtime";
+
+    /**
+     * Holds the string that will be returned by the system property for
+     * {@link NativeImageUtils#PROPERTY_IMAGE_CODE_KEY} if code is executing at image runtime.
+     */
+    public static final String PROPERTY_IMAGE_CODE_VALUE_RUNTIME = "runtime";
+
+    private NativeImageUtils() {
+    }
+
+    /**
+     * Returns true if (at the time of the call) code is executing in the context of image building
+     * or during image runtime, else false. This method will be const-folded so that it can be used
+     * to hide parts of an application that only work when running on the JVM. For example:
+     * {@code if (!ImageInfo.inImageCode()) { ... JVM specific code here ... }}
+     * @return true if (at the time of the call) code is executing in the context of image building or during image runtime, else false
+     */
+    public static boolean inImageCode() {
+        return inImageBuildtimeCode() || inImageRuntimeCode();
+    }
+
+    /**
+     * Returns true if (at the time of the call) code is executing at image runtime. This method
+     * will be const-folded. It can be used to hide parts of an application that only work when
+     * running as native image.
+     * @return true if (at the time of the call) code is executing at image runtime.
+     */
+    public static boolean inImageRuntimeCode() {
+        return PROPERTY_IMAGE_CODE_VALUE_RUNTIME.equals(System.getProperty(PROPERTY_IMAGE_CODE_KEY));
+    }
+
+    /**
+     * Returns true if (at the time of the call) code is executing in the context of image building
+     * (e.g. in a static initializer of class that will be contained in the image).
+     * @return true if (at the time of the call) code is executing in the context of image building
+     */
+    public static boolean inImageBuildtimeCode() {
+        return PROPERTY_IMAGE_CODE_VALUE_BUILDTIME.equals(System.getProperty(PROPERTY_IMAGE_CODE_KEY));
+    }
+}

core/src/test/java/io/micronaut/core/util/NativeImageUtilsInNativeImageTest.java

@@ -0,0 +1,17 @@
+package io.micronaut.core.util;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledInNativeImage;
+import org.junit.jupiter.api.condition.EnabledInNativeImage;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class NativeImageUtilsInNativeImageTest {
+
+    @EnabledInNativeImage
+    @Test
+    void testInImageCode() {
+        assertTrue(NativeImageUtils.inImageCode());
+    }
+}

core/src/test/java/io/micronaut/core/util/NativeImageUtilsTest.java

@@ -0,0 +1,17 @@
+package io.micronaut.core.util;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledInNativeImage;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+class NativeImageUtilsTest {
+
+    @DisabledInNativeImage
+    @Test
+    void testInImageCode() {
+        assertFalse(NativeImageUtils.inImageCode());
+        assertFalse(NativeImageUtils.inImageRuntimeCode());
+        assertFalse(NativeImageUtils.inImageBuildtimeCode());
+    }
+}

inject/src/main/java/io/micronaut/context/condition/NotInNativeImage.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2017-2024 original authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.micronaut.context.condition;
+
+import io.micronaut.core.util.NativeImageUtils;
+
+/**
+ * Condition to hide parts of an application that only work when running on the JVM.
+ * Internal implementation is identical to {@code if (!ImageInfo.inImageCode()).
+ * @author Sergio del Amo
+ * @since 4.8.0
+ */
+public class NotInNativeImage implements Condition {
+    @Override
+    public boolean matches(ConditionContext context) {
+        return !NativeImageUtils.inImageCode();
+    }
+}

management/build.gradle.kts

@@ -4,6 +4,7 @@ plugins {
 
 dependencies {
     annotationProcessor(projects.micronautInjectJava)
+    testAnnotationProcessor(projects.micronautInjectJava)
     annotationProcessor(projects.micronautGraal)
 
     api(projects.micronautRouter)
@@ -19,7 +20,6 @@ dependencies {
         exclude(group = "io.micronaut")
     }
     compileOnly(projects.micronautHttpClientCore)
-
     implementation(libs.managed.reactor)
 
     testImplementation(projects.micronautHttpClient)
@@ -38,5 +38,5 @@ dependencies {
 
     compileOnly(libs.logback.classic)
     compileOnly(libs.log4j)
-
+    testImplementation(libs.awaitility)
 }

management/src/main/java/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicator.java

@@ -0,0 +1,139 @@
+/*
+ * Copyright 2017-2024 original authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.micronaut.management.health.indicator.threads;
+
+import io.micronaut.context.annotation.Requires;
+import io.micronaut.context.condition.NotInNativeImage;
+import io.micronaut.core.annotation.Internal;
+import io.micronaut.core.util.StringUtils;
+import io.micronaut.health.HealthStatus;
+import io.micronaut.management.endpoint.health.HealthEndpoint;
+import io.micronaut.management.health.indicator.AbstractHealthIndicator;
+import io.micronaut.management.health.indicator.HealthIndicator;
+import io.micronaut.management.health.indicator.annotation.Liveness;
+import jakarta.inject.Singleton;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MonitorInfo;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * <p>A {@link HealthIndicator} that uses the {@link ThreadMXBean} to check for deadlocked threads.
+ * Returns {@link HealthStatus#DOWN} if any are found and their {@link ThreadInfo} in the details.</p>
+ *
+ * @author Andreas Brenk
+ * @since 4.8.0
+ */
+@Singleton
+@Liveness
+@Requires(condition = NotInNativeImage.class)
+@Requires(property = HealthEndpoint.PREFIX + ".deadlocked-threads.enabled", notEquals = StringUtils.FALSE)
+@Requires(beans = HealthEndpoint.class)
+@Internal
+class DeadlockedThreadsHealthIndicator extends AbstractHealthIndicator {
+
+    private static final String NAME = "deadlockedThreads";
+    private static final String KEY_THREAD_ID = "threadId";
+    private static final String KEY_THREAD_NAME = "threadName";
+    private static final String KEY_THREAD_STATE = "threadState";
+    private static final String KEY_DAEMON = "daemon";
+    private static final String KEY_PRIORITY = "priority";
+    private static final String KEY_SUSPENDED = "suspended";
+    private static final String KEY_IN_NATIVE = "inNative";
+    private static final String KEY_LOCK_NAME = "lockName";
+    private static final String KEY_LOCK_OWNER_NAME = "lockOwnerName";
+    private static final String KEY_LOCK_OWNER_ID = "lockOwnerId";
+    private static final String KEY_LOCKED_SYNCHRONIZERS = "lockedSynchronizers";
+    private static final String KEY_STACK_TRACE = "stackTrace";
+
+    @Override
+    protected Object getHealthInformation() {
+            ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+            long[] deadlockedThreads = threadMXBean.findDeadlockedThreads();
+            if (deadlockedThreads == null) {
+                this.healthStatus = HealthStatus.UP;
+                return null;
+            }
+            this.healthStatus = HealthStatus.DOWN;
+            return Arrays.stream(threadMXBean.getThreadInfo(deadlockedThreads, true, true, Integer.MAX_VALUE))
+                    .map(DeadlockedThreadsHealthIndicator::getDetails)
+                    .toList();
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    private static Map<String, Object> getDetails(ThreadInfo threadInfo) {
+        Map<String, Object> details = new LinkedHashMap<>();
+        details.put(KEY_THREAD_ID, String.valueOf(threadInfo.getThreadId()));
+        details.put(KEY_THREAD_NAME, threadInfo.getThreadName());
+        details.put(KEY_THREAD_STATE, threadInfo.getThreadState().name());
+        details.put(KEY_DAEMON, String.valueOf(threadInfo.isDaemon()));
+        details.put(KEY_PRIORITY, String.valueOf(threadInfo.getPriority()));
+        details.put(KEY_SUSPENDED, String.valueOf(threadInfo.isSuspended()));
+        details.put(KEY_IN_NATIVE, String.valueOf(threadInfo.isInNative()));
+        details.put(KEY_LOCK_NAME, threadInfo.getLockName());
+        details.put(KEY_LOCK_OWNER_NAME, threadInfo.getLockOwnerName());
+        details.put(KEY_LOCK_OWNER_ID, String.valueOf(threadInfo.getLockOwnerId()));
+        details.put(KEY_LOCKED_SYNCHRONIZERS, Arrays.stream(threadInfo.getLockedSynchronizers()).map(String::valueOf).toList());
+        details.put(KEY_STACK_TRACE, formatStackTrace(threadInfo));
+        return details;
+    }
+
+    private static String formatStackTrace(ThreadInfo threadInfo) {
+        StringBuilder sb = new StringBuilder();
+
+        int i = 0;
+        StackTraceElement[] stackTrace = threadInfo.getStackTrace();
+        for (; i < stackTrace.length; i++) {
+            StackTraceElement ste = stackTrace[i];
+            sb.append(ste.toString());
+            sb.append('\n');
+
+            if (i == 0 && threadInfo.getLockInfo() != null) {
+                switch (threadInfo.getThreadState()) {
+                    case BLOCKED:
+                        sb.append("-  blocked on ");
+                        sb.append(threadInfo.getLockInfo());
+                        sb.append('\n');
+                        break;
+                    case WAITING, TIMED_WAITING:
+                        sb.append("-  waiting on ");
+                        sb.append(threadInfo.getLockInfo());
+                        sb.append('\n');
+                        break;
+                    default:
+                }
+            }
+
+            for (MonitorInfo mi : threadInfo.getLockedMonitors()) {
+                if (mi.getLockedStackDepth() == i) {
+                    sb.append("-  locked ");
+                    sb.append(mi);
+                    sb.append('\n');
+                }
+            }
+        }
+
+        return sb.toString();
+    }
+}

management/src/main/java/io/micronaut/management/health/indicator/threads/package-info.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2017-2024 original authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Thread deadlock indicator.
+ *
+ * @author Andreas Brenk
+ * @since 4.8.0
+ */
+package io.micronaut.management.health.indicator.threads;

management/src/test/groovy/io/micronaut/management/health/aggregator/HealthAggregatorSpec.groovy

@@ -43,24 +43,26 @@ class HealthAggregatorSpec extends Specification {
                 assert appender.events.isEmpty()
                 break
             case Level.DEBUG:
-                assert appender.events.size() == 7
+                assert appender.events.size() == 8
                 assert appender.events[0] == 'Health result for compositeDiscoveryClient(): status UP'
-                assert appender.events[1] == 'Health result for diskSpace: status UP'
-                assert appender.events[2] == 'Health result for jdbc: status UP'
-                assert appender.events[3] == 'Health result for jdbc:h2:mem:oneDb: status UP'
-                assert appender.events[4] == 'Health result for liveness: status UP'
-                assert appender.events[5] == 'Health result for readiness: status UP'
-                assert appender.events[6] == 'Health result for service: status UP'
+                assert appender.events[1] == 'Health result for deadlockedThreads: status UP'
+                assert appender.events[2] == 'Health result for diskSpace: status UP'
+                assert appender.events[3] == 'Health result for jdbc: status UP'
+                assert appender.events[4] == 'Health result for jdbc:h2:mem:oneDb: status UP'
+                assert appender.events[5] == 'Health result for liveness: status UP'
+                assert appender.events[6] == 'Health result for readiness: status UP'
+                assert appender.events[7] == 'Health result for service: status UP'
                 break
             case Level.TRACE:
-                assert appender.events.size() == 7
+                assert appender.events.size() == 8
                 assert appender.events[0].contains('Health result for compositeDiscoveryClient(): status UP, details {')
-                assert appender.events[1].contains('Health result for diskSpace: status UP, details {')
-                assert appender.events[2].contains('Health result for jdbc: status UP, details {')
-                assert appender.events[3].contains('Health result for jdbc:h2:mem:oneDb: status UP, details {')
-                assert appender.events[4] == 'Health result for liveness: status UP, details {}'
-                assert appender.events[5] == 'Health result for readiness: status UP, details {}'
-                assert appender.events[6] == 'Health result for service: status UP, details {}'
+                assert appender.events[1] == 'Health result for deadlockedThreads: status UP, details {}'
+                assert appender.events[2].contains('Health result for diskSpace: status UP, details {')
+                assert appender.events[3].contains('Health result for jdbc: status UP, details {')
+                assert appender.events[4].contains('Health result for jdbc:h2:mem:oneDb: status UP, details {')
+                assert appender.events[5] == 'Health result for liveness: status UP, details {}'
+                assert appender.events[6] == 'Health result for readiness: status UP, details {}'
+                assert appender.events[7] == 'Health result for service: status UP, details {}'
                 break
         }