diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java index 997e3dfa357a..9d4ea87e0ec1 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseJupiterExtension.java @@ -37,7 +37,9 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.yetus.audience.InterfaceAudience; import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ExtensionContext.Store; import org.junit.jupiter.api.extension.InvocationInterceptor; @@ -60,14 +62,18 @@ * the tag. *

* It also controls the timeout for the whole test class running, while the timeout annotation in - * JUnit5 can only enforce the timeout for each test method. + * JUnit5 can only enforce the timeout for each test method. When a test is timed out, a thread dump + * will be printed to log output. *

- * Finally, it also forbid System.exit call in tests. TODO: need to find a new way as - * SecurityManager has been removed since Java 21. + * It also implements resource check for each test method, using the {@link ResourceChecker} class. + *

+ * Finally, it also forbid System.exit call in tests.
+ * TODO: need to find a new way as SecurityManager was deprecated in Java 17 and permanently + * disabled since Java 24. */ @InterfaceAudience.Private -public class HBaseJupiterExtension - implements InvocationInterceptor, BeforeAllCallback, AfterAllCallback { +public class HBaseJupiterExtension implements InvocationInterceptor, BeforeAllCallback, + AfterAllCallback, BeforeEachCallback, AfterEachCallback { private static final Logger LOG = LoggerFactory.getLogger(HBaseJupiterExtension.class); @@ -84,6 +90,8 @@ public class HBaseJupiterExtension private static final String DEADLINE = "deadline"; + private static final String RESOURCE_CHECK = "rc"; + private Duration pickTimeout(ExtensionContext ctx) { Set timeoutTags = TAG_TO_TIMEOUT.keySet(); Set timeoutTag = Sets.intersection(timeoutTags, ctx.getTags()); @@ -130,7 +138,8 @@ public void afterAll(ExtensionContext ctx) throws Exception { System.setSecurityManager(null); } - private T runWithTimeout(Invocation invocation, ExtensionContext ctx) throws Throwable { + private T runWithTimeout(Invocation invocation, ExtensionContext ctx, String name) + throws Throwable { Store store = ctx.getStore(NAMESPACE); ExecutorService executor = store.get(EXECUTOR, ExecutorService.class); if (executor == null) { @@ -139,12 +148,12 @@ private T runWithTimeout(Invocation invocation, ExtensionContext ctx) thr Instant deadline = store.get(DEADLINE, Instant.class); Instant now = Instant.now(); if (!now.isBefore(deadline)) { - fail("Test " + ctx.getDisplayName() + " timed out, deadline is " + deadline); + fail("Test " + name + " timed out, deadline is " + deadline); return null; } Duration remaining = Duration.between(now, deadline); - LOG.info("remaining timeout for {} is {}", ctx.getDisplayName(), remaining); + LOG.info("remaining timeout for {} is {}", name, remaining); Future future = executor.submit(() -> { try { return invocation.proceed(); @@ -157,14 +166,13 @@ private T runWithTimeout(Invocation invocation, ExtensionContext ctx) thr return future.get(remaining.toNanos(), TimeUnit.NANOSECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - fail("Test " + ctx.getDisplayName() + " interrupted"); + fail("Test " + name + " interrupted"); return null; } catch (ExecutionException e) { throw ExceptionUtils.throwAsUncheckedException(e.getCause()); } catch (TimeoutException e) { printThreadDump(); - throw new JUnitException( - "Test " + ctx.getDisplayName() + " timed out, deadline is " + deadline, e); + throw new JUnitException("Test " + name + " timed out, deadline is " + deadline, e); } } @@ -177,41 +185,62 @@ private void printThreadDump() { public void interceptBeforeAllMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { - runWithTimeout(invocation, extensionContext); + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".beforeAll"); } @Override public void interceptBeforeEachMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { - runWithTimeout(invocation, extensionContext); + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".beforeEach"); } @Override public void interceptTestMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { - runWithTimeout(invocation, extensionContext); + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName()); } @Override public void interceptAfterEachMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { - runWithTimeout(invocation, extensionContext); + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".afterEach"); } @Override public void interceptAfterAllMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { - runWithTimeout(invocation, extensionContext); + runWithTimeout(invocation, extensionContext, extensionContext.getDisplayName() + ".afterAll"); } @Override public T interceptTestClassConstructor(Invocation invocation, ReflectiveInvocationContext> invocationContext, ExtensionContext extensionContext) throws Throwable { - return runWithTimeout(invocation, extensionContext); + return runWithTimeout(invocation, extensionContext, + extensionContext.getDisplayName() + ".constructor"); + } + + // below are for implementing resource checker around test method + + @Override + public void beforeEach(ExtensionContext ctx) throws Exception { + ResourceChecker rc = new ResourceChecker(ctx.getDisplayName()); + JUnitResourceCheckers.addResourceAnalyzer(rc); + Store store = ctx.getStore(NAMESPACE); + store.put(RESOURCE_CHECK, rc); + rc.start(); + } + + @Override + public void afterEach(ExtensionContext ctx) throws Exception { + Store store = ctx.getStore(NAMESPACE); + ResourceChecker rc = store.remove(RESOURCE_CHECK, ResourceChecker.class); + if (rc != null) { + rc.end(); + } } } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/JUnitResourceCheckers.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/JUnitResourceCheckers.java new file mode 100644 index 000000000000..aee49dc2c60f --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/JUnitResourceCheckers.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.hbase.ResourceChecker.Phase; +import org.apache.hadoop.hbase.util.JVM; + +/** + * ResourceCheckers when running JUnit tests. + */ +public final class JUnitResourceCheckers { + + private JUnitResourceCheckers() { + } + + private static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + private Set initialThreadNames = new HashSet<>(); + private List stringsToLog = null; + + @Override + public int getVal(Phase phase) { + Map stackTraces = Thread.getAllStackTraces(); + if (phase == Phase.INITIAL) { + stringsToLog = null; + for (Thread t : stackTraces.keySet()) { + initialThreadNames.add(t.getName()); + } + } else if (phase == Phase.END) { + if (stackTraces.size() > initialThreadNames.size()) { + stringsToLog = new ArrayList<>(); + for (Thread t : stackTraces.keySet()) { + if (!initialThreadNames.contains(t.getName())) { + stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); + StackTraceElement[] stackElements = stackTraces.get(t); + for (StackTraceElement ele : stackElements) { + stringsToLog.add("\t" + ele + "\n"); + } + } + } + } + } + return stackTraces.size(); + } + + @Override + public int getMax() { + return 500; + } + + @Override + public List getStringsToLog() { + return stringsToLog; + } + } + + private static class OpenFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + JVM jvm = new JVM(); + return (int) jvm.getOpenFileDescriptorCount(); + } + + @Override + public int getMax() { + return 1024; + } + } + + private static class MaxFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + JVM jvm = new JVM(); + return (int) jvm.getMaxFileDescriptorCount(); + } + } + + private static class SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + return (int) (new JVM().getSystemLoadAverage() * 100); + } + } + + private static class ProcessCountResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + return new JVM().getNumberOfRunningProcess(); + } + } + + private static class AvailableMemoryMBResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + @Override + public int getVal(Phase phase) { + if (!JVM.isUnix()) { + return 0; + } + return (int) (new JVM().getFreeMemory() / (1024L * 1024L)); + } + } + + public static void addResourceAnalyzer(ResourceChecker rc) { + rc.addResourceAnalyzer(new ThreadResourceAnalyzer()); + rc.addResourceAnalyzer(new OpenFileDescriptorResourceAnalyzer()); + rc.addResourceAnalyzer(new MaxFileDescriptorResourceAnalyzer()); + rc.addResourceAnalyzer(new SystemLoadAverageResourceAnalyzer()); + rc.addResourceAnalyzer(new ProcessCountResourceAnalyzer()); + rc.addResourceAnalyzer(new AvailableMemoryMBResourceAnalyzer()); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java index 4dfce7f536b5..2a796cc40774 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java @@ -17,14 +17,8 @@ */ package org.apache.hadoop.hbase; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.hadoop.hbase.ResourceChecker.Phase; -import org.apache.hadoop.hbase.util.JVM; import org.junit.runner.notification.RunListener; /** @@ -38,104 +32,8 @@ * When surefire forkMode=once/always/perthread, this code is executed on the forked process. */ public class ResourceCheckerJUnitListener extends RunListener { - private Map rcs = new ConcurrentHashMap<>(); - static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - private static Set initialThreadNames = new HashSet<>(); - private static List stringsToLog = null; - - @Override - public int getVal(Phase phase) { - Map stackTraces = Thread.getAllStackTraces(); - if (phase == Phase.INITIAL) { - stringsToLog = null; - for (Thread t : stackTraces.keySet()) { - initialThreadNames.add(t.getName()); - } - } else if (phase == Phase.END) { - if (stackTraces.size() > initialThreadNames.size()) { - stringsToLog = new ArrayList<>(); - for (Thread t : stackTraces.keySet()) { - if (!initialThreadNames.contains(t.getName())) { - stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); - StackTraceElement[] stackElements = stackTraces.get(t); - for (StackTraceElement ele : stackElements) { - stringsToLog.add("\t" + ele + "\n"); - } - } - } - } - } - return stackTraces.size(); - } - - @Override - public int getMax() { - return 500; - } - - @Override - public List getStringsToLog() { - return stringsToLog; - } - } - - static class OpenFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - JVM jvm = new JVM(); - return (int) jvm.getOpenFileDescriptorCount(); - } - - @Override - public int getMax() { - return 1024; - } - } - - static class MaxFileDescriptorResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - JVM jvm = new JVM(); - return (int) jvm.getMaxFileDescriptorCount(); - } - } - - static class SystemLoadAverageResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - return (int) (new JVM().getSystemLoadAverage() * 100); - } - } - - static class ProcessCountResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - return new JVM().getNumberOfRunningProcess(); - } - } - - static class AvailableMemoryMBResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { - @Override - public int getVal(Phase phase) { - if (!JVM.isUnix()) { - return 0; - } - return (int) (new JVM().getFreeMemory() / (1024L * 1024L)); - } - } + private final Map rcs = new ConcurrentHashMap<>(); /** * To be implemented by sub classes if they want to add specific ResourceAnalyzer. @@ -145,17 +43,9 @@ protected void addResourceAnalyzer(ResourceChecker rc) { private void start(String testName) { ResourceChecker rc = new ResourceChecker(testName); - rc.addResourceAnalyzer(new ThreadResourceAnalyzer()); - rc.addResourceAnalyzer(new OpenFileDescriptorResourceAnalyzer()); - rc.addResourceAnalyzer(new MaxFileDescriptorResourceAnalyzer()); - rc.addResourceAnalyzer(new SystemLoadAverageResourceAnalyzer()); - rc.addResourceAnalyzer(new ProcessCountResourceAnalyzer()); - rc.addResourceAnalyzer(new AvailableMemoryMBResourceAnalyzer()); - + JUnitResourceCheckers.addResourceAnalyzer(rc); addResourceAnalyzer(rc); - rcs.put(testName, rc); - rc.start(); }