Skip to content

Commit ebfac21

Browse files
ndimidukndimiduk
committed
HBASE-26648 Improve fidelity of RegionLocator spans
Signed-off-by: Duo Zhang <zhangduo@apache.org>
1 parent 7d5bf1c commit ebfac21

5 files changed

Lines changed: 137 additions & 42 deletions

File tree

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/trace/ConnectionSpanBuilder.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,7 @@ public <T> ConnectionSpanBuilder addAttribute(final AttributeKey<T> key, T value
6767
public Span build() {
6868
final SpanBuilder builder = TraceUtil.getGlobalTracer()
6969
.spanBuilder(name)
70-
// TODO: what about clients embedded in Master/RegionServer/Gateways/&c?
71-
.setSpanKind(SpanKind.CLIENT);
70+
.setSpanKind(SpanKind.INTERNAL);
7271
attributes.forEach((k, v) -> builder.setAttribute((AttributeKey<? super Object>) k, v));
7372
return builder.startSpan();
7473
}

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/trace/TableSpanBuilder.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,7 @@ public TableSpanBuilder setTableName(final TableName tableName) {
6464
public Span build() {
6565
final SpanBuilder builder = TraceUtil.getGlobalTracer()
6666
.spanBuilder(name)
67-
// TODO: what about clients embedded in Master/RegionServer/Gateways/&c?
68-
.setSpanKind(SpanKind.CLIENT);
67+
.setSpanKind(SpanKind.INTERNAL);
6968
attributes.forEach((k, v) -> builder.setAttribute((AttributeKey<? super Object>) k, v));
7069
return builder.startSpan();
7170
}

‎hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTracing.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ public void testClearCache() {
132132
SpanData span = waitSpan("AsyncRegionLocator.clearCache");
133133
assertThat(span, allOf(
134134
hasStatusWithCode(StatusCode.OK),
135-
hasKind(SpanKind.CLIENT),
135+
hasKind(SpanKind.INTERNAL),
136136
buildConnectionAttributesMatcher(conn)));
137137
}
138138

@@ -144,7 +144,7 @@ public void testClearCacheServerName() {
144144
SpanData span = waitSpan("AsyncRegionLocator.clearCache");
145145
assertThat(span, allOf(
146146
hasStatusWithCode(StatusCode.OK),
147-
hasKind(SpanKind.CLIENT),
147+
hasKind(SpanKind.INTERNAL),
148148
buildConnectionAttributesMatcher(conn),
149149
hasAttributes(containsEntry("db.hbase.server.name", sn.getServerName()))));
150150
}
@@ -155,7 +155,7 @@ public void testClearCacheTableName() {
155155
SpanData span = waitSpan("AsyncRegionLocator.clearCache");
156156
assertThat(span, allOf(
157157
hasStatusWithCode(StatusCode.OK),
158-
hasKind(SpanKind.CLIENT),
158+
hasKind(SpanKind.INTERNAL),
159159
buildConnectionAttributesMatcher(conn),
160160
buildTableAttributesMatcher(TableName.META_TABLE_NAME)));
161161
}
@@ -167,7 +167,7 @@ public void testGetRegionLocation() {
167167
SpanData span = waitSpan("AsyncRegionLocator.getRegionLocation");
168168
assertThat(span, allOf(
169169
hasStatusWithCode(StatusCode.OK),
170-
hasKind(SpanKind.CLIENT),
170+
hasKind(SpanKind.INTERNAL),
171171
buildConnectionAttributesMatcher(conn),
172172
buildTableAttributesMatcher(TableName.META_TABLE_NAME),
173173
hasAttributes(
@@ -186,7 +186,7 @@ public void testGetRegionLocations() {
186186
.toArray(String[]::new);
187187
assertThat(span, allOf(
188188
hasStatusWithCode(StatusCode.OK),
189-
hasKind(SpanKind.CLIENT),
189+
hasKind(SpanKind.INTERNAL),
190190
buildConnectionAttributesMatcher(conn),
191191
buildTableAttributesMatcher(TableName.META_TABLE_NAME),
192192
hasAttributes(

‎hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import java.util.Random;
5050
import java.util.Set;
5151
import java.util.TreeSet;
52+
import java.util.concurrent.ExecutionException;
5253
import java.util.concurrent.ThreadLocalRandom;
5354
import java.util.concurrent.TimeUnit;
5455
import java.util.concurrent.atomic.AtomicReference;
@@ -61,6 +62,7 @@
6162
import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
6263
import org.apache.hadoop.hbase.Waiter.Predicate;
6364
import org.apache.hadoop.hbase.client.Admin;
65+
import org.apache.hadoop.hbase.client.AsyncAdmin;
6466
import org.apache.hadoop.hbase.client.AsyncClusterConnection;
6567
import org.apache.hadoop.hbase.client.BufferedMutator;
6668
import org.apache.hadoop.hbase.client.ClusterConnectionFactory;
@@ -1534,6 +1536,16 @@ public static void setReplicas(Admin admin, TableName table, int replicaCount)
15341536
admin.modifyTable(desc);
15351537
}
15361538

1539+
/**
1540+
* Set the number of Region replicas.
1541+
*/
1542+
public static void setReplicas(AsyncAdmin admin, TableName table, int replicaCount)
1543+
throws ExecutionException, IOException, InterruptedException {
1544+
TableDescriptor desc = TableDescriptorBuilder.newBuilder(admin.getDescriptor(table).get())
1545+
.setRegionReplication(replicaCount).build();
1546+
admin.modifyTable(desc).get();
1547+
}
1548+
15371549
/**
15381550
* Drop an existing table
15391551
* @param tableName existing table

‎hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMetaRegionLocator.java‎

Lines changed: 118 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -18,68 +18,153 @@
1818
package org.apache.hadoop.hbase.client;
1919

2020
import static org.apache.hadoop.hbase.client.RegionReplicaTestHelper.testLocator;
21-
21+
import static org.apache.hadoop.hbase.client.trace.hamcrest.SpanDataMatchers.hasEnded;
22+
import static org.apache.hadoop.hbase.client.trace.hamcrest.SpanDataMatchers.hasKind;
23+
import static org.apache.hadoop.hbase.client.trace.hamcrest.SpanDataMatchers.hasName;
24+
import static org.apache.hadoop.hbase.client.trace.hamcrest.SpanDataMatchers.hasParentSpanId;
25+
import static org.hamcrest.MatcherAssert.assertThat;
26+
import static org.hamcrest.Matchers.allOf;
27+
import static org.hamcrest.Matchers.endsWith;
28+
import static org.hamcrest.Matchers.hasItem;
29+
import io.opentelemetry.api.trace.SpanKind;
30+
import io.opentelemetry.sdk.trace.data.SpanData;
31+
import java.util.List;
32+
import java.util.concurrent.TimeUnit;
33+
import org.apache.hadoop.conf.Configuration;
34+
import org.apache.hadoop.hbase.ConnectionRule;
2235
import org.apache.hadoop.hbase.HBaseClassTestRule;
2336
import org.apache.hadoop.hbase.HBaseTestingUtil;
2437
import org.apache.hadoop.hbase.HRegionLocation;
38+
import org.apache.hadoop.hbase.MatcherPredicate;
39+
import org.apache.hadoop.hbase.MiniClusterRule;
2540
import org.apache.hadoop.hbase.RegionLocations;
41+
import org.apache.hadoop.hbase.StartTestingClusterOption;
2642
import org.apache.hadoop.hbase.TableName;
43+
import org.apache.hadoop.hbase.Waiter;
2744
import org.apache.hadoop.hbase.client.RegionReplicaTestHelper.Locator;
45+
import org.apache.hadoop.hbase.client.trace.StringTraceRenderer;
2846
import org.apache.hadoop.hbase.testclassification.ClientTests;
2947
import org.apache.hadoop.hbase.testclassification.MediumTests;
30-
import org.junit.AfterClass;
31-
import org.junit.BeforeClass;
48+
import org.apache.hadoop.hbase.trace.OpenTelemetryClassRule;
49+
import org.apache.hadoop.hbase.trace.OpenTelemetryTestRule;
50+
import org.apache.hadoop.hbase.trace.TraceUtil;
51+
import org.hamcrest.Matcher;
3252
import org.junit.ClassRule;
53+
import org.junit.Rule;
3354
import org.junit.Test;
3455
import org.junit.experimental.categories.Category;
35-
36-
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
56+
import org.junit.rules.ExternalResource;
57+
import org.junit.rules.RuleChain;
58+
import org.junit.rules.TestRule;
59+
import org.slf4j.Logger;
60+
import org.slf4j.LoggerFactory;
3761

3862
@Category({ MediumTests.class, ClientTests.class })
3963
public class TestAsyncMetaRegionLocator {
64+
private static final Logger logger = LoggerFactory.getLogger(TestAsyncMetaRegionLocator.class);
4065

4166
@ClassRule
4267
public static final HBaseClassTestRule CLASS_RULE =
4368
HBaseClassTestRule.forClass(TestAsyncMetaRegionLocator.class);
4469

45-
private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
70+
private static final OpenTelemetryClassRule otelClassRule = OpenTelemetryClassRule.create();
71+
private static final MiniClusterRule miniClusterRule = MiniClusterRule.newBuilder()
72+
.setMiniClusterOption(StartTestingClusterOption.builder()
73+
.numWorkers(3)
74+
.build())
75+
.build();
76+
private static final ConnectionRule connectionRule =
77+
ConnectionRule.createAsyncConnectionRule(miniClusterRule::createAsyncConnection);
4678

47-
private static ConnectionRegistry REGISTRY;
79+
private static final class Setup extends ExternalResource {
80+
private ConnectionRegistry registry;
4881

49-
private static AsyncMetaRegionLocator LOCATOR;
82+
@Override
83+
protected void before() throws Throwable {
84+
final AsyncAdmin admin = connectionRule.getAsyncConnection().getAdmin();
85+
TEST_UTIL = miniClusterRule.getTestingUtility();
86+
HBaseTestingUtil.setReplicas(admin, TableName.META_TABLE_NAME, 3);
87+
TEST_UTIL.waitUntilNoRegionsInTransition();
88+
registry = ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration());
89+
RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, registry);
90+
admin.balancerSwitch(false).get();
91+
LOCATOR = new AsyncMetaRegionLocator(registry);
92+
}
5093

51-
@BeforeClass
52-
public static void setUp() throws Exception {
53-
TEST_UTIL.startMiniCluster(3);
54-
HBaseTestingUtil.setReplicas(TEST_UTIL.getAdmin(), TableName.META_TABLE_NAME, 3);
55-
TEST_UTIL.waitUntilNoRegionsInTransition();
56-
REGISTRY = ConnectionRegistryFactory.getRegistry(TEST_UTIL.getConfiguration());
57-
RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(TEST_UTIL, REGISTRY);
58-
TEST_UTIL.getAdmin().balancerSwitch(false, true);
59-
LOCATOR = new AsyncMetaRegionLocator(REGISTRY);
94+
@Override
95+
protected void after() {
96+
registry.close();
97+
}
6098
}
6199

62-
@AfterClass
63-
public static void tearDown() throws Exception {
64-
Closeables.close(REGISTRY, true);
65-
TEST_UTIL.shutdownMiniCluster();
66-
}
100+
@ClassRule
101+
public static final TestRule classRule = RuleChain.outerRule(otelClassRule)
102+
.around(miniClusterRule)
103+
.around(connectionRule)
104+
.around(new Setup());
105+
106+
private static HBaseTestingUtil TEST_UTIL;
107+
private static AsyncMetaRegionLocator LOCATOR;
108+
109+
@Rule
110+
public final OpenTelemetryTestRule otelTestRule = new OpenTelemetryTestRule(otelClassRule);
67111

68112
@Test
69113
public void test() throws Exception {
70-
testLocator(TEST_UTIL, TableName.META_TABLE_NAME, new Locator() {
114+
TraceUtil.trace(() -> {
115+
try {
116+
testLocator(miniClusterRule.getTestingUtility(), TableName.META_TABLE_NAME, new Locator() {
117+
@Override
118+
public void updateCachedLocationOnError(HRegionLocation loc, Throwable error) {
119+
LOCATOR.updateCachedLocationOnError(loc, error);
120+
}
71121

72-
@Override
73-
public void updateCachedLocationOnError(HRegionLocation loc, Throwable error)
74-
throws Exception {
75-
LOCATOR.updateCachedLocationOnError(loc, error);
122+
@Override
123+
public RegionLocations getRegionLocations(
124+
TableName tableName,
125+
int replicaId,
126+
boolean reload
127+
) throws Exception {
128+
return LOCATOR.getRegionLocations(replicaId, reload).get();
129+
}
130+
});
131+
} catch (Exception e) {
132+
throw new RuntimeException(e);
76133
}
134+
}, "test");
77135

78-
@Override
79-
public RegionLocations getRegionLocations(TableName tableName, int replicaId, boolean reload)
80-
throws Exception {
81-
return LOCATOR.getRegionLocations(replicaId, reload).get();
82-
}
83-
});
136+
final Configuration conf = TEST_UTIL.getConfiguration();
137+
final Matcher<SpanData> parentSpanMatcher = allOf(hasName("test"), hasEnded());
138+
Waiter.waitFor(conf, TimeUnit.SECONDS.toMillis(5), new MatcherPredicate<>(
139+
otelClassRule::getSpans, hasItem(parentSpanMatcher)));
140+
final List<SpanData> spans = otelClassRule.getSpans();
141+
if (logger.isDebugEnabled()) {
142+
StringTraceRenderer renderer = new StringTraceRenderer(spans);
143+
renderer.render(logger::debug);
144+
}
145+
146+
assertThat(spans, hasItem(parentSpanMatcher));
147+
final SpanData parentSpan = spans.stream()
148+
.filter(parentSpanMatcher::matches)
149+
.findAny()
150+
.orElseThrow(AssertionError::new);
151+
152+
final Matcher<SpanData> registryGetMetaRegionLocationsMatcher = allOf(
153+
hasName(endsWith("ConnectionRegistry.getMetaRegionLocations")),
154+
hasParentSpanId(parentSpan),
155+
hasKind(SpanKind.INTERNAL),
156+
hasEnded());
157+
assertThat(spans, hasItem(registryGetMetaRegionLocationsMatcher));
158+
final SpanData registry_getMetaRegionLocationsSpan = spans.stream()
159+
.filter(registryGetMetaRegionLocationsMatcher::matches)
160+
.findAny()
161+
.orElseThrow(AssertionError::new);
162+
163+
final Matcher<SpanData> clientGetMetaRegionLocationsMatcher = allOf(
164+
hasName(endsWith("ClientMetaService/GetMetaRegionLocations")),
165+
hasParentSpanId(registry_getMetaRegionLocationsSpan),
166+
hasKind(SpanKind.CLIENT),
167+
hasEnded());
168+
assertThat(spans, hasItem(clientGetMetaRegionLocationsMatcher));
84169
}
85170
}

0 commit comments

Comments
 (0)