Skip to content

Commit 70a561e

Browse files
apurtellapurtell
committed
HBASE-27194 Add test coverage for SimpleRpcServer (#4616)
Add test coverage for SimpleRpcServer. Improve the way we test both SimpleRpcServer and NettyRpcServer. Use LoadTestKVGenerator to generate random values with varying sizes between 1000 bytes and 1M bytes, and also to verify them when reading the values back. Add secure test coverage for both SimpleRpcServer and NettyRpcServer. Signed-off-by: Duo Zhang <zhangduo@apache.org> Signed-off-by: Viraj Jasani <vjasani@apache.org>
1 parent 4e19892 commit 70a561e

4 files changed

Lines changed: 351 additions & 38 deletions

File tree

‎hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyRpcServer.java‎

Lines changed: 40 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -17,31 +17,32 @@
1717
*/
1818
package org.apache.hadoop.hbase.ipc;
1919

20-
import static org.junit.Assert.assertEquals;
20+
import static org.junit.Assert.assertNotNull;
2121
import static org.junit.Assert.assertTrue;
2222

23-
import java.util.ArrayList;
2423
import java.util.Arrays;
2524
import java.util.Collection;
26-
import java.util.List;
2725
import org.apache.hadoop.hbase.HBaseClassTestRule;
2826
import org.apache.hadoop.hbase.HBaseTestingUtility;
2927
import org.apache.hadoop.hbase.TableName;
28+
import org.apache.hadoop.hbase.TableNameTestRule;
29+
import org.apache.hadoop.hbase.client.Get;
3030
import org.apache.hadoop.hbase.client.Put;
3131
import org.apache.hadoop.hbase.client.Result;
32-
import org.apache.hadoop.hbase.client.ResultScanner;
33-
import org.apache.hadoop.hbase.client.Scan;
3432
import org.apache.hadoop.hbase.client.Table;
33+
import org.apache.hadoop.hbase.client.TableDescriptor;
34+
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
35+
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
3536
import org.apache.hadoop.hbase.testclassification.MediumTests;
3637
import org.apache.hadoop.hbase.testclassification.RPCTests;
3738
import org.apache.hadoop.hbase.util.Bytes;
39+
import org.apache.hadoop.hbase.util.LoadTestKVGenerator;
3840
import org.junit.After;
3941
import org.junit.Before;
4042
import org.junit.ClassRule;
4143
import org.junit.Rule;
4244
import org.junit.Test;
4345
import org.junit.experimental.categories.Category;
44-
import org.junit.rules.TestName;
4546
import org.junit.runner.RunWith;
4647
import org.junit.runners.Parameterized;
4748
import org.junit.runners.Parameterized.Parameters;
@@ -54,14 +55,17 @@ public class TestNettyRpcServer {
5455
public static final HBaseClassTestRule CLASS_RULE =
5556
HBaseClassTestRule.forClass(TestNettyRpcServer.class);
5657

58+
private static final byte[] FAMILY = Bytes.toBytes("f");
59+
private static final byte[] QUALIFIER = Bytes.toBytes("q");
60+
private static final int NUM_ROWS = 100;
61+
private static final int MIN_LEN = 1000;
62+
private static final int MAX_LEN = 1000000;
63+
protected static final LoadTestKVGenerator GENERATOR = new LoadTestKVGenerator(MIN_LEN, MAX_LEN);
64+
protected static HBaseTestingUtility TEST_UTIL;
65+
5766
@Rule
58-
public TestName name = new TestName();
59-
private static HBaseTestingUtility TEST_UTIL;
67+
public TableNameTestRule name = new TableNameTestRule();
6068

61-
private static TableName TABLE;
62-
private static byte[] FAMILY = Bytes.toBytes("f1");
63-
private static byte[] PRIVATE_COL = Bytes.toBytes("private");
64-
private static byte[] PUBLIC_COL = Bytes.toBytes("public");
6569
@Parameterized.Parameter
6670
public String allocatorType;
6771

@@ -74,8 +78,10 @@ public static Collection<Object[]> parameters() {
7478

7579
@Before
7680
public void setup() throws Exception {
77-
TABLE = TableName.valueOf(name.getMethodName().replace('[', '_').replace(']', '_'));
78-
TEST_UTIL = new HBaseTestingUtility();
81+
// A subclass may have already created TEST_UTIL and is now upcalling to us
82+
if (TEST_UTIL == null) {
83+
TEST_UTIL = new HBaseTestingUtility();
84+
}
7985
TEST_UTIL.getConfiguration().set(RpcServerFactory.CUSTOM_RPC_SERVER_IMPL_CONF_KEY,
8086
NettyRpcServer.class.getName());
8187
TEST_UTIL.getConfiguration().set(NettyRpcServer.HBASE_NETTY_ALLOCATOR_KEY, allocatorType);
@@ -89,34 +95,30 @@ public void tearDown() throws Exception {
8995

9096
@Test
9197
public void testNettyRpcServer() throws Exception {
92-
final Table table = TEST_UTIL.createTable(TABLE, FAMILY);
93-
try {
98+
doTest(name.getTableName());
99+
}
100+
101+
protected void doTest(TableName tableName) throws Exception {
102+
// Splitting just complicates the test scenario, disable it
103+
final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
104+
.setRegionSplitPolicyClassName(DisabledRegionSplitPolicy.class.getName()).build();
105+
try (Table table =
106+
TEST_UTIL.createTable(desc, new byte[][] { FAMILY }, TEST_UTIL.getConfiguration())) {
94107
// put some test data
95-
List<Put> puts = new ArrayList<Put>(100);
96-
for (int i = 0; i < 100; i++) {
97-
Put p = new Put(Bytes.toBytes(i));
98-
p.addColumn(FAMILY, PRIVATE_COL, Bytes.toBytes("secret " + i));
99-
p.addColumn(FAMILY, PUBLIC_COL, Bytes.toBytes("info " + i));
100-
puts.add(p);
108+
for (int i = 0; i < NUM_ROWS; i++) {
109+
final byte[] rowKey = Bytes.toBytes(LoadTestKVGenerator.md5PrefixedKey(i));
110+
final byte[] v = GENERATOR.generateRandomSizeValue(rowKey, QUALIFIER);
111+
table.put(new Put(rowKey).addColumn(FAMILY, QUALIFIER, v));
101112
}
102-
table.put(puts);
103-
104113
// read to verify it.
105-
Scan scan = new Scan();
106-
scan.setCaching(16);
107-
ResultScanner rs = table.getScanner(scan);
108-
int rowcnt = 0;
109-
for (Result r : rs) {
110-
rowcnt++;
111-
int rownum = Bytes.toInt(r.getRow());
112-
assertTrue(r.containsColumn(FAMILY, PRIVATE_COL));
113-
assertEquals("secret " + rownum, Bytes.toString(r.getValue(FAMILY, PRIVATE_COL)));
114-
assertTrue(r.containsColumn(FAMILY, PUBLIC_COL));
115-
assertEquals("info " + rownum, Bytes.toString(r.getValue(FAMILY, PUBLIC_COL)));
114+
for (int i = 0; i < NUM_ROWS; i++) {
115+
final byte[] rowKey = Bytes.toBytes(LoadTestKVGenerator.md5PrefixedKey(i));
116+
final Result r = table.get(new Get(rowKey).addColumn(FAMILY, QUALIFIER));
117+
assertNotNull("Result was empty", r);
118+
final byte[] v = r.getValue(FAMILY, QUALIFIER);
119+
assertNotNull("Result did not contain expected value", v);
120+
assertTrue("Value was not verified", LoadTestKVGenerator.verify(v, rowKey, QUALIFIER));
116121
}
117-
assertEquals("Expected 100 rows returned", 100, rowcnt);
118-
} finally {
119-
table.close();
120122
}
121123
}
122124

‎hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSecureNettyRpcServer.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.ipc;
19+
20+
import java.io.File;
21+
import java.security.PrivilegedExceptionAction;
22+
import org.apache.hadoop.conf.Configuration;
23+
import org.apache.hadoop.fs.CommonConfigurationKeys;
24+
import org.apache.hadoop.hbase.HBaseClassTestRule;
25+
import org.apache.hadoop.hbase.HBaseTestingUtility;
26+
import org.apache.hadoop.hbase.TableNameTestRule;
27+
import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
28+
import org.apache.hadoop.hbase.testclassification.MediumTests;
29+
import org.apache.hadoop.hbase.testclassification.RPCTests;
30+
import org.apache.hadoop.minikdc.MiniKdc;
31+
import org.apache.hadoop.security.UserGroupInformation;
32+
import org.junit.After;
33+
import org.junit.Before;
34+
import org.junit.ClassRule;
35+
import org.junit.Rule;
36+
import org.junit.Test;
37+
import org.junit.experimental.categories.Category;
38+
39+
@Category({ RPCTests.class, MediumTests.class })
40+
public class TestSecureNettyRpcServer extends TestNettyRpcServer {
41+
42+
@ClassRule
43+
public static final HBaseClassTestRule CLASS_RULE =
44+
HBaseClassTestRule.forClass(TestSecureNettyRpcServer.class);
45+
46+
private static File KEYTAB_FILE;
47+
private static MiniKdc KDC;
48+
private static String HOST = "localhost";
49+
private static String PRINCIPAL;
50+
private static UserGroupInformation UGI;
51+
52+
@Rule
53+
public TableNameTestRule name = new TableNameTestRule();
54+
55+
@Before
56+
public void setup() throws Exception {
57+
TEST_UTIL = new HBaseTestingUtility();
58+
KEYTAB_FILE = new File(TEST_UTIL.getDataTestDir("keytab").toUri().getPath());
59+
KDC = TEST_UTIL.setupMiniKdc(KEYTAB_FILE);
60+
PRINCIPAL = "hbase/" + HOST;
61+
KDC.createPrincipal(KEYTAB_FILE, PRINCIPAL);
62+
String principalName = PRINCIPAL + "@" + KDC.getRealm();
63+
HBaseKerberosUtils.setPrincipalForTesting(principalName);
64+
Configuration conf = TEST_UTIL.getConfiguration();
65+
HBaseKerberosUtils.setSecuredConfiguration(conf, principalName, principalName);
66+
UGI = login(KEYTAB_FILE.toString(), principalName);
67+
super.setup();
68+
}
69+
70+
@After
71+
public void tearDown() throws Exception {
72+
if (KDC != null) {
73+
KDC.stop();
74+
}
75+
KEYTAB_FILE.delete();
76+
super.tearDown();
77+
TEST_UTIL.cleanupTestDir();
78+
}
79+
80+
@Override
81+
@Test
82+
public void testNettyRpcServer() throws Exception {
83+
UGI.doAs(new PrivilegedExceptionAction<Void>() {
84+
@Override
85+
public Void run() throws Exception {
86+
doTest(name.getTableName());
87+
return null;
88+
}
89+
});
90+
}
91+
92+
static UserGroupInformation login(String krbKeytab, String krbPrincipal) throws Exception {
93+
Configuration conf = new Configuration();
94+
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
95+
UserGroupInformation.setConfiguration(conf);
96+
UserGroupInformation.loginUserFromKeytab(krbPrincipal, krbKeytab);
97+
return UserGroupInformation.getLoginUser();
98+
}
99+
100+
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSecureSimpleRpcServer.java‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.ipc;
19+
20+
import java.io.File;
21+
import java.security.PrivilegedExceptionAction;
22+
import org.apache.hadoop.conf.Configuration;
23+
import org.apache.hadoop.fs.CommonConfigurationKeys;
24+
import org.apache.hadoop.hbase.HBaseClassTestRule;
25+
import org.apache.hadoop.hbase.HBaseTestingUtility;
26+
import org.apache.hadoop.hbase.TableNameTestRule;
27+
import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
28+
import org.apache.hadoop.hbase.testclassification.MediumTests;
29+
import org.apache.hadoop.hbase.testclassification.RPCTests;
30+
import org.apache.hadoop.minikdc.MiniKdc;
31+
import org.apache.hadoop.security.UserGroupInformation;
32+
import org.junit.AfterClass;
33+
import org.junit.BeforeClass;
34+
import org.junit.ClassRule;
35+
import org.junit.Rule;
36+
import org.junit.Test;
37+
import org.junit.experimental.categories.Category;
38+
39+
@Category({ RPCTests.class, MediumTests.class })
40+
public class TestSecureSimpleRpcServer extends TestSimpleRpcServer {
41+
42+
@ClassRule
43+
public static final HBaseClassTestRule CLASS_RULE =
44+
HBaseClassTestRule.forClass(TestSecureSimpleRpcServer.class);
45+
46+
private static File KEYTAB_FILE;
47+
private static MiniKdc KDC;
48+
private static String HOST = "localhost";
49+
private static String PRINCIPAL;
50+
private static UserGroupInformation UGI;
51+
52+
@Rule
53+
public TableNameTestRule name = new TableNameTestRule();
54+
55+
@BeforeClass
56+
public static void setupClass() throws Exception {
57+
TEST_UTIL = new HBaseTestingUtility();
58+
KEYTAB_FILE = new File(TEST_UTIL.getDataTestDir("keytab").toUri().getPath());
59+
KDC = TEST_UTIL.setupMiniKdc(KEYTAB_FILE);
60+
PRINCIPAL = "hbase/" + HOST;
61+
KDC.createPrincipal(KEYTAB_FILE, PRINCIPAL);
62+
String principalName = PRINCIPAL + "@" + KDC.getRealm();
63+
HBaseKerberosUtils.setPrincipalForTesting(principalName);
64+
Configuration conf = TEST_UTIL.getConfiguration();
65+
HBaseKerberosUtils.setSecuredConfiguration(conf, principalName, principalName);
66+
UGI = login(KEYTAB_FILE.toString(), principalName);
67+
TestSimpleRpcServer.setupClass();
68+
69+
}
70+
71+
@AfterClass
72+
public static void tearDownClass() throws Exception {
73+
if (KDC != null) {
74+
KDC.stop();
75+
}
76+
KEYTAB_FILE.delete();
77+
TestSimpleRpcServer.tearDownClass();
78+
TEST_UTIL.cleanupTestDir();
79+
}
80+
81+
@Override
82+
@Test
83+
public void testSimpleRpcServer() throws Exception {
84+
UGI.doAs(new PrivilegedExceptionAction<Void>() {
85+
@Override
86+
public Void run() throws Exception {
87+
doTest(name.getTableName());
88+
return null;
89+
}
90+
});
91+
}
92+
93+
static UserGroupInformation login(String krbKeytab, String krbPrincipal) throws Exception {
94+
Configuration conf = new Configuration();
95+
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
96+
UserGroupInformation.setConfiguration(conf);
97+
UserGroupInformation.loginUserFromKeytab(krbPrincipal, krbKeytab);
98+
return UserGroupInformation.getLoginUser();
99+
}
100+
101+
}

0 commit comments

Comments
 (0)