Skip to content

Commit 34b5d64

Browse files
Apache9Apache9
committed
HBASE-27818 Split TestReplicationDroppedTables (apache#5206)
Signed-off-by: Wellington Chevreuil <[email protected]> (cherry picked from commit 1ced254)
1 parent 14bee69 commit 34b5d64

File tree

5 files changed

+264
-120
lines changed

5 files changed

+264
-120
lines changed

hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDroppedTables.java renamed to hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationDroppedTablesTestBase.java

Lines changed: 12 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,7 @@
2121
import static org.junit.Assert.fail;
2222

2323
import java.io.IOException;
24-
import org.apache.hadoop.hbase.HBaseClassTestRule;
2524
import org.apache.hadoop.hbase.HConstants;
26-
import org.apache.hadoop.hbase.NamespaceDescriptor;
2725
import org.apache.hadoop.hbase.TableName;
2826
import org.apache.hadoop.hbase.client.Admin;
2927
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
@@ -37,29 +35,23 @@
3735
import org.apache.hadoop.hbase.client.TableDescriptor;
3836
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
3937
import org.apache.hadoop.hbase.ipc.RpcServer;
40-
import org.apache.hadoop.hbase.testclassification.LargeTests;
41-
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
4238
import org.apache.hadoop.hbase.util.Bytes;
4339
import org.apache.hadoop.hbase.util.JVMClusterUtil;
4440
import org.junit.Assert;
4541
import org.junit.Before;
46-
import org.junit.ClassRule;
47-
import org.junit.Test;
48-
import org.junit.experimental.categories.Category;
4942
import org.slf4j.Logger;
5043
import org.slf4j.LoggerFactory;
5144

52-
@Category({ ReplicationTests.class, LargeTests.class })
53-
public class TestReplicationDroppedTables extends TestReplicationBase {
54-
55-
@ClassRule
56-
public static final HBaseClassTestRule CLASS_RULE =
57-
HBaseClassTestRule.forClass(TestReplicationDroppedTables.class);
45+
/**
46+
* Base class for testing replication for dropped tables.
47+
*/
48+
public class ReplicationDroppedTablesTestBase extends TestReplicationBase {
5849

59-
private static final Logger LOG = LoggerFactory.getLogger(TestReplicationDroppedTables.class);
60-
private static final int ROWS_COUNT = 1000;
50+
private static final Logger LOG = LoggerFactory.getLogger(ReplicationDroppedTablesTestBase.class);
51+
protected static final int ROWS_COUNT = 1000;
6152

6253
@Before
54+
@Override
6355
public void setUpBase() throws Exception {
6456
// Starting and stopping replication can make us miss new logs,
6557
// rolling like this makes sure the most recent one gets added to the queue
@@ -103,38 +95,12 @@ public void setUpBase() throws Exception {
10395
CONF1.setInt(RpcServer.MAX_REQUEST_SIZE, 10 * 1024);
10496
}
10597

106-
@Test
107-
public void testEditsStuckBehindDroppedTable() throws Exception {
108-
// Sanity check Make sure by default edits for dropped tables stall the replication queue, even
109-
// when the table(s) in question have been deleted on both ends.
110-
testEditsBehindDroppedTable(false, "test_dropped");
111-
}
112-
113-
@Test
114-
public void testEditsDroppedWithDroppedTable() throws Exception {
115-
// Make sure by default edits for dropped tables are themselves dropped when the
116-
// table(s) in question have been deleted on both ends.
117-
testEditsBehindDroppedTable(true, "test_dropped");
118-
}
119-
120-
@Test
121-
public void testEditsDroppedWithDroppedTableNS() throws Exception {
122-
// also try with a namespace
123-
UTIL1.getAdmin().createNamespace(NamespaceDescriptor.create("NS").build());
124-
UTIL2.getAdmin().createNamespace(NamespaceDescriptor.create("NS").build());
125-
try {
126-
testEditsBehindDroppedTable(true, "NS:test_dropped");
127-
} finally {
128-
UTIL1.getAdmin().deleteNamespace("NS");
129-
UTIL2.getAdmin().deleteNamespace("NS");
130-
}
131-
}
132-
133-
private byte[] generateRowKey(int id) {
98+
protected final byte[] generateRowKey(int id) {
13499
return Bytes.toBytes(String.format("NormalPut%03d", id));
135100
}
136101

137-
private void testEditsBehindDroppedTable(boolean allowProceeding, String tName) throws Exception {
102+
protected final void testEditsBehindDroppedTable(boolean allowProceeding, String tName)
103+
throws Exception {
138104
CONF1.setBoolean(REPLICATION_DROP_ON_DELETED_TABLE_KEY, allowProceeding);
139105
CONF1.setInt(HConstants.REPLICATION_SOURCE_MAXTHREADS_KEY, 1);
140106

@@ -205,80 +171,6 @@ private void testEditsBehindDroppedTable(boolean allowProceeding, String tName)
205171
CONF1.setBoolean(REPLICATION_DROP_ON_DELETED_TABLE_KEY, false);
206172
}
207173

208-
@Test
209-
public void testEditsBehindDroppedTableTiming() throws Exception {
210-
CONF1.setBoolean(REPLICATION_DROP_ON_DELETED_TABLE_KEY, true);
211-
CONF1.setInt(HConstants.REPLICATION_SOURCE_MAXTHREADS_KEY, 1);
212-
213-
// make sure we have a single region server only, so that all
214-
// edits for all tables go there
215-
restartSourceCluster(1);
216-
217-
TableName tablename = TableName.valueOf("testdroppedtimed");
218-
byte[] familyName = Bytes.toBytes("fam");
219-
byte[] row = Bytes.toBytes("row");
220-
221-
TableDescriptor table =
222-
TableDescriptorBuilder.newBuilder(tablename).setColumnFamily(ColumnFamilyDescriptorBuilder
223-
.newBuilder(familyName).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build();
224-
225-
Connection connection1 = ConnectionFactory.createConnection(CONF1);
226-
Connection connection2 = ConnectionFactory.createConnection(CONF2);
227-
try (Admin admin1 = connection1.getAdmin()) {
228-
admin1.createTable(table);
229-
}
230-
try (Admin admin2 = connection2.getAdmin()) {
231-
admin2.createTable(table);
232-
}
233-
UTIL1.waitUntilAllRegionsAssigned(tablename);
234-
UTIL2.waitUntilAllRegionsAssigned(tablename);
235-
236-
// now suspend replication
237-
try (Admin admin1 = connection1.getAdmin()) {
238-
admin1.disableReplicationPeer(PEER_ID2);
239-
}
240-
241-
// put some data (lead with 0 so the edit gets sorted before the other table's edits
242-
// in the replication batch) write a bunch of edits, making sure we fill a batch
243-
try (Table droppedTable = connection1.getTable(tablename)) {
244-
byte[] rowKey = Bytes.toBytes(0 + " put on table to be dropped");
245-
Put put = new Put(rowKey);
246-
put.addColumn(familyName, row, row);
247-
droppedTable.put(put);
248-
}
249-
250-
try (Table table1 = connection1.getTable(tableName)) {
251-
for (int i = 0; i < ROWS_COUNT; i++) {
252-
Put put = new Put(generateRowKey(i)).addColumn(famName, row, row);
253-
table1.put(put);
254-
}
255-
}
256-
257-
try (Admin admin2 = connection2.getAdmin()) {
258-
admin2.disableTable(tablename);
259-
admin2.deleteTable(tablename);
260-
}
261-
262-
// edit should still be stuck
263-
try (Admin admin1 = connection1.getAdmin()) {
264-
// enable the replication peer.
265-
admin1.enableReplicationPeer(PEER_ID2);
266-
// the source table still exists, replication should be stalled
267-
verifyReplicationStuck();
268-
269-
admin1.disableTable(tablename);
270-
// still stuck, source table still exists
271-
verifyReplicationStuck();
272-
273-
admin1.deleteTable(tablename);
274-
// now the source table is gone, replication should proceed, the
275-
// offending edits be dropped
276-
verifyReplicationProceeded();
277-
}
278-
// just to be safe
279-
CONF1.setBoolean(REPLICATION_DROP_ON_DELETED_TABLE_KEY, false);
280-
}
281-
282174
private boolean peerHasAllNormalRows() throws IOException {
283175
try (ResultScanner scanner = htable2.getScanner(new Scan())) {
284176
Result[] results = scanner.next(ROWS_COUNT);
@@ -292,7 +184,7 @@ private boolean peerHasAllNormalRows() throws IOException {
292184
}
293185
}
294186

295-
private void verifyReplicationProceeded() throws Exception {
187+
protected final void verifyReplicationProceeded() throws Exception {
296188
for (int i = 0; i < NB_RETRIES; i++) {
297189
if (i == NB_RETRIES - 1) {
298190
fail("Waited too much time for put replication");
@@ -306,7 +198,7 @@ private void verifyReplicationProceeded() throws Exception {
306198
}
307199
}
308200

309-
private void verifyReplicationStuck() throws Exception {
201+
protected final void verifyReplicationStuck() throws Exception {
310202
for (int i = 0; i < NB_RETRIES; i++) {
311203
if (peerHasAllNormalRows()) {
312204
fail("Edit should have been stuck behind dropped tables");

‎hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestEditsBehindDroppedTableTiming.java‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.replication;
19+
20+
import static org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.REPLICATION_DROP_ON_DELETED_TABLE_KEY;
21+
22+
import org.apache.hadoop.hbase.HBaseClassTestRule;
23+
import org.apache.hadoop.hbase.HConstants;
24+
import org.apache.hadoop.hbase.TableName;
25+
import org.apache.hadoop.hbase.client.Admin;
26+
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
27+
import org.apache.hadoop.hbase.client.Connection;
28+
import org.apache.hadoop.hbase.client.ConnectionFactory;
29+
import org.apache.hadoop.hbase.client.Put;
30+
import org.apache.hadoop.hbase.client.Table;
31+
import org.apache.hadoop.hbase.client.TableDescriptor;
32+
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
33+
import org.apache.hadoop.hbase.testclassification.LargeTests;
34+
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
35+
import org.apache.hadoop.hbase.util.Bytes;
36+
import org.junit.ClassRule;
37+
import org.junit.Test;
38+
import org.junit.experimental.categories.Category;
39+
40+
@Category({ ReplicationTests.class, LargeTests.class })
41+
public class TestEditsBehindDroppedTableTiming extends ReplicationDroppedTablesTestBase {
42+
43+
@ClassRule
44+
public static final HBaseClassTestRule CLASS_RULE =
45+
HBaseClassTestRule.forClass(TestEditsBehindDroppedTableTiming.class);
46+
47+
@Override
48+
public void setUpBase() throws Exception {
49+
CONF1.setBoolean(REPLICATION_DROP_ON_DELETED_TABLE_KEY, true);
50+
CONF1.setInt(HConstants.REPLICATION_SOURCE_MAXTHREADS_KEY, 1);
51+
super.setUpBase();
52+
// make sure we have a single region server only, so that all
53+
// edits for all tables go there
54+
restartSourceCluster(1);
55+
}
56+
57+
@Test
58+
public void testEditsBehindDroppedTableTiming() throws Exception {
59+
TableName tablename = TableName.valueOf("testdroppedtimed");
60+
byte[] familyName = Bytes.toBytes("fam");
61+
byte[] row = Bytes.toBytes("row");
62+
63+
TableDescriptor table =
64+
TableDescriptorBuilder.newBuilder(tablename).setColumnFamily(ColumnFamilyDescriptorBuilder
65+
.newBuilder(familyName).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build();
66+
67+
Connection connection1 = ConnectionFactory.createConnection(CONF1);
68+
Connection connection2 = ConnectionFactory.createConnection(CONF2);
69+
try (Admin admin1 = connection1.getAdmin()) {
70+
admin1.createTable(table);
71+
}
72+
try (Admin admin2 = connection2.getAdmin()) {
73+
admin2.createTable(table);
74+
}
75+
UTIL1.waitUntilAllRegionsAssigned(tablename);
76+
UTIL2.waitUntilAllRegionsAssigned(tablename);
77+
78+
// now suspend replication
79+
try (Admin admin1 = connection1.getAdmin()) {
80+
admin1.disableReplicationPeer(PEER_ID2);
81+
}
82+
83+
// put some data (lead with 0 so the edit gets sorted before the other table's edits
84+
// in the replication batch) write a bunch of edits, making sure we fill a batch
85+
try (Table droppedTable = connection1.getTable(tablename)) {
86+
byte[] rowKey = Bytes.toBytes(0 + " put on table to be dropped");
87+
Put put = new Put(rowKey);
88+
put.addColumn(familyName, row, row);
89+
droppedTable.put(put);
90+
}
91+
92+
try (Table table1 = connection1.getTable(tableName)) {
93+
for (int i = 0; i < ROWS_COUNT; i++) {
94+
Put put = new Put(generateRowKey(i)).addColumn(famName, row, row);
95+
table1.put(put);
96+
}
97+
}
98+
99+
try (Admin admin2 = connection2.getAdmin()) {
100+
admin2.disableTable(tablename);
101+
admin2.deleteTable(tablename);
102+
}
103+
104+
// edit should still be stuck
105+
try (Admin admin1 = connection1.getAdmin()) {
106+
// enable the replication peer.
107+
admin1.enableReplicationPeer(PEER_ID2);
108+
// the source table still exists, replication should be stalled
109+
verifyReplicationStuck();
110+
111+
admin1.disableTable(tablename);
112+
// still stuck, source table still exists
113+
verifyReplicationStuck();
114+
115+
admin1.deleteTable(tablename);
116+
// now the source table is gone, replication should proceed, the
117+
// offending edits be dropped
118+
verifyReplicationProceeded();
119+
}
120+
}
121+
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestEditsDroppedWithDroppedTable.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.replication;
19+
20+
import org.apache.hadoop.hbase.HBaseClassTestRule;
21+
import org.apache.hadoop.hbase.testclassification.LargeTests;
22+
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
23+
import org.junit.ClassRule;
24+
import org.junit.Test;
25+
import org.junit.experimental.categories.Category;
26+
27+
@Category({ ReplicationTests.class, LargeTests.class })
28+
public class TestEditsDroppedWithDroppedTable extends ReplicationDroppedTablesTestBase {
29+
30+
@ClassRule
31+
public static final HBaseClassTestRule CLASS_RULE =
32+
HBaseClassTestRule.forClass(TestEditsDroppedWithDroppedTable.class);
33+
34+
@Test
35+
public void testEditsDroppedWithDroppedTable() throws Exception {
36+
// Make sure by default edits for dropped tables are themselves dropped when the
37+
// table(s) in question have been deleted on both ends.
38+
testEditsBehindDroppedTable(true, "test_dropped");
39+
}
40+
41+
}

0 commit comments

Comments
 (0)