diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 4d579c16af26..161e11ec3a8b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -1044,7 +1044,24 @@ default void modifyTable(TableDescriptor td) throws IOException { * @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the * operation to complete */ - Future modifyTableAsync(TableDescriptor td) throws IOException; + default Future modifyTableAsync(TableDescriptor td) throws IOException { + return modifyTableAsync(td, true); + } + + /** + * The same as {@link #modifyTableAsync(TableDescriptor td)}, except for the reopenRegions + * parameter, which controls whether the process of modifying the table should reopen all regions. + * @param td description of the table + * @param reopenRegions By default, 'modifyTable' reopens all regions, potentially causing a RIT + * (Region In Transition) storm in large tables. If set to 'false', regions + * will remain unaware of the modification until they are individually + * reopened. Please note that this may temporarily result in configuration + * inconsistencies among regions. + * @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the + * operation to complete + * @throws IOException if a remote or network exception occurs + */ + Future modifyTableAsync(TableDescriptor td, boolean reopenRegions) throws IOException; /** * Change the store file tracker of the given table. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java index 690b6406fd3a..ad7b98413c94 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java @@ -492,7 +492,13 @@ public Future splitRegionAsync(byte[] regionName, byte[] splitPoint) throw @Override public Future modifyTableAsync(TableDescriptor td) throws IOException { - return admin.modifyTable(td); + return modifyTableAsync(td, true); + } + + @Override + public Future modifyTableAsync(TableDescriptor td, boolean reopenRegions) + throws IOException { + return admin.modifyTable(td, reopenRegions); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 960982f5e3f1..44d8b6b5e999 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -198,7 +198,20 @@ CompletableFuture createTable(TableDescriptor desc, byte[] startKey, byte[ * Modify an existing table, more IRB friendly version. * @param desc modified description of the table */ - CompletableFuture modifyTable(TableDescriptor desc); + default CompletableFuture modifyTable(TableDescriptor desc) { + return modifyTable(desc, true); + } + + /** + * Modify an existing table, more IRB friendly version. + * @param desc description of the table + * @param reopenRegions By default, 'modifyTable' reopens all regions, potentially causing a RIT + * (Region In Transition) storm in large tables. If set to 'false', regions + * will remain unaware of the modification until they are individually + * reopened. Please note that this may temporarily result in configuration + * inconsistencies among regions. + */ + CompletableFuture modifyTable(TableDescriptor desc, boolean reopenRegions); /** * Change the store file tracker of the given table. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 5ee8a6ab8269..1a0f27d86b3f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -153,7 +153,12 @@ public CompletableFuture createTable(TableDescriptor desc, byte[][] splitK @Override public CompletableFuture modifyTable(TableDescriptor desc) { - return wrap(rawAdmin.modifyTable(desc)); + return modifyTable(desc, true); + } + + @Override + public CompletableFuture modifyTable(TableDescriptor desc, boolean reopenRegions) { + return wrap(rawAdmin.modifyTable(desc, reopenRegions)); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index ee1dfac16bd3..7e8ce957cc8a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -692,9 +692,14 @@ private CompletableFuture createTable(TableName tableName, CreateTableRequ @Override public CompletableFuture modifyTable(TableDescriptor desc) { + return modifyTable(desc, true); + } + + @Override + public CompletableFuture modifyTable(TableDescriptor desc, boolean reopenRegions) { return this. procedureCall(desc.getTableName(), RequestConverter.buildModifyTableRequest(desc.getTableName(), desc, ng.getNonceGroup(), - ng.newNonce()), + ng.newNonce(), reopenRegions), (s, c, req, done) -> s.modifyTable(c, req, done), (resp) -> resp.getProcId(), new ModifyTableProcedureBiConsumer(this, desc.getTableName())); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index 33884158da48..66feb67e4412 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -1097,12 +1097,14 @@ public static CreateTableRequest buildCreateTableRequest(final TableDescriptor t * @return a ModifyTableRequest */ public static ModifyTableRequest buildModifyTableRequest(final TableName tableName, - final TableDescriptor tableDesc, final long nonceGroup, final long nonce) { + final TableDescriptor tableDesc, final long nonceGroup, final long nonce, + final boolean reopenRegions) { ModifyTableRequest.Builder builder = ModifyTableRequest.newBuilder(); builder.setTableName(ProtobufUtil.toProtoTableName(tableName)); builder.setTableSchema(ProtobufUtil.toTableSchema(tableDesc)); builder.setNonceGroup(nonceGroup); builder.setNonce(nonce); + builder.setReopenRegions(reopenRegions); return builder.build(); } diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto index 5d715fdcdd16..e8fb00320e66 100644 --- a/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto @@ -194,6 +194,7 @@ message ModifyTableRequest { required TableSchema table_schema = 2; optional uint64 nonce_group = 3 [default = 0]; optional uint64 nonce = 4 [default = 0]; + optional bool reopen_regions = 5 [default = true]; } message ModifyTableResponse { diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto index 3f3ecd63b002..715d1dbbd4b4 100644 --- a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto @@ -82,6 +82,7 @@ message ModifyTableStateData { required TableSchema modified_table_schema = 3; required bool delete_column_family_in_modify = 4; optional bool should_check_descriptor = 5; + optional bool reopen_regions = 6; } enum TruncateTableState { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 995bff17724e..1964c45f81ca 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1163,7 +1163,7 @@ private void finishActiveMasterInitialization() throws IOException, InterruptedE procedureExecutor.submitProcedure(new ModifyTableProcedure( procedureExecutor.getEnvironment(), TableDescriptorBuilder.newBuilder(metaDesc) .setRegionReplication(replicasNumInConf).build(), - null, metaDesc, false)); + null, metaDesc, false, true)); } } } @@ -2763,6 +2763,13 @@ protected String getDescription() { private long modifyTable(final TableName tableName, final TableDescriptorGetter newDescriptorGetter, final long nonceGroup, final long nonce, final boolean shouldCheckDescriptor) throws IOException { + return modifyTable(tableName, newDescriptorGetter, nonceGroup, nonce, shouldCheckDescriptor, + true); + } + + private long modifyTable(final TableName tableName, + final TableDescriptorGetter newDescriptorGetter, final long nonceGroup, final long nonce, + final boolean shouldCheckDescriptor, final boolean reopenRegions) throws IOException { return MasterProcedureUtil .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) { @Override @@ -2781,7 +2788,7 @@ protected void run() throws IOException { // checks. This will block only the beginning of the procedure. See HBASE-19953. ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch(); submitProcedure(new ModifyTableProcedure(procedureExecutor.getEnvironment(), - newDescriptor, latch, oldDescriptor, shouldCheckDescriptor)); + newDescriptor, latch, oldDescriptor, shouldCheckDescriptor, reopenRegions)); latch.await(); getMaster().getMasterCoprocessorHost().postModifyTable(tableName, oldDescriptor, @@ -2798,14 +2805,14 @@ protected String getDescription() { @Override public long modifyTable(final TableName tableName, final TableDescriptor newDescriptor, - final long nonceGroup, final long nonce) throws IOException { + final long nonceGroup, final long nonce, final boolean reopenRegions) throws IOException { checkInitialized(); return modifyTable(tableName, new TableDescriptorGetter() { @Override public TableDescriptor get() throws IOException { return newDescriptor; } - }, nonceGroup, nonce, false); + }, nonceGroup, nonce, false, reopenRegions); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index b6a17d8503b2..5c28a3527573 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1530,7 +1530,8 @@ public ModifyTableResponse modifyTable(RpcController controller, ModifyTableRequ throws ServiceException { try { long procId = server.modifyTable(ProtobufUtil.toTableName(req.getTableName()), - ProtobufUtil.toTableDescriptor(req.getTableSchema()), req.getNonceGroup(), req.getNonce()); + ProtobufUtil.toTableDescriptor(req.getTableSchema()), req.getNonceGroup(), req.getNonce(), + req.getReopenRegions()); return ModifyTableResponse.newBuilder().setProcId(procId).build(); } catch (IOException ioe) { throw new ServiceException(ioe); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index 933bf0d18150..d8cae2279d53 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -158,8 +158,19 @@ public long truncateTable(final TableName tableName, final boolean preserveSplit * @param tableName The table name * @param descriptor The updated table descriptor */ + default long modifyTable(final TableName tableName, final TableDescriptor descriptor, + final long nonceGroup, final long nonce) throws IOException { + return modifyTable(tableName, descriptor, nonceGroup, nonce, true); + } + + /** + * Modify the descriptor of an existing table + * @param tableName The table name + * @param descriptor The updated table descriptor + * @param reopenRegions Whether to reopen regions after modifying the table descriptor + */ long modifyTable(final TableName tableName, final TableDescriptor descriptor, - final long nonceGroup, final long nonce) throws IOException; + final long nonceGroup, final long nonce, final boolean reopenRegions) throws IOException; /** * Modify the store file tracker of an existing table diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java index 28f955126bcd..ff0d7d2cc94b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Supplier; @@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; @@ -56,6 +58,7 @@ public class ModifyTableProcedure extends AbstractStateMachineTableProcedure s = new HashSet<>(Arrays.asList(TableDescriptorBuilder.REGION_REPLICATION, + TableDescriptorBuilder.REGION_MEMSTORE_REPLICATION, RSGroupInfo.TABLE_DESC_PROP_GROUP)); + for (String k : s) { + if ( + isTablePropertyModified(this.unmodifiedTableDescriptor, this.modifiedTableDescriptor, k) + ) { + throw new HBaseIOException( + "Can not modify " + k + " of a table when modification won't reopen regions"); + } + } + } + } + + /** + * Comparing the value associated with a given key across two TableDescriptor instances' + * properties. + * @return True if the table property key is the same in both. + */ + private boolean isTablePropertyModified(TableDescriptor oldDescriptor, + TableDescriptor newDescriptor, String key) { + String oldV = oldDescriptor.getValue(key); + String newV = newDescriptor.getValue(key); + if (oldV == null && newV == null) { + return false; + } else if (oldV != null && newV != null && oldV.equals(newV)) { + return false; + } + return true; } private void initialize(final TableDescriptor unmodifiedTableDescriptor, @@ -125,7 +183,13 @@ protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableS break; case MODIFY_TABLE_PRE_OPERATION: preModify(env, state); - setNextState(ModifyTableState.MODIFY_TABLE_CLOSE_EXCESS_REPLICAS); + // We cannot allow changes to region replicas when 'reopenRegions==false', + // as this mode bypasses the state management required for modifying region replicas. + if (reopenRegions) { + setNextState(ModifyTableState.MODIFY_TABLE_CLOSE_EXCESS_REPLICAS); + } else { + setNextState(ModifyTableState.MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR); + } break; case MODIFY_TABLE_CLOSE_EXCESS_REPLICAS: if (isTableEnabled(env)) { @@ -135,7 +199,11 @@ protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableS break; case MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR: updateTableDescriptor(env); - setNextState(ModifyTableState.MODIFY_TABLE_REMOVE_REPLICA_COLUMN); + if (reopenRegions) { + setNextState(ModifyTableState.MODIFY_TABLE_REMOVE_REPLICA_COLUMN); + } else { + setNextState(ModifyTableState.MODIFY_TABLE_POST_OPERATION); + } break; case MODIFY_TABLE_REMOVE_REPLICA_COLUMN: removeReplicaColumnsIfNeeded(env); @@ -143,7 +211,11 @@ protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableS break; case MODIFY_TABLE_POST_OPERATION: postModify(env, state); - setNextState(ModifyTableState.MODIFY_TABLE_REOPEN_ALL_REGIONS); + if (reopenRegions) { + setNextState(ModifyTableState.MODIFY_TABLE_REOPEN_ALL_REGIONS); + } else { + return Flow.NO_MORE_STATE; + } break; case MODIFY_TABLE_REOPEN_ALL_REGIONS: if (isTableEnabled(env)) { @@ -238,7 +310,7 @@ protected void serializeStateData(ProcedureStateSerializer serializer) throws IO .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) .setModifiedTableSchema(ProtobufUtil.toTableSchema(modifiedTableDescriptor)) .setDeleteColumnFamilyInModify(deleteColumnFamilyInModify) - .setShouldCheckDescriptor(shouldCheckDescriptor); + .setShouldCheckDescriptor(shouldCheckDescriptor).setReopenRegions(reopenRegions); if (unmodifiedTableDescriptor != null) { modifyTableMsg @@ -260,6 +332,7 @@ protected void deserializeStateData(ProcedureStateSerializer serializer) throws deleteColumnFamilyInModify = modifyTableMsg.getDeleteColumnFamilyInModify(); shouldCheckDescriptor = modifyTableMsg.hasShouldCheckDescriptor() ? modifyTableMsg.getShouldCheckDescriptor() : false; + reopenRegions = modifyTableMsg.hasReopenRegions() ? modifyTableMsg.getReopenRegions() : true; if (modifyTableMsg.hasUnmodifiedTableSchema()) { unmodifiedTableDescriptor = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java index a19b6ffbec64..f37fd86807a7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java @@ -257,6 +257,12 @@ public long modifyTable(final TableName tableName, final TableDescriptor descrip return -1; } + @Override + public long modifyTable(TableName tableName, TableDescriptor descriptor, long nonceGroup, + long nonce, boolean reopenRegions) throws IOException { + return -1; + } + @Override public long enableTable(final TableName tableName, final long nonceGroup, final long nonce) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java index 51be6385319f..c8043b8ee3b2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java @@ -25,18 +25,22 @@ import org.apache.hadoop.hbase.ConcurrentTableModificationException; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.InvalidFamilyOperationException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.CoprocessorDescriptorBuilder; import org.apache.hadoop.hbase.client.PerClientRandomNonceGenerator; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.StepHook; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; @@ -584,4 +588,114 @@ public void run() { t2.join(); assertFalse("Expected ConcurrentTableModificationException.", (t1.exception || t2.exception)); } + + @Test + public void testModifyWillNotReopenRegions() throws IOException { + final boolean reopenRegions = false; + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf"); + + // Test 1: Modify table without reopening any regions + TableDescriptor htd = UTIL.getAdmin().getDescriptor(tableName); + TableDescriptor modifiedDescriptor = TableDescriptorBuilder.newBuilder(htd) + .setValue("test" + ".hbase.conf", "test.hbase.conf.value").build(); + long procId1 = ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1)); + TableDescriptor currentHtd = UTIL.getAdmin().getDescriptor(tableName); + assertEquals("test.hbase.conf.value", currentHtd.getValue("test.hbase.conf")); + // Regions should not aware of any changes. + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + Assert.assertNull(r.getTableDescriptor().getValue("test.hbase.conf")); + } + // Force regions to reopen + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + getMaster().getAssignmentManager().move(r.getRegionInfo()); + } + // After the regions reopen, ensure that the configuration is updated. + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + assertEquals("test.hbase.conf.value", r.getTableDescriptor().getValue("test.hbase.conf")); + } + + // Test 2: Modifying region replication is not allowed + htd = UTIL.getAdmin().getDescriptor(tableName); + long oldRegionReplication = htd.getRegionReplication(); + modifiedDescriptor = TableDescriptorBuilder.newBuilder(htd).setRegionReplication(3).build(); + try { + ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + Assert.fail( + "An exception should have been thrown while modifying region replication properties."); + } catch (HBaseIOException e) { + assertTrue(e.getMessage().contains("Can not modify")); + } + currentHtd = UTIL.getAdmin().getDescriptor(tableName); + // Nothing changed + assertEquals(oldRegionReplication, currentHtd.getRegionReplication()); + + // Test 3: Adding CFs is not allowed + htd = UTIL.getAdmin().getDescriptor(tableName); + modifiedDescriptor = TableDescriptorBuilder.newBuilder(htd) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder("NewCF".getBytes()).build()) + .build(); + try { + ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + Assert.fail("Should have thrown an exception while modifying CF!"); + } catch (HBaseIOException e) { + assertTrue(e.getMessage().contains("Cannot add or remove column families")); + } + currentHtd = UTIL.getAdmin().getDescriptor(tableName); + Assert.assertNull(currentHtd.getColumnFamily("NewCF".getBytes())); + + // Test 4: Modifying CF property is allowed + htd = UTIL.getAdmin().getDescriptor(tableName); + modifiedDescriptor = + TableDescriptorBuilder + .newBuilder(htd).modifyColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder("cf".getBytes()).setCompressionType(Compression.Algorithm.SNAPPY).build()) + .build(); + ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + Assert.assertEquals(Compression.Algorithm.NONE, + r.getTableDescriptor().getColumnFamily("cf".getBytes()).getCompressionType()); + } + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + getMaster().getAssignmentManager().move(r.getRegionInfo()); + } + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + Assert.assertEquals(Compression.Algorithm.SNAPPY, + r.getTableDescriptor().getColumnFamily("cf".getBytes()).getCompressionType()); + } + + // Test 5: Modifying coprocessor is not allowed + htd = UTIL.getAdmin().getDescriptor(tableName); + modifiedDescriptor = + TableDescriptorBuilder.newBuilder(htd).setCoprocessor(CoprocessorDescriptorBuilder + .newBuilder("any.coprocessor.name").setJarPath("fake/path").build()).build(); + try { + ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + Assert.fail("Should have thrown an exception while modifying coprocessor!"); + } catch (HBaseIOException e) { + assertTrue(e.getMessage().contains("Can not modify Coprocessor")); + } + currentHtd = UTIL.getAdmin().getDescriptor(tableName); + assertEquals(0, currentHtd.getCoprocessorDescriptors().size()); + + // Test 6: Modifying is not allowed + htd = UTIL.getAdmin().getDescriptor(tableName); + modifiedDescriptor = TableDescriptorBuilder.newBuilder(htd).setRegionReplication(3).build(); + try { + ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + Assert.fail("Should have thrown an exception while modifying coprocessor!"); + } catch (HBaseIOException e) { + System.out.println(e.getMessage()); + assertTrue(e.getMessage().contains("Can not modify REGION_REPLICATION")); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java index 9b1d8524d003..813d3e121edb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java @@ -413,7 +413,12 @@ public Future splitRegionAsync(byte[] regionName, byte[] splitPoint) throw } public Future modifyTableAsync(TableDescriptor td) throws IOException { - return admin.modifyTableAsync(td); + return modifyTableAsync(td, true); + } + + public Future modifyTableAsync(TableDescriptor td, boolean reopenRegions) + throws IOException { + return admin.modifyTableAsync(td, reopenRegions); } public void shutdown() throws IOException { diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 7477b8ec164f..4695b277b849 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -774,6 +774,7 @@ def alter(table_name_str, wait = true, *args) # Get table descriptor tdb = TableDescriptorBuilder.newBuilder(@admin.getDescriptor(table_name)) hasTableUpdate = false + reopen_regions = true # Process all args args.each do |arg| @@ -783,6 +784,14 @@ def alter(table_name_str, wait = true, *args) # Normalize args to support shortcut delete syntax arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] + if arg.key?(REOPEN_REGIONS) + if !['true', 'false'].include?(arg[REOPEN_REGIONS].downcase) + raise(ArgumentError, "Invalid 'REOPEN_REGIONS' for non-boolean value.") + end + reopen_regions = JBoolean.valueOf(arg[REOPEN_REGIONS]) + arg.delete(REOPEN_REGIONS) + end + # There are 3 possible options. # 1) Column family spec. Distinguished by having a NAME and no METHOD. method = arg.delete(METHOD) @@ -906,9 +915,13 @@ def alter(table_name_str, wait = true, *args) # Bulk apply all table modifications. if hasTableUpdate - future = @admin.modifyTableAsync(tdb.build) - - if wait == true + future = @admin.modifyTableAsync(tdb.build, reopen_regions) + if reopen_regions == false + puts("WARNING: You are using REOPEN_REGIONS => 'false' to modify a table, which will + result in inconsistencies in the configuration of online regions and other risks. If you + encounter any issues, use the original 'alter' command to make the modification again!") + future.get + elsif wait == true puts 'Updating all regions with the new schema...' future.get end diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb b/hbase-shell/src/main/ruby/hbase_constants.rb index 5f994c7b5ae0..d4df1f8f5821 100644 --- a/hbase-shell/src/main/ruby/hbase_constants.rb +++ b/hbase-shell/src/main/ruby/hbase_constants.rb @@ -107,6 +107,7 @@ module HBaseConstants VALUE = 'VALUE'.freeze VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS VISIBILITY = 'VISIBILITY'.freeze + REOPEN_REGIONS = 'REOPEN_REGIONS'.freeze # aliases ENDKEY = STOPROW diff --git a/hbase-shell/src/main/ruby/shell/commands/alter.rb b/hbase-shell/src/main/ruby/shell/commands/alter.rb index ad0cb5a5a49b..18ec24be7be6 100644 --- a/hbase-shell/src/main/ruby/shell/commands/alter.rb +++ b/hbase-shell/src/main/ruby/shell/commands/alter.rb @@ -72,6 +72,27 @@ def help hbase> alter 't1', CONFIGURATION => {'hbase.hregion.scan.loadColumnFamiliesOnDemand' => 'true'} hbase> alter 't1', {NAME => 'f2', CONFIGURATION => {'hbase.hstore.blockingStoreFiles' => '10'}} +You can also set configuration setting with REOPEN_REGIONS=>'false' to avoid regions RIT, which +let the modification take effect after regions was reopened (Be careful, the regions of the table +may be configured inconsistently If regions are not reopened after the modification) + + hbase> alter 't1', REOPEN_REGIONS => 'false', MAX_FILESIZE => '134217728' + hbase> alter 't1', REOPEN_REGIONS => 'false', CONFIGURATION => {'hbase.hregion.scan + .loadColumnFamiliesOnDemand' => 'true'} + +However, be aware that: +1. Inconsistency Risks: If the regions are not reopened after the modification, the table's regions +may become inconsistently configured. Ensure that you manually reopen the regions as soon as +possible to apply the changes consistently across the entire table. +2. If changes are made to the table without reopening the regions, we currently only allow +lightweight operations. The following types of changes, which may lead to unknown situations, +will throw an exception: + a. Adding or removing CFs, coprocessors. + b. Modifying the table name. + c. Changing region replica related configurations such as 'REGION_REPLICATION' + and 'REGION_MEMSTORE_REPLICATION'. + d. Changing the rsgroup. + You can also unset configuration settings specific to this table: hbase> alter 't1', METHOD => 'table_conf_unset', NAME => 'hbase.hregion.majorcompaction' diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 1b7b6938524a..72fc025e9461 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -735,6 +735,11 @@ public Future modifyTableAsync(TableDescriptor td) { throw new NotImplementedException("modifyTableAsync not supported in ThriftAdmin"); } + @Override + public Future modifyTableAsync(TableDescriptor td, boolean reopenRegions) { + throw new NotImplementedException("modifyTableAsync not supported in ThriftAdmin"); + } + @Override public void shutdown() { throw new NotImplementedException("shutdown not supported in ThriftAdmin"); diff --git a/pom.xml b/pom.xml index 56058b8e2b97..66d632209d6e 100644 --- a/pom.xml +++ b/pom.xml @@ -2819,6 +2819,7 @@ **/generated/* **/package-info.java + **/.idea/**