Skip to content

Commit 3915c42

Browse files
wangxianghuyuzhaojing
authored andcommitted
Revert "[HUDI-3523] Introduce AddColumnSchemaPostProcessor to support add columns to the end of a schema (apache#5031)" (apache#6768)
This reverts commit 092375f.
1 parent 740c747 commit 3915c42

File tree

2 files changed

+5
-229
lines changed

2 files changed

+5
-229
lines changed

hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/AddColumnSchemaPostProcessor.java

Lines changed: 0 additions & 172 deletions
This file was deleted.

hudi-utilities/src/test/java/org/apache/hudi/utilities/TestSchemaPostProcessor.java

Lines changed: 5 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020

2121
import org.apache.hudi.common.config.TypedProperties;
2222
import org.apache.hudi.utilities.exception.HoodieSchemaPostProcessException;
23-
import org.apache.hudi.utilities.schema.AddColumnSchemaPostProcessor;
2423
import org.apache.hudi.utilities.schema.DeleteSupportSchemaPostProcessor;
2524
import org.apache.hudi.utilities.schema.DropColumnSchemaPostProcessor;
2625
import org.apache.hudi.utilities.schema.SchemaPostProcessor;
@@ -34,14 +33,10 @@
3433
import org.apache.avro.Schema.Type;
3534
import org.junit.jupiter.api.Assertions;
3635
import org.junit.jupiter.api.Test;
37-
import org.junit.jupiter.params.ParameterizedTest;
38-
import org.junit.jupiter.params.provider.Arguments;
39-
import org.junit.jupiter.params.provider.MethodSource;
4036

4137
import java.io.IOException;
4238
import java.util.ArrayList;
4339
import java.util.List;
44-
import java.util.stream.Stream;
4540

4641
import static org.junit.jupiter.api.Assertions.assertEquals;
4742
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -60,18 +55,13 @@ public class TestSchemaPostProcessor extends UtilitiesTestBase {
6055
+ "{\"name\":\"_row_key\",\"type\":\"string\"},{\"name\":\"rider\",\"type\":\"string\"},{\"name\":\"driver\","
6156
+ "\"type\":\"string\"},{\"name\":\"fare\",\"type\":\"double\"}]}";
6257

63-
private static Stream<Arguments> configParams() {
64-
String[] types = {"bytes", "string", "int", "long", "float", "double", "boolean"};
65-
return Stream.of(types).map(Arguments::of);
66-
}
67-
6858
@Test
6959
public void testPostProcessor() throws IOException {
7060
properties.put(Config.SCHEMA_POST_PROCESSOR_PROP, DummySchemaPostProcessor.class.getName());
7161
SchemaProvider provider =
7262
UtilHelpers.wrapSchemaProviderWithPostProcessor(
73-
UtilHelpers.createSchemaProvider(DummySchemaProvider.class.getName(), properties, jsc),
74-
properties, jsc, null);
63+
UtilHelpers.createSchemaProvider(DummySchemaProvider.class.getName(), properties, jsc),
64+
properties, jsc,null);
7565

7666
Schema schema = provider.getSourceSchema();
7767
assertEquals(schema.getType(), Type.RECORD);
@@ -86,9 +76,9 @@ public void testSparkAvro() throws IOException {
8676
transformerClassNames.add(FlatteningTransformer.class.getName());
8777

8878
SchemaProvider provider =
89-
UtilHelpers.wrapSchemaProviderWithPostProcessor(
90-
UtilHelpers.createSchemaProvider(SparkAvroSchemaProvider.class.getName(), properties, jsc),
91-
properties, jsc, transformerClassNames);
79+
UtilHelpers.wrapSchemaProviderWithPostProcessor(
80+
UtilHelpers.createSchemaProvider(SparkAvroSchemaProvider.class.getName(), properties, jsc),
81+
properties, jsc, transformerClassNames);
9282

9383
Schema schema = provider.getSourceSchema();
9484
assertEquals(schema.getType(), Type.RECORD);
@@ -154,48 +144,6 @@ public void testDeleteColumnThrows() {
154144
Assertions.assertThrows(HoodieSchemaPostProcessException.class, () -> processor.processSchema(schema));
155145
}
156146

157-
@ParameterizedTest
158-
@MethodSource("configParams")
159-
public void testAddPrimitiveTypeColumn(String type) {
160-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_NAME_PROP.key(), "primitive_column");
161-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_TYPE_PROP.key(), type);
162-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_NEXT_PROP.key(), "fare");
163-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_DOC_PROP.key(), "primitive column test");
164-
165-
AddColumnSchemaPostProcessor processor = new AddColumnSchemaPostProcessor(properties, null);
166-
Schema schema = new Schema.Parser().parse(ORIGINAL_SCHEMA);
167-
Schema targetSchema = processor.processSchema(schema);
168-
169-
Schema.Field newColumn = targetSchema.getField("primitive_column");
170-
Schema.Field nextColumn = targetSchema.getField("fare");
171-
172-
assertNotNull(newColumn);
173-
assertEquals("primitive column test", newColumn.doc());
174-
assertEquals(type, newColumn.schema().getType().getName());
175-
assertEquals(nextColumn.pos(), newColumn.pos() + 1);
176-
}
177-
178-
@Test
179-
public void testAddDecimalColumn() {
180-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_NAME_PROP.key(), "decimal_column");
181-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_TYPE_PROP.key(), "decimal");
182-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_DOC_PROP.key(), "decimal column test");
183-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_DEFAULT_PROP.key(), "0.75");
184-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_PRECISION_PROP.key(), "8");
185-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_SCALE_PROP.key(), "6");
186-
properties.put(AddColumnSchemaPostProcessor.Config.SCHEMA_POST_PROCESSOR_ADD_COLUMN_SIZE_PROP.key(), "8");
187-
188-
AddColumnSchemaPostProcessor processor = new AddColumnSchemaPostProcessor(properties, null);
189-
Schema schema = new Schema.Parser().parse(ORIGINAL_SCHEMA);
190-
Schema targetSchema = processor.processSchema(schema);
191-
192-
Schema.Field newColumn = targetSchema.getField("decimal_column");
193-
194-
assertNotNull(newColumn);
195-
assertEquals("decimal", newColumn.schema().getLogicalType().getName());
196-
assertEquals(5, newColumn.pos());
197-
}
198-
199147
@Test
200148
public void testSparkAvroSchema() throws IOException {
201149
SparkAvroPostProcessor processor = new SparkAvroPostProcessor(properties, null);

0 commit comments

Comments
 (0)