Skip to content

[HUDI-2410] Fix getDefaultBootstrapIndexClass logical error #3633

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yanghua merged 1 commit into from
Sep 13, 2021
Merged

[HUDI-2410] Fix getDefaultBootstrapIndexClass logical error #3633

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1252,7 +1252,7 @@ private Map<String, List<BootstrapFileMapping>> generateBootstrapIndexAndSourceD
assertTrue(new File(sourcePath.toString()).exists());

// recreate metaClient with Bootstrap base path
metaClient = HoodieTestUtils.init(basePath, getTableType(), sourcePath.toString());
metaClient = HoodieTestUtils.init(basePath, getTableType(), sourcePath.toString(), true);

// generate bootstrap index
Map<String, List<BootstrapFileMapping>> bootstrapMapping = TestBootstrapIndex.generateBootstrapIndex(metaClient, sourcePath.toString(),
Expand Down
9 changes: 5 additions & 4 deletions hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,10 +136,10 @@ public class HoodieTableConfig extends HoodieConfig implements Serializable {
.defaultValue("archived")
.withDocumentation("path under the meta folder, to store archived timeline instants at.");

public static final ConfigProperty<String> BOOTSTRAP_INDEX_ENABLE = ConfigProperty
public static final ConfigProperty<Boolean> BOOTSTRAP_INDEX_ENABLE = ConfigProperty
.key("hoodie.bootstrap.index.enable")
.noDefaultValue()
.withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined.");
.defaultValue(true)
.withDocumentation("Whether or not, this is a bootstrapped table, with bootstrap base data and an mapping index defined, default true.");

public static final ConfigProperty<String> BOOTSTRAP_INDEX_CLASS_NAME = ConfigProperty
.key("hoodie.bootstrap.index.class")
Expand Down Expand Up @@ -298,8 +298,9 @@ public String getBootstrapIndexClass() {
}

public static String getDefaultBootstrapIndexClass(Properties props) {
HoodieConfig hoodieConfig = new HoodieConfig(props);
String defaultClass = BOOTSTRAP_INDEX_CLASS_NAME.defaultValue();
if ("false".equalsIgnoreCase(props.getProperty(BOOTSTRAP_INDEX_ENABLE.key()))) {
if (!hoodieConfig.getBooleanOrDefault(BOOTSTRAP_INDEX_ENABLE)) {
Copy link
Member

@SteNicholas SteNicholas Sep 11, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The option hoodie.bootstrap.index.class could not have the default value. If the default value of the hoodie.bootstrap.index.class is HFileBootstrapIndex.class.getName(), the method getDefaultBootstrapIndexClass should be renamed.

defaultClass = NO_OP_BOOTSTRAP_INDEX_CLASS;
}
return defaultClass;
Expand Down
15 changes: 15 additions & 0 deletions hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,7 @@ public static class PropertyBuilder {
private String partitionFields;
private String bootstrapIndexClass;
private String bootstrapBasePath;
private Boolean bootstrapIndexEnable;
private Boolean populateMetaFields;
private String keyGeneratorClassProp;

Expand Down Expand Up @@ -702,6 +703,11 @@ public PropertyBuilder setBootstrapBasePath(String bootstrapBasePath) {
return this;
}

public PropertyBuilder setBootstrapIndexEnable(Boolean bootstrapIndexEnable) {
this.bootstrapIndexEnable = bootstrapIndexEnable;
return this;
}

public PropertyBuilder setPopulateMetaFields(boolean populateMetaFields) {
this.populateMetaFields = populateMetaFields;
return this;
Expand Down Expand Up @@ -749,6 +755,11 @@ public PropertyBuilder fromProperties(Properties properties) {
if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_BASE_PATH)) {
setBootstrapBasePath(hoodieConfig.getString(HoodieTableConfig.BOOTSTRAP_BASE_PATH));
}

if (hoodieConfig.contains(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE)) {
setBootstrapIndexEnable(hoodieConfig.getBoolean(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE));
}

if (hoodieConfig.contains(HoodieTableConfig.PRECOMBINE_FIELD)) {
setPreCombineField(hoodieConfig.getString(HoodieTableConfig.PRECOMBINE_FIELD));
}
Expand Down Expand Up @@ -807,6 +818,10 @@ public Properties build() {
tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_CLASS_NAME, bootstrapIndexClass);
}

if (null != bootstrapIndexEnable) {
tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE, Boolean.toString(bootstrapIndexEnable));
}

if (null != bootstrapBasePath) {
tableConfig.setValue(HoodieTableConfig.BOOTSTRAP_BASE_PATH, bootstrapBasePath);
}
Expand Down
7 changes: 6 additions & 1 deletion ...common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public static Stream<Arguments> configParams() {

@BeforeEach
public void setup() throws IOException {
metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH);
metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH, false);
basePath = metaClient.getBasePath();
refreshFsView();
}
Expand Down Expand Up @@ -303,6 +303,11 @@ private void checkExternalFile(HoodieFileStatus srcFileStatus, Option<BaseFile>
protected void testViewForFileSlicesWithAsyncCompaction(boolean skipCreatingDataFile, boolean isCompactionInFlight,
int expTotalFileSlices, int expTotalDataFiles, boolean includeInvalidAndInflight, boolean testBootstrap)
throws Exception {

if (testBootstrap) {
metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType(), BOOTSTRAP_SOURCE_PATH, testBootstrap);
}

String partitionPath = "2016/05/01";
new File(basePath + "/" + partitionPath).mkdirs();
String fileId = UUID.randomUUID().toString();
Expand Down
3 changes: 2 additions & 1 deletion hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,10 @@ public static HoodieTableMetaClient init(String basePath, HoodieTableType tableT
return init(getDefaultHadoopConf(), basePath, tableType);
}

public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath) throws IOException {
public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath, boolean bootstrapIndexEnable) throws IOException {
Properties props = new Properties();
props.setProperty(HoodieTableConfig.BOOTSTRAP_BASE_PATH.key(), bootstrapBasePath);
props.put(HoodieTableConfig.BOOTSTRAP_INDEX_ENABLE.key(), bootstrapIndexEnable);
return init(getDefaultHadoopConf(), basePath, tableType, props);
}

Expand Down
4 changes: 2 additions & 2 deletions hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,9 +195,9 @@ private enum EffectiveMode {
private void testBootstrapCommon(boolean partitioned, boolean deltaCommit, EffectiveMode mode) throws Exception {

if (deltaCommit) {
metaClient = HoodieTestUtils.init(basePath, HoodieTableType.MERGE_ON_READ, bootstrapBasePath);
metaClient = HoodieTestUtils.init(basePath, HoodieTableType.MERGE_ON_READ, bootstrapBasePath, true);
} else {
metaClient = HoodieTestUtils.init(basePath, HoodieTableType.COPY_ON_WRITE, bootstrapBasePath);
metaClient = HoodieTestUtils.init(basePath, HoodieTableType.COPY_ON_WRITE, bootstrapBasePath, true);
}

int totalRecords = 100;
Expand Down