Skip to content

Commit a645571

Browse files
authored
Revert "[HUDI-4324] Remove use_jdbc config from hudi sync (#6072)"
This reverts commit 046044c.
1 parent 348519f commit a645571

20 files changed

Lines changed: 72 additions & 29 deletions

File tree

conf/hudi-defaults.conf.template

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
# Example:
2222
# hoodie.datasource.hive_sync.jdbcurl jdbc:hive2://localhost:10000
23-
# hoodie.datasource.hive_sync.mode jdbc
23+
# hoodie.datasource.hive_sync.use_jdbc true
2424
# hoodie.datasource.hive_sync.support_timestamp false
2525
# hoodie.index.type BLOOM
2626
# hoodie.metadata.enable false

docker/demo/config/hoodie-incr.properties

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,5 @@ hoodie.deltastreamer.source.hoodieincr.path=/docker_hoodie_sync_valid_test
2828
hoodie.deltastreamer.source.hoodieincr.read_latest_on_missing_ckpt=true
2929
# hive sync
3030
hoodie.datasource.hive_sync.table=docker_hoodie_sync_valid_test_2
31-
hoodie.datasource.hive_sync.mode=jdbc
3231
hoodie.datasource.hive_sync.jdbcurl=jdbc:hive2://hiveserver:10000
33-
hoodie.datasource.hive_sync.partition_fields=partition
32+
hoodie.datasource.hive_sync.partition_fields=partition

docker/demo/sparksql-incremental.commands

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ spark.sql("select key, `_hoodie_partition_path` as datestr, symbol, ts, open, cl
4747
option(HoodieWriteConfig.TBL_NAME.key(), "stock_ticks_derived_mor").
4848
option(HoodieSyncConfig.META_SYNC_TABLE_NAME.key(), "stock_ticks_derived_mor").
4949
option(HoodieSyncConfig.META_SYNC_DATABASE_NAME.key(), "default").
50-
option(HiveSyncConfigHolder.HIVE_SYNC_MODE.key(), "jdbc").
5150
option(HiveSyncConfigHolder.HIVE_URL.key(), "jdbc:hive2://hiveserver:10000").
5251
option(HiveSyncConfigHolder.HIVE_USER.key(), "hive").
5352
option(HiveSyncConfigHolder.HIVE_PASS.key(), "hive").
@@ -80,7 +79,6 @@ spark.sql("select key, `_hoodie_partition_path` as datestr, symbol, ts, open, cl
8079
option(HoodieWriteConfig.TBL_NAME.key(), "stock_ticks_derived_mor_bs").
8180
option(HoodieSyncConfig.META_SYNC_TABLE_NAME.key(), "stock_ticks_derived_mor_bs").
8281
option(HoodieSyncConfig.META_SYNC_DATABASE_NAME.key(), "default").
83-
option(HiveSyncConfigHolder.HIVE_SYNC_MODE.key(), "jdbc").
8482
option(HiveSyncConfigHolder.HIVE_URL.key(), "jdbc:hive2://hiveserver:10000").
8583
option(HiveSyncConfigHolder.HIVE_USER.key(), "hive").
8684
option(HiveSyncConfigHolder.HIVE_PASS.key(), "hive").

hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ public void testLoadGlobalConfFile() {
185185
DFSPropertiesConfiguration.refreshGlobalProps();
186186
assertEquals(5, DFSPropertiesConfiguration.getGlobalProps().size());
187187
assertEquals("jdbc:hive2://localhost:10000", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.datasource.hive_sync.jdbcurl"));
188-
assertEquals("jdbc", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.datasource.hive_sync.mode"));
188+
assertEquals("true", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.datasource.hive_sync.use_jdbc"));
189189
assertEquals("false", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.datasource.hive_sync.support_timestamp"));
190190
assertEquals("BLOOM", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.index.type"));
191191
assertEquals("true", DFSPropertiesConfiguration.getGlobalProps().get("hoodie.metadata.enable"));

hudi-common/src/test/resources/external-config/hudi-defaults.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
# Example:
2222
hoodie.datasource.hive_sync.jdbcurl jdbc:hive2://localhost:10000
23-
hoodie.datasource.hive_sync.mode jdbc
23+
hoodie.datasource.hive_sync.use_jdbc true
2424
hoodie.datasource.hive_sync.support_timestamp false
2525
hoodie.index.type BLOOM
2626
hoodie.metadata.enable true

hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -755,6 +755,12 @@ private FlinkOptions() {
755755
.defaultValue(false)
756756
.withDescription("Assume partitioning is yyyy/mm/dd, default false");
757757

758+
public static final ConfigOption<Boolean> HIVE_SYNC_USE_JDBC = ConfigOptions
759+
.key("hive_sync.use_jdbc")
760+
.booleanType()
761+
.defaultValue(true)
762+
.withDescription("Use JDBC when hive synchronization is enabled, default true");
763+
758764
public static final ConfigOption<Boolean> HIVE_SYNC_AUTO_CREATE_DB = ConfigOptions
759765
.key("hive_sync.auto_create_db")
760766
.booleanType()

hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_TABLE_SERDE_PROPERTIES;
4444
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_URL;
4545
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_USER;
46+
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_USE_JDBC;
4647
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_USE_PRE_APACHE_INPUT_FORMAT;
4748
import static org.apache.hudi.hive.HiveSyncConfigHolder.METASTORE_URIS;
4849
import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_ASSUME_DATE_PARTITION;
@@ -104,6 +105,7 @@ public static Properties buildSyncConfig(Configuration conf) {
104105
props.setPropertyIfNonNull(HIVE_TABLE_SERDE_PROPERTIES.key(), conf.getString(FlinkOptions.HIVE_SYNC_TABLE_SERDE_PROPERTIES));
105106
props.setPropertyIfNonNull(META_SYNC_PARTITION_FIELDS.key(), String.join(",", FilePathUtils.extractHivePartitionFields(conf)));
106107
props.setPropertyIfNonNull(META_SYNC_PARTITION_EXTRACTOR_CLASS.key(), conf.getString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME));
108+
props.setPropertyIfNonNull(HIVE_USE_JDBC.key(), String.valueOf(conf.getBoolean(FlinkOptions.HIVE_SYNC_USE_JDBC)));
107109
props.setPropertyIfNonNull(META_SYNC_USE_FILE_LISTING_FROM_METADATA.key(), String.valueOf(conf.getBoolean(FlinkOptions.METADATA_ENABLED)));
108110
props.setPropertyIfNonNull(HIVE_IGNORE_EXCEPTIONS.key(), String.valueOf(conf.getBoolean(FlinkOptions.HIVE_SYNC_IGNORE_EXCEPTIONS)));
109111
props.setPropertyIfNonNull(HIVE_SUPPORT_TIMESTAMP_TYPE.key(), String.valueOf(conf.getBoolean(FlinkOptions.HIVE_SYNC_SUPPORT_TIMESTAMP)));

hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/FlinkStreamerConfig.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,9 @@ public class FlinkStreamerConfig extends Configuration {
320320
@Parameter(names = {"--hive-sync-assume-date-partitioning"}, description = "Assume partitioning is yyyy/mm/dd, default false")
321321
public Boolean hiveSyncAssumeDatePartition = false;
322322

323+
@Parameter(names = {"--hive-sync-use-jdbc"}, description = "Use JDBC when hive synchronization is enabled, default true")
324+
public Boolean hiveSyncUseJdbc = true;
325+
323326
@Parameter(names = {"--hive-sync-auto-create-db"}, description = "Auto create hive database if it does not exists, default true")
324327
public Boolean hiveSyncAutoCreateDb = true;
325328

@@ -416,6 +419,7 @@ public static org.apache.flink.configuration.Configuration toFlinkConfig(FlinkSt
416419
conf.setString(FlinkOptions.HIVE_SYNC_PARTITION_FIELDS, config.hiveSyncPartitionFields);
417420
conf.setString(FlinkOptions.HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME, config.hiveSyncPartitionExtractorClass);
418421
conf.setBoolean(FlinkOptions.HIVE_SYNC_ASSUME_DATE_PARTITION, config.hiveSyncAssumeDatePartition);
422+
conf.setBoolean(FlinkOptions.HIVE_SYNC_USE_JDBC, config.hiveSyncUseJdbc);
419423
conf.setBoolean(FlinkOptions.HIVE_SYNC_AUTO_CREATE_DB, config.hiveSyncAutoCreateDb);
420424
conf.setBoolean(FlinkOptions.HIVE_SYNC_IGNORE_EXCEPTIONS, config.hiveSyncIgnoreExceptions);
421425
conf.setBoolean(FlinkOptions.HIVE_SYNC_SKIP_RO_SUFFIX, config.hiveSyncSkipRoSuffix);

hudi-kafka-connect/demo/config-sink-hive.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"hoodie.datasource.hive_sync.table": "huditesttopic",
2222
"hoodie.datasource.hive_sync.partition_fields": "date",
2323
"hoodie.datasource.hive_sync.partition_extractor_class": "org.apache.hudi.hive.MultiPartKeysValueExtractor",
24+
"hoodie.datasource.hive_sync.use_jdbc": "false",
2425
"hoodie.datasource.hive_sync.mode": "hms",
2526
"dfs.client.use.datanode.hostname": "true",
2627
"hive.metastore.uris": "thrift://hivemetastore:9083",

hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/KafkaConnectConfigs.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ public String getHadoopConfHome() {
172172
public static final String HIVE_URL = "hoodie.datasource.hive_sync.jdbcurl";
173173
public static final String HIVE_PARTITION_FIELDS = "hoodie.datasource.hive_sync.partition_fields";
174174
public static final String HIVE_PARTITION_EXTRACTOR_CLASS = "hoodie.datasource.hive_sync.partition_extractor_class";
175+
public static final String HIVE_USE_JDBC = "hoodie.datasource.hive_sync.use_jdbc";
175176
public static final String HIVE_SYNC_MODE = "hoodie.datasource.hive_sync.mode";
176177
public static final String HIVE_AUTO_CREATE_DATABASE = "hoodie.datasource.hive_sync.auto_create_database";
177178
public static final String HIVE_IGNORE_EXCEPTIONS = "hoodie.datasource.hive_sync.ignore_exceptions";

0 commit comments

Comments
 (0)