Skip to content

[HUDI-3193] Decouple hudi-aws from hudi-client-common #5666

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
codope merged 1 commit into from
May 25, 2022
Merged

[HUDI-3193] Decouple hudi-aws from hudi-client-common #5666

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public static Builder forRegistry(MetricRegistry registry) {
}

public static class Builder {
private MetricRegistry registry;
private final MetricRegistry registry;
private Clock clock;
private String prefix;
private TimeUnit rateUnit;
Expand Down
1 change: 1 addition & 0 deletions hudi-client/hudi-client-common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-aws</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
Expand Down
1 change: 1 addition & 0 deletions hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.metrics.HoodieMetricsCloudWatchConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig;
import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig;
Expand Down
47 changes: 24 additions & 23 deletions ...config/HoodieMetricsCloudWatchConfig.java → ...etrics/HoodieMetricsCloudWatchConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,17 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.hudi.config;
package org.apache.hudi.config.metrics;

import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
Expand All @@ -36,22 +37,22 @@ public class HoodieMetricsCloudWatchConfig extends HoodieConfig {
public static final String CLOUDWATCH_PREFIX = "hoodie.metrics.cloudwatch";

public static final ConfigProperty<Integer> REPORT_PERIOD_SECONDS = ConfigProperty
.key(CLOUDWATCH_PREFIX + ".report.period.seconds")
.defaultValue(60)
.sinceVersion("0.10.0")
.withDocumentation("Reporting interval in seconds");
.key(CLOUDWATCH_PREFIX + ".report.period.seconds")
.defaultValue(60)
.sinceVersion("0.10.0")
.withDocumentation("Reporting interval in seconds");

public static final ConfigProperty<String> METRIC_PREFIX = ConfigProperty
.key(CLOUDWATCH_PREFIX + ".metric.prefix")
.defaultValue("")
.sinceVersion("0.10.0")
.withDocumentation("Metric prefix of reporter");
.key(CLOUDWATCH_PREFIX + ".metric.prefix")
.defaultValue("")
.sinceVersion("0.10.0")
.withDocumentation("Metric prefix of reporter");

public static final ConfigProperty<String> METRIC_NAMESPACE = ConfigProperty
.key(CLOUDWATCH_PREFIX + ".namespace")
.defaultValue("Hudi")
.sinceVersion("0.10.0")
.withDocumentation("Namespace of reporter");
.key(CLOUDWATCH_PREFIX + ".namespace")
.defaultValue("Hudi")
.sinceVersion("0.10.0")
.withDocumentation("Namespace of reporter");
/*
Amazon CloudWatch allows a maximum of 20 metrics per request. Choosing this as the default maximum.
Reference: https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html
Expand All @@ -66,13 +67,13 @@ public HoodieMetricsCloudWatchConfig() {
super();
}

public static HoodieMetricsCloudWatchConfig.Builder newBuilder() {
return new HoodieMetricsCloudWatchConfig.Builder();
static Builder newBuilder() {
return new Builder();
}

public static class Builder {
static class Builder {

private HoodieMetricsCloudWatchConfig hoodieMetricsCloudWatchConfig = new HoodieMetricsCloudWatchConfig();
private final HoodieMetricsCloudWatchConfig hoodieMetricsCloudWatchConfig = new HoodieMetricsCloudWatchConfig();

public HoodieMetricsCloudWatchConfig.Builder fromProperties(Properties props) {
this.hoodieMetricsCloudWatchConfig.getProps().putAll(props);
Expand Down
1 change: 0 additions & 1 deletion .../hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieMetricsCloudWatchConfig;
import org.apache.hudi.metrics.MetricsReporterType;

import javax.annotation.concurrent.Immutable;
Expand Down
6 changes: 6 additions & 0 deletions hudi-flink-datasource/hudi-flink/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,12 @@
<artifactId>hudi-client-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-aws</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
Copy link
Contributor

@danny0405 danny0405 May 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We still add a dependency for the hudi-client-common, if we want to avoid clazz conflicts, a better solution is to create a shaded module for these vendor jars.

Copy link
Member Author

@codope codope May 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was going to take it up after this patch lands. HUDI-4011

Copy link
Member Author

@codope codope May 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The provided dependencies in hudi-client-common should not be included in hudi-flink.

<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-flink-client</artifactId>
Expand Down