Skip to content

[HUDI-4201] Cli tool to get warned about empty non-completed instants from timeline #6867

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nsivabalan merged 5 commits into from
Oct 25, 2022
Merged

[HUDI-4201] Cli tool to get warned about empty non-completed instants from timeline #6867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1567926
current state, doesn't pass tests
Oct 1, 2022
87c1fe4
test is now working. I was using Log instead of LOG
Oct 4, 2022
69462fe
Merge branch 'apache:master' into cli_tool_delete_noncomplete_instants
jonvex Oct 24, 2022
eade6c1
changed name to reflect what the command is doing
Oct 24, 2022
a869c54
fixed checkstyle issues
Oct 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@

package org.apache.hudi.cli.commands;

import org.apache.avro.AvroRuntimeException;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.cli.DeDupeType;
import org.apache.hudi.cli.HoodieCLI;
import org.apache.hudi.cli.HoodiePrintHelper;
Expand All @@ -38,14 +36,16 @@
import org.apache.hudi.common.util.PartitionPathEncodeUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieIOException;

import org.apache.avro.AvroRuntimeException;
import org.apache.hadoop.fs.Path;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.spark.launcher.SparkLauncher;
import org.apache.spark.util.Utils;
import org.springframework.shell.standard.ShellComponent;
import org.springframework.shell.standard.ShellMethod;
import org.springframework.shell.standard.ShellOption;
import scala.collection.JavaConverters;

import java.io.FileInputStream;
import java.io.IOException;
Expand All @@ -55,6 +55,8 @@
import java.util.TreeSet;
import java.util.stream.Collectors;

import scala.collection.JavaConverters;

import static org.apache.hudi.common.table.HoodieTableMetaClient.METAFOLDER_NAME;

/**
Expand Down Expand Up @@ -205,6 +207,13 @@ public void removeCorruptedPendingCleanAction() {
});
}

@ShellMethod(key = "repair show empty commit metadata", value = "show failed commits")
public void showFailedCommits() {
HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
activeTimeline.filterCompletedInstants().getInstants().filter(activeTimeline::isEmpty).forEach(hoodieInstant -> LOG.warn("Empty Commit: " + hoodieInstant.toString()));
}

@ShellMethod(key = "repair migrate-partition-meta", value = "Migrate all partition meta file currently stored in text format "
+ "to be stored in base file format. See HoodieTableConfig#PARTITION_METAFILE_USE_DATA_FORMAT.")
public String migratePartitionMeta(
Expand Down
69 changes: 69 additions & 0 deletions hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.appender.AbstractAppender;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.SQLContext;
import org.junit.jupiter.api.AfterEach;
Expand All @@ -54,6 +57,8 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.shell.Shell;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.Logger;

import java.io.FileInputStream;
import java.io.IOException;
Expand All @@ -65,6 +70,7 @@
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import java.util.stream.Collectors;

import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER;
Expand Down Expand Up @@ -259,6 +265,50 @@ public void testRemoveCorruptedPendingCleanAction() throws IOException {
assertEquals(0, metaClient.getActiveTimeline().filterInflightsAndRequested().getInstants().count());
}

/**
* Testcase for "repair cleanup empty commit metadata"
*
*/
@Test
public void testShowFailedCommits() {
HoodieCLI.conf = hadoopConf();

Configuration conf = HoodieCLI.conf;

HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();

for (int i = 1; i < 20; i++) {
String timestamp = String.valueOf(i);
// Write corrupted requested Clean File
HoodieTestCommitMetadataGenerator.createCommitFile(tablePath, timestamp, conf);
}

metaClient.getActiveTimeline().getInstants().filter(hoodieInstant -> Integer.parseInt(hoodieInstant.getTimestamp()) % 4 == 0).forEach(hoodieInstant -> {
metaClient.getActiveTimeline().deleteInstantFileIfExists(hoodieInstant);
metaClient.getActiveTimeline().createNewInstant(hoodieInstant);
});

final TestLogAppender appender = new TestLogAppender();
final Logger logger = (Logger) LogManager.getLogger(RepairsCommand.class);
try {
appender.start();
logger.addAppender(appender);
Object result = shell.evaluate(() -> "repair show empty commit metadata");
assertTrue(ShellEvaluationResultUtil.isSuccess(result));
final List<LogEvent> log = appender.getLog();
assertEquals(log.size(),4);
log.forEach(LoggingEvent -> {
assertEquals(LoggingEvent.getLevel(), Level.WARN);
assertTrue(LoggingEvent.getMessage().getFormattedMessage().contains("Empty Commit: "));
assertTrue(LoggingEvent.getMessage().getFormattedMessage().contains("COMPLETED]"));
});
} finally {
logger.removeAppender(appender);
}


}

@Test
public void testRepairDeprecatedPartition() throws IOException {
tablePath = tablePath + "/repair_test/";
Expand Down Expand Up @@ -374,4 +424,23 @@ public void testRenamePartition() throws IOException {
assertEquals(totalRecs, totalRecsInOldPartition);
}
}

class TestLogAppender extends AbstractAppender {
private final List<LogEvent> log = new ArrayList<>();

protected TestLogAppender() {
super(UUID.randomUUID().toString(), null, null, false, null);
}

@Override
public void append(LogEvent event) {
log.add(event);
}

public List<LogEvent> getLog() {
return new ArrayList<LogEvent>(log);
}
}
}