Skip to content

Custom line‑ending characters for CSV Import #1929

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
work-lto wants to merge 1 commit into
base: releases/24.2
Choose a base branch
from
+37 −3
Open

Custom line‑ending characters for CSV Import #1929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions ...pse.scout.rt.shared.test/src/test/java/org/eclipse/scout/rt/shared/csv/CsvHelperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
Expand Down Expand Up @@ -301,6 +302,26 @@ public void testImportWithoutHeaderRowsAndUtfWithBomEncoding() throws IOExceptio
}
}

@Test
public void testEndOfLineDefault() {
CsvHelper csvHelper = new CsvHelper();
String testInputLine = "1,2,3#4,5,6\n7,8,9,10,11";
Object[][] expectedData = new Object[][]{{"1", "2", "3#4", "5", "6"}, {"7", "8", "9", "10", "11"}};

Object[][] actualData = csvHelper.importData(new StringReader(testInputLine), 0, null, Integer.MAX_VALUE);
assertEquals(expectedData, actualData);
}

@Test
public void testEndOfLine() {
CsvHelper csvHelper = new CsvHelper(null, (char) 0, (char) 0, null, ch -> ch == '#');
String testInputLine = "1;2;3#4;5;6\n7;8;9;10;11";
Object[][] expectedData = new Object[][]{{"1", "2", "3"}, {"4", "5", "6\n7", "8", "9", "10", "11"}};

Object[][] actualData = csvHelper.importData(new StringReader(testInputLine), 0, null, Integer.MAX_VALUE);
assertEquals(expectedData, actualData);
}

protected void addIfSupported(String charset, List<Charset> charsets) {
if (Charset.isSupported(charset)) {
charsets.add(Charset.forName(charset));
Expand Down
19 changes: 16 additions & 3 deletions org.eclipse.scout.rt.shared/src/main/java/org/eclipse/scout/rt/shared/csv/CsvHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,14 @@
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.function.Function;

import org.eclipse.scout.rt.platform.BEANS;
import org.eclipse.scout.rt.platform.exception.PlatformExceptionTranslator;
import org.eclipse.scout.rt.platform.exception.ProcessingException;
import org.eclipse.scout.rt.platform.nls.NlsLocale;
import org.eclipse.scout.rt.platform.util.BomInputStreamReader;
import org.eclipse.scout.rt.platform.util.BooleanUtility;
import org.eclipse.scout.rt.platform.util.CollectionUtility;
import org.eclipse.scout.rt.platform.util.NumberFormatProvider;
import org.eclipse.scout.rt.platform.util.StringUtility;
Expand All @@ -54,11 +56,13 @@ public class CsvHelper {
*/
public static final String IGNORED_COLUMN_NAME = "null";
private static final Logger LOG = LoggerFactory.getLogger(CsvHelper.class);
public static final Function<Character, Boolean> DEFAULT_NEW_LINE_FUNCTION = ch -> ch == '\n' || ch == '\r';

private final Locale m_locale;
private final char m_separatorChar;// ";"
private final char m_textDelimiterChar;// "\""
private final String m_lineSeparator;// "\n"
private final Function<Character, Boolean> m_isEndOfLine; // ch -> ch == '\n' || ch == '\r'
private int m_colCount;
private List<String> m_colNames;
private List<String> m_colTypes;
Expand All @@ -75,11 +79,16 @@ public CsvHelper(Locale locale, String separatorChar, String textDelimiterChar,
}

public CsvHelper(Locale locale, char separatorChar, char textDelimiterChar, String lineSeparator) {
this(locale, separatorChar, textDelimiterChar, lineSeparator, DEFAULT_NEW_LINE_FUNCTION);
}

public CsvHelper(Locale locale, char separatorChar, char textDelimiterChar, String lineSeparator, Function<Character, Boolean> isEndOfLine) {
m_locale = locale == null ? NlsLocale.get() : locale;
m_separatorChar = separatorChar != 0x00 ? separatorChar : ';';
m_textDelimiterChar = textDelimiterChar != 0x00 ? textDelimiterChar : '"';
m_lineSeparator = lineSeparator != null ? lineSeparator : "\n";
m_colFormat = new ArrayList<>();
m_isEndOfLine = isEndOfLine != null ? isEndOfLine : DEFAULT_NEW_LINE_FUNCTION;
}

public Locale getLocale() {
Expand All @@ -102,6 +111,10 @@ public boolean isEncodeLineSeparator() {
return m_encodeLineSeparator;
}

public boolean isEndOfLine(char ch) {
return BooleanUtility.nvl(m_isEndOfLine.apply(ch));
}

/**
* Configures whether a text containing the line separator should be encoded.
*/
Expand Down Expand Up @@ -426,7 +439,7 @@ protected List<String> importRow(Reader reader) throws IOException {
String token;
int ch = reader.read();

while (ch == '\n' || ch == '\r') {
while (isEndOfLine((char) ch)) {
ch = reader.read();
}
if (ch < 0) {
Expand All @@ -443,7 +456,7 @@ protected List<String> importRow(Reader reader) throws IOException {
curBuf.append((char) ch);
}
else {// ch<0 or out of string or end of line
if (ch == getSeparatorChar() || ch < 0 || ch == '\n' || ch == '\r') {
if (ch == getSeparatorChar() || ch < 0 || isEndOfLine((char) ch)) {
// consume token
token = curBuf.toString();
curBuf.setLength(0);
Expand All @@ -464,7 +477,7 @@ protected List<String> importRow(Reader reader) throws IOException {
token = decodeText(token);
cellList.add(token);
// check if end of current line
if (ch < 0 || ch == '\n' || ch == '\r') {
if (ch < 0 || isEndOfLine((char) ch)) {
break;
}
}
Expand Down