Skip to content

Support new data type: STRING, BLOB, TIMESTAMP, DATE #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JackieTien97 merged 4 commits into from
May 16, 2024
Merged

Support new data type: STRING, BLOB, TIMESTAMP, DATE #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4001b17
done
Cpaulyz May 13, 2024
5d2ab11
fix ci
Cpaulyz May 13, 2024
0da6911
done
Cpaulyz May 16, 2024
75a566c
add license
Cpaulyz May 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 34 additions & 1 deletion java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,20 @@ public enum TSDataType {
VECTOR((byte) 6),

/** UNKNOWN. */
UNKNOWN((byte) 7);
UNKNOWN((byte) 7),

/** TIMESTAMP. */
TIMESTAMP((byte) 8),

/** DATE. */
DATE((byte) 9),

/** BLOB. */
BLOB((byte) 10),

/** STRING */
STRING((byte) 11);
;

private final byte type;

Expand Down Expand Up @@ -90,6 +103,14 @@ public static TSDataType getTsDataType(byte type) {
return TSDataType.VECTOR;
case 7:
return TSDataType.UNKNOWN;
case 8:
return TSDataType.TIMESTAMP;
case 9:
return TSDataType.DATE;
case 10:
return TSDataType.BLOB;
case 11:
return TSDataType.STRING;
default:
throw new IllegalArgumentException("Invalid input: " + type);
}
Expand Down Expand Up @@ -125,12 +146,16 @@ public int getDataTypeSize() {
return 1;
case INT32:
case FLOAT:
case DATE:
return 4;
// For text: return the size of reference here
case TEXT:
case INT64:
case DOUBLE:
case VECTOR:
case BLOB:
case STRING:
case TIMESTAMP:
return 8;
default:
throw new UnSupportedDataTypeException(this.toString());
Expand Down Expand Up @@ -160,6 +185,10 @@ public boolean isNumeric() {
case DOUBLE:
return true;
// For text: return the size of reference here
case BLOB:
case TIMESTAMP:
case DATE:
case STRING:
case BOOLEAN:
case TEXT:
case VECTOR:
Expand All @@ -183,8 +212,12 @@ public boolean isComparable() {
case DOUBLE:
case TEXT:
case BOOLEAN:
case TIMESTAMP:
case DATE:
case STRING:
return true;
case VECTOR:
case BLOB:
return false;
default:
throw new UnSupportedDataTypeException(this.toString());
Expand Down
8 changes: 8 additions & 0 deletions java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,18 @@ public static TsPrimitiveType getByType(TSDataType dataType) {
case BOOLEAN:
return new TsPrimitiveType.TsBoolean();
case INT32:
case DATE:
return new TsPrimitiveType.TsInt();
case INT64:
case TIMESTAMP:
return new TsPrimitiveType.TsLong();
case FLOAT:
return new TsPrimitiveType.TsFloat();
case DOUBLE:
return new TsPrimitiveType.TsDouble();
case TEXT:
case BLOB:
case STRING:
return new TsPrimitiveType.TsBinary();
case VECTOR:
return new TsPrimitiveType.TsVector();
Expand All @@ -63,14 +67,18 @@ public static TsPrimitiveType getByType(TSDataType dataType, Object v) {
case BOOLEAN:
return new TsPrimitiveType.TsBoolean((boolean) v);
case INT32:
case DATE:
return new TsPrimitiveType.TsInt((int) v);
case INT64:
case TIMESTAMP:
return new TsPrimitiveType.TsLong((long) v);
case FLOAT:
return new TsPrimitiveType.TsFloat((float) v);
case DOUBLE:
return new TsPrimitiveType.TsDouble((double) v);
case TEXT:
case BLOB:
case STRING:
return new TsPrimitiveType.TsBinary((Binary) v);
case VECTOR:
return new TsPrimitiveType.TsVector((TsPrimitiveType[]) v);
Expand Down
16 changes: 16 additions & 0 deletions java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,11 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType)
switch (dataType) {
case BOOLEAN:
case INT32:
case DATE:
return new IntRleDecoder();
case INT64:
case VECTOR:
case TIMESTAMP:
return new LongRleDecoder();
case FLOAT:
case DOUBLE:
Expand All @@ -67,9 +69,11 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType)
case TS_2DIFF:
switch (dataType) {
case INT32:
case DATE:
return new DeltaBinaryDecoder.IntDeltaDecoder();
case INT64:
case VECTOR:
case TIMESTAMP:
return new DeltaBinaryDecoder.LongDeltaDecoder();
case FLOAT:
case DOUBLE:
Expand All @@ -89,9 +93,11 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType)
case REGULAR:
switch (dataType) {
case INT32:
case DATE:
return new RegularDataDecoder.IntRegularDecoder();
case INT64:
case VECTOR:
case TIMESTAMP:
return new RegularDataDecoder.LongRegularDecoder();
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
Expand All @@ -103,9 +109,11 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType)
case DOUBLE:
return new DoublePrecisionDecoderV2();
case INT32:
case DATE:
return new IntGorillaDecoder();
case INT64:
case VECTOR:
case TIMESTAMP:
return new LongGorillaDecoder();
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
Expand All @@ -115,8 +123,10 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType)
case ZIGZAG:
switch (dataType) {
case INT32:
case DATE:
return new IntZigzagDecoder();
case INT64:
case TIMESTAMP:
return new LongZigzagDecoder();
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
Expand All @@ -128,18 +138,22 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType)
case DOUBLE:
return new DoublePrecisionChimpDecoder();
case INT32:
case DATE:
return new IntChimpDecoder();
case INT64:
case VECTOR:
case TIMESTAMP:
return new LongChimpDecoder();
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
}
case SPRINTZ:
switch (dataType) {
case INT32:
case DATE:
return new IntSprintzDecoder();
case INT64:
case TIMESTAMP:
return new LongSprintzDecoder();
case FLOAT:
return new FloatSprintzDecoder();
Expand All @@ -151,8 +165,10 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType)
case RLBE:
switch (dataType) {
case INT32:
case DATE:
return new IntRLBEDecoder();
case INT64:
case TIMESTAMP:
return new LongRLBEDecoder();
case FLOAT:
return new FloatRLBEDecoder();
Expand Down
4 changes: 4 additions & 0 deletions java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,14 +116,18 @@ public int getOneItemMaxSize() {
case BOOLEAN:
return 1;
case INT32:
case DATE:
return 4;
case INT64:
case TIMESTAMP:
return 8;
case FLOAT:
return 4;
case DOUBLE:
return 8;
case TEXT:
case STRING:
case BLOB:
// refer to encode(Binary,ByteArrayOutputStream)
return 4 + TSFileConfig.BYTE_SIZE_PER_CHAR * maxStringLength;
default:
Expand Down
16 changes: 16 additions & 0 deletions java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,11 @@ public static class Rle extends TSEncodingBuilder {
public Encoder getEncoder(TSDataType type) {
switch (type) {
case INT32:
case DATE:
case BOOLEAN:
return new IntRleEncoder();
case INT64:
case TIMESTAMP:
return new LongRleEncoder();
case FLOAT:
case DOUBLE:
Expand Down Expand Up @@ -196,8 +198,10 @@ public static class Ts2Diff extends TSEncodingBuilder {
public Encoder getEncoder(TSDataType type) {
switch (type) {
case INT32:
case DATE:
return new DeltaBinaryEncoder.IntDeltaEncoder();
case INT64:
case TIMESTAMP:
return new DeltaBinaryEncoder.LongDeltaEncoder();
case FLOAT:
case DOUBLE:
Expand Down Expand Up @@ -268,8 +272,10 @@ public static class Regular extends TSEncodingBuilder {
public Encoder getEncoder(TSDataType type) {
switch (type) {
case INT32:
case DATE:
return new RegularDataEncoder.IntRegularEncoder();
case INT64:
case TIMESTAMP:
return new RegularDataEncoder.LongRegularEncoder();
default:
throw new UnSupportedDataTypeException("REGULAR doesn't support data type: " + type);
Expand All @@ -293,8 +299,10 @@ public Encoder getEncoder(TSDataType type) {
case DOUBLE:
return new DoublePrecisionEncoderV2();
case INT32:
case DATE:
return new IntGorillaEncoder();
case INT64:
case TIMESTAMP:
return new LongGorillaEncoder();
default:
throw new UnSupportedDataTypeException("GORILLA doesn't support data type: " + type);
Expand All @@ -312,8 +320,10 @@ public static class Sprintz extends TSEncodingBuilder {
public Encoder getEncoder(TSDataType type) {
switch (type) {
case INT32:
case DATE:
return new IntSprintzEncoder();
case INT64:
case TIMESTAMP:
return new LongSprintzEncoder();
case FLOAT:
return new FloatSprintzEncoder();
Expand All @@ -335,8 +345,10 @@ public static class RLBE extends TSEncodingBuilder {
public Encoder getEncoder(TSDataType type) {
switch (type) {
case INT32:
case DATE:
return new IntRLBE();
case INT64:
case TIMESTAMP:
return new LongRLBE();
case FLOAT:
return new FloatRLBE();
Expand Down Expand Up @@ -375,8 +387,10 @@ public static class Zigzag extends TSEncodingBuilder {
public Encoder getEncoder(TSDataType type) {
switch (type) {
case INT32:
case DATE:
return new IntZigzagEncoder();
case INT64:
case TIMESTAMP:
return new LongZigzagEncoder();
default:
throw new UnSupportedDataTypeException("ZIGZAG doesn't support data type: " + type);
Expand All @@ -400,8 +414,10 @@ public Encoder getEncoder(TSDataType type) {
case DOUBLE:
return new DoublePrecisionChimpEncoder();
case INT32:
case DATE:
return new IntChimpEncoder();
case INT64:
case TIMESTAMP:
return new LongChimpEncoder();
default:
throw new UnSupportedDataTypeException("CHIMP doesn't support data type: " + type);
Expand Down
Loading