diff --git a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java index efb8fa027..226c2e099 100644 --- a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java +++ b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java @@ -50,7 +50,20 @@ public enum TSDataType { VECTOR((byte) 6), /** UNKNOWN. */ - UNKNOWN((byte) 7); + UNKNOWN((byte) 7), + + /** TIMESTAMP. */ + TIMESTAMP((byte) 8), + + /** DATE. */ + DATE((byte) 9), + + /** BLOB. */ + BLOB((byte) 10), + + /** STRING */ + STRING((byte) 11); + ; private final byte type; @@ -90,6 +103,14 @@ public static TSDataType getTsDataType(byte type) { return TSDataType.VECTOR; case 7: return TSDataType.UNKNOWN; + case 8: + return TSDataType.TIMESTAMP; + case 9: + return TSDataType.DATE; + case 10: + return TSDataType.BLOB; + case 11: + return TSDataType.STRING; default: throw new IllegalArgumentException("Invalid input: " + type); } @@ -125,12 +146,16 @@ public int getDataTypeSize() { return 1; case INT32: case FLOAT: + case DATE: return 4; // For text: return the size of reference here case TEXT: case INT64: case DOUBLE: case VECTOR: + case BLOB: + case STRING: + case TIMESTAMP: return 8; default: throw new UnSupportedDataTypeException(this.toString()); @@ -160,6 +185,10 @@ public boolean isNumeric() { case DOUBLE: return true; // For text: return the size of reference here + case BLOB: + case TIMESTAMP: + case DATE: + case STRING: case BOOLEAN: case TEXT: case VECTOR: @@ -183,8 +212,12 @@ public boolean isComparable() { case DOUBLE: case TEXT: case BOOLEAN: + case TIMESTAMP: + case DATE: + case STRING: return true; case VECTOR: + case BLOB: return false; default: throw new UnSupportedDataTypeException(this.toString()); diff --git a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java index 62e2b1eb9..50a744365 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java @@ -36,14 +36,18 @@ public static TsPrimitiveType getByType(TSDataType dataType) { case BOOLEAN: return new TsPrimitiveType.TsBoolean(); case INT32: + case DATE: return new TsPrimitiveType.TsInt(); case INT64: + case TIMESTAMP: return new TsPrimitiveType.TsLong(); case FLOAT: return new TsPrimitiveType.TsFloat(); case DOUBLE: return new TsPrimitiveType.TsDouble(); case TEXT: + case BLOB: + case STRING: return new TsPrimitiveType.TsBinary(); case VECTOR: return new TsPrimitiveType.TsVector(); @@ -63,14 +67,18 @@ public static TsPrimitiveType getByType(TSDataType dataType, Object v) { case BOOLEAN: return new TsPrimitiveType.TsBoolean((boolean) v); case INT32: + case DATE: return new TsPrimitiveType.TsInt((int) v); case INT64: + case TIMESTAMP: return new TsPrimitiveType.TsLong((long) v); case FLOAT: return new TsPrimitiveType.TsFloat((float) v); case DOUBLE: return new TsPrimitiveType.TsDouble((double) v); case TEXT: + case BLOB: + case STRING: return new TsPrimitiveType.TsBinary((Binary) v); case VECTOR: return new TsPrimitiveType.TsVector((TsPrimitiveType[]) v); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java index f1404ae1f..36c3d8261 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/Decoder.java @@ -54,9 +54,11 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) switch (dataType) { case BOOLEAN: case INT32: + case DATE: return new IntRleDecoder(); case INT64: case VECTOR: + case TIMESTAMP: return new LongRleDecoder(); case FLOAT: case DOUBLE: @@ -67,9 +69,11 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) case TS_2DIFF: switch (dataType) { case INT32: + case DATE: return new DeltaBinaryDecoder.IntDeltaDecoder(); case INT64: case VECTOR: + case TIMESTAMP: return new DeltaBinaryDecoder.LongDeltaDecoder(); case FLOAT: case DOUBLE: @@ -89,9 +93,11 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) case REGULAR: switch (dataType) { case INT32: + case DATE: return new RegularDataDecoder.IntRegularDecoder(); case INT64: case VECTOR: + case TIMESTAMP: return new RegularDataDecoder.LongRegularDecoder(); default: throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType)); @@ -103,9 +109,11 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) case DOUBLE: return new DoublePrecisionDecoderV2(); case INT32: + case DATE: return new IntGorillaDecoder(); case INT64: case VECTOR: + case TIMESTAMP: return new LongGorillaDecoder(); default: throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType)); @@ -115,8 +123,10 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) case ZIGZAG: switch (dataType) { case INT32: + case DATE: return new IntZigzagDecoder(); case INT64: + case TIMESTAMP: return new LongZigzagDecoder(); default: throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType)); @@ -128,9 +138,11 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) case DOUBLE: return new DoublePrecisionChimpDecoder(); case INT32: + case DATE: return new IntChimpDecoder(); case INT64: case VECTOR: + case TIMESTAMP: return new LongChimpDecoder(); default: throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType)); @@ -138,8 +150,10 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) case SPRINTZ: switch (dataType) { case INT32: + case DATE: return new IntSprintzDecoder(); case INT64: + case TIMESTAMP: return new LongSprintzDecoder(); case FLOAT: return new FloatSprintzDecoder(); @@ -151,8 +165,10 @@ public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) case RLBE: switch (dataType) { case INT32: + case DATE: return new IntRLBEDecoder(); case INT64: + case TIMESTAMP: return new LongRLBEDecoder(); case FLOAT: return new FloatRLBEDecoder(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java index e93f1f080..09c28cbef 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java @@ -116,14 +116,18 @@ public int getOneItemMaxSize() { case BOOLEAN: return 1; case INT32: + case DATE: return 4; case INT64: + case TIMESTAMP: return 8; case FLOAT: return 4; case DOUBLE: return 8; case TEXT: + case STRING: + case BLOB: // refer to encode(Binary,ByteArrayOutputStream) return 4 + TSFileConfig.BYTE_SIZE_PER_CHAR * maxStringLength; default: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java index 65ef569be..232ddd3c1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -142,9 +142,11 @@ public static class Rle extends TSEncodingBuilder { public Encoder getEncoder(TSDataType type) { switch (type) { case INT32: + case DATE: case BOOLEAN: return new IntRleEncoder(); case INT64: + case TIMESTAMP: return new LongRleEncoder(); case FLOAT: case DOUBLE: @@ -196,8 +198,10 @@ public static class Ts2Diff extends TSEncodingBuilder { public Encoder getEncoder(TSDataType type) { switch (type) { case INT32: + case DATE: return new DeltaBinaryEncoder.IntDeltaEncoder(); case INT64: + case TIMESTAMP: return new DeltaBinaryEncoder.LongDeltaEncoder(); case FLOAT: case DOUBLE: @@ -268,8 +272,10 @@ public static class Regular extends TSEncodingBuilder { public Encoder getEncoder(TSDataType type) { switch (type) { case INT32: + case DATE: return new RegularDataEncoder.IntRegularEncoder(); case INT64: + case TIMESTAMP: return new RegularDataEncoder.LongRegularEncoder(); default: throw new UnSupportedDataTypeException("REGULAR doesn't support data type: " + type); @@ -293,8 +299,10 @@ public Encoder getEncoder(TSDataType type) { case DOUBLE: return new DoublePrecisionEncoderV2(); case INT32: + case DATE: return new IntGorillaEncoder(); case INT64: + case TIMESTAMP: return new LongGorillaEncoder(); default: throw new UnSupportedDataTypeException("GORILLA doesn't support data type: " + type); @@ -312,8 +320,10 @@ public static class Sprintz extends TSEncodingBuilder { public Encoder getEncoder(TSDataType type) { switch (type) { case INT32: + case DATE: return new IntSprintzEncoder(); case INT64: + case TIMESTAMP: return new LongSprintzEncoder(); case FLOAT: return new FloatSprintzEncoder(); @@ -335,8 +345,10 @@ public static class RLBE extends TSEncodingBuilder { public Encoder getEncoder(TSDataType type) { switch (type) { case INT32: + case DATE: return new IntRLBE(); case INT64: + case TIMESTAMP: return new LongRLBE(); case FLOAT: return new FloatRLBE(); @@ -375,8 +387,10 @@ public static class Zigzag extends TSEncodingBuilder { public Encoder getEncoder(TSDataType type) { switch (type) { case INT32: + case DATE: return new IntZigzagEncoder(); case INT64: + case TIMESTAMP: return new LongZigzagEncoder(); default: throw new UnSupportedDataTypeException("ZIGZAG doesn't support data type: " + type); @@ -400,8 +414,10 @@ public Encoder getEncoder(TSDataType type) { case DOUBLE: return new DoublePrecisionChimpEncoder(); case INT32: + case DATE: return new IntChimpEncoder(); case INT64: + case TIMESTAMP: return new LongChimpEncoder(); default: throw new UnSupportedDataTypeException("CHIMP doesn't support data type: " + type); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BlobStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BlobStatistics.java new file mode 100644 index 000000000..c0f3d2bcb --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BlobStatistics.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.file.metadata.statistics; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.filter.StatisticsClassException; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +public class BlobStatistics extends Statistics { + + public static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(BlobStatistics.class); + + // no statistics for blob data type + + @Override + public TSDataType getType() { + return TSDataType.BLOB; + } + + /** The output of this method should be identical to the method "serializeStats(outputStream)". */ + @Override + public int getStatsSize() { + return 0; + } + + @Override + public long getRetainedSizeInBytes() { + return INSTANCE_SIZE; + } + + @Override + int serializeStats(OutputStream outputStream) throws IOException { + return 0; + } + + public void updateStats(Binary value) { + // do nothing + } + + @Override + public void deserialize(InputStream inputStream) throws IOException { + // do nothing + } + + @Override + public void deserialize(ByteBuffer byteBuffer) { + // do nothing + } + + @Override + public Binary getMinValue() { + throw new StatisticsClassException( + String.format(STATS_UNSUPPORTED_MSG, TSDataType.BLOB, "min")); + } + + @Override + public Binary getMaxValue() { + throw new StatisticsClassException( + String.format(STATS_UNSUPPORTED_MSG, TSDataType.BLOB, "max")); + } + + @Override + public Binary getFirstValue() { + throw new StatisticsClassException( + String.format(STATS_UNSUPPORTED_MSG, TSDataType.BLOB, "first")); + } + + @Override + public Binary getLastValue() { + throw new StatisticsClassException( + String.format(STATS_UNSUPPORTED_MSG, TSDataType.BLOB, "last")); + } + + @Override + public double getSumDoubleValue() { + throw new StatisticsClassException( + String.format(STATS_UNSUPPORTED_MSG, TSDataType.BLOB, "sum")); + } + + @Override + public long getSumLongValue() { + + throw new StatisticsClassException( + String.format(STATS_UNSUPPORTED_MSG, TSDataType.BLOB, "sum")); + } + + @Override + protected void mergeStatisticsValue(Statistics stats) { + // do nothing + } + + @Override + public String toString() { + return "BlobStatistics{}"; + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/DateStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/DateStatistics.java new file mode 100644 index 000000000..d79e31fd1 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/DateStatistics.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.file.metadata.statistics; + +import org.apache.tsfile.enums.TSDataType; + +public class DateStatistics extends IntegerStatistics { + @Override + public TSDataType getType() { + return TSDataType.DATE; + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java index 4f92b4553..addec4262 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java @@ -85,6 +85,14 @@ public static Statistics getStatsByType(TSDataType type) return new FloatStatistics(); case VECTOR: return new TimeStatistics(); + case DATE: + return new DateStatistics(); + case TIMESTAMP: + return new TimestampStatistics(); + case STRING: + return new StringStatistics(); + case BLOB: + return new BlobStatistics(); default: throw new UnknownColumnTypeException(type.toString()); } @@ -106,6 +114,14 @@ public static long getSizeByType(TSDataType type) { return FloatStatistics.INSTANCE_SIZE; case VECTOR: return TimeStatistics.INSTANCE_SIZE; + case DATE: + return DateStatistics.INSTANCE_SIZE; + case TIMESTAMP: + return TimestampStatistics.INSTANCE_SIZE; + case STRING: + return StringStatistics.INSTANCE_SIZE; + case BLOB: + return BlobStatistics.INSTANCE_SIZE; default: throw new UnknownColumnTypeException(type.toString()); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/StringStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/StringStatistics.java new file mode 100644 index 000000000..de6cccc3b --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/StringStatistics.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.file.metadata.statistics; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.filter.StatisticsClassException; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Objects; + +import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfCharArray; + +public class StringStatistics extends Statistics { + public static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(StringStatistics.class) + + 4 * RamUsageEstimator.shallowSizeOfInstance(Binary.class); + + private static final Binary EMPTY_VALUE = new Binary("", TSFileConfig.STRING_CHARSET); + + private Binary firstValue = EMPTY_VALUE; + private Binary lastValue = EMPTY_VALUE; + private Binary minValue = EMPTY_VALUE; + private Binary maxValue = EMPTY_VALUE; + + @Override + public TSDataType getType() { + return TSDataType.BLOB; + } + + /** The output of this method should be identical to the method "serializeStats(outputStream)". */ + @Override + public int getStatsSize() { + return 4 * 4 + + firstValue.getValues().length + + lastValue.getValues().length + + minValue.getValues().length + + maxValue.getValues().length; + } + + @Override + public long getRetainedSizeInBytes() { + return INSTANCE_SIZE + + sizeOfCharArray(firstValue.getLength()) + + sizeOfCharArray(lastValue.getLength()) + + sizeOfCharArray(minValue.getLength()) + + sizeOfCharArray(maxValue.getLength()); + } + + public void initializeStats(Binary first, Binary last, Binary min, Binary max) { + this.firstValue = first; + this.lastValue = last; + this.minValue = min; + this.maxValue = max; + } + + private void updateStats(Binary minValue, Binary maxValue, Binary lastValue) { + if (this.minValue.compareTo(minValue) > 0) { + this.minValue = minValue; + } + if (this.maxValue.compareTo(maxValue) < 0) { + this.maxValue = maxValue; + } + this.lastValue = lastValue; + } + + private void updateStats( + Binary firstValue, + Binary lastValue, + Binary minValue, + Binary maxValue, + long startTime, + long endTime) { + // only if endTime greater or equals to the current endTime need we update the last value + // only if startTime less or equals to the current startTime need we update the first value + // otherwise, just ignore + if (this.minValue.compareTo(minValue) > 0) { + this.minValue = minValue; + } + if (this.maxValue.compareTo(maxValue) < 0) { + this.maxValue = maxValue; + } + if (startTime <= this.getStartTime()) { + this.firstValue = firstValue; + } + if (endTime >= this.getEndTime()) { + this.lastValue = lastValue; + } + } + + @Override + public Binary getMinValue() { + return minValue; + } + + @Override + public Binary getMaxValue() { + return maxValue; + } + + @Override + public Binary getFirstValue() { + return firstValue; + } + + @Override + public Binary getLastValue() { + return lastValue; + } + + @Override + public double getSumDoubleValue() { + throw new StatisticsClassException( + String.format(STATS_UNSUPPORTED_MSG, TSDataType.STRING, "double sum")); + } + + @Override + public long getSumLongValue() { + throw new StatisticsClassException( + String.format(STATS_UNSUPPORTED_MSG, TSDataType.STRING, "long sum")); + } + + @Override + protected void mergeStatisticsValue(Statistics stats) { + StringStatistics stringStats = (StringStatistics) stats; + if (isEmpty) { + initializeStats( + stringStats.getFirstValue(), + stringStats.getLastValue(), + stringStats.getMinValue(), + stringStats.getMaxValue()); + isEmpty = false; + } else { + updateStats( + stringStats.getFirstValue(), + stringStats.getLastValue(), + stringStats.getMinValue(), + stringStats.getMaxValue(), + stats.getStartTime(), + stats.getEndTime()); + } + } + + @Override + void updateStats(Binary value) { + if (isEmpty) { + initializeStats(value, value, value, value); + isEmpty = false; + } else { + updateStats(value, value, value); + } + } + + @Override + void updateStats(Binary[] values, int batchSize) { + for (int i = 0; i < batchSize; i++) { + updateStats(values[i]); + } + } + + @Override + public int serializeStats(OutputStream outputStream) throws IOException { + int byteLen = 0; + byteLen += ReadWriteIOUtils.write(firstValue, outputStream); + byteLen += ReadWriteIOUtils.write(lastValue, outputStream); + byteLen += ReadWriteIOUtils.write(minValue, outputStream); + byteLen += ReadWriteIOUtils.write(maxValue, outputStream); + return byteLen; + } + + @Override + public void deserialize(InputStream inputStream) throws IOException { + this.firstValue = ReadWriteIOUtils.readBinary(inputStream); + this.lastValue = ReadWriteIOUtils.readBinary(inputStream); + this.minValue = ReadWriteIOUtils.readBinary(inputStream); + this.maxValue = ReadWriteIOUtils.readBinary(inputStream); + } + + @Override + public void deserialize(ByteBuffer byteBuffer) { + this.firstValue = ReadWriteIOUtils.readBinary(byteBuffer); + this.lastValue = ReadWriteIOUtils.readBinary(byteBuffer); + this.minValue = ReadWriteIOUtils.readBinary(byteBuffer); + this.maxValue = ReadWriteIOUtils.readBinary(byteBuffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + StringStatistics that = (StringStatistics) o; + return firstValue.equals(that.firstValue) + && lastValue.equals(that.lastValue) + && minValue.equals(that.minValue) + && maxValue.equals(that.maxValue); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), firstValue, lastValue, minValue, maxValue); + } + + @Override + public String toString() { + return super.toString() + + " [firstValue:" + + firstValue + + ", lastValue:" + + lastValue + + ", minValue:" + + minValue + + ", maxValue:" + + maxValue + + "]"; + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/TimestampStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/TimestampStatistics.java new file mode 100644 index 000000000..8c634d122 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/TimestampStatistics.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.file.metadata.statistics; + +import org.apache.tsfile.enums.TSDataType; + +public class TimestampStatistics extends LongStatistics { + @Override + public TSDataType getType() { + return TSDataType.TIMESTAMP; + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index a15c98909..4c8ce31d9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -1839,9 +1839,11 @@ public long selfCheck( long timeStamp = timeBatch.get(0)[i]; switch (dataType) { case INT32: + case DATE: chunkStatistics.update(timeStamp, value.getInt()); break; case INT64: + case TIMESTAMP: chunkStatistics.update(timeStamp, value.getLong()); break; case FLOAT: @@ -1854,6 +1856,8 @@ public long selfCheck( chunkStatistics.update(timeStamp, value.getBoolean()); break; case TEXT: + case BLOB: + case STRING: chunkStatistics.update(timeStamp, value.getBinary()); break; default: @@ -1874,9 +1878,11 @@ public long selfCheck( while (batchData.hasCurrent()) { switch (dataType) { case INT32: + case DATE: chunkStatistics.update(batchData.currentTime(), batchData.getInt()); break; case INT64: + case TIMESTAMP: chunkStatistics.update(batchData.currentTime(), batchData.getLong()); break; case FLOAT: @@ -1889,6 +1895,8 @@ public long selfCheck( chunkStatistics.update(batchData.currentTime(), batchData.getBoolean()); break; case TEXT: + case BLOB: + case STRING: chunkStatistics.update(batchData.currentTime(), batchData.getBinary()); break; default: @@ -2086,9 +2094,11 @@ public long checkChunkAndPagesStatistics(IChunkMetadata chunkMetadata) throws IO while (batchData.hasCurrent()) { switch (dataType) { case INT32: + case DATE: chunkStatistics.update(batchData.currentTime(), batchData.getInt()); break; case INT64: + case TIMESTAMP: chunkStatistics.update(batchData.currentTime(), batchData.getLong()); break; case FLOAT: @@ -2101,6 +2111,8 @@ public long checkChunkAndPagesStatistics(IChunkMetadata chunkMetadata) throws IO chunkStatistics.update(batchData.currentTime(), batchData.getBoolean()); break; case TEXT: + case BLOB: + case STRING: chunkStatistics.update(batchData.currentTime(), batchData.getBinary()); break; default: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java index bd89bcc70..504f7a9cb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java @@ -130,8 +130,10 @@ public long currentTime() { public Object currentValue() { switch (dataType) { case INT32: + case DATE: return getInt(); case INT64: + case TIMESTAMP: return getLong(); case FLOAT: return getFloat(); @@ -140,6 +142,8 @@ public Object currentValue() { case BOOLEAN: return getBoolean(); case TEXT: + case BLOB: + case STRING: return getBinary(); case VECTOR: return getVector(); @@ -151,8 +155,10 @@ public Object currentValue() { public TsPrimitiveType currentTsPrimitiveType() { switch (dataType) { case INT32: + case DATE: return new TsInt(getInt()); case INT64: + case TIMESTAMP: return new TsLong(getLong()); case FLOAT: return new TsFloat(getFloat()); @@ -161,6 +167,8 @@ public TsPrimitiveType currentTsPrimitiveType() { case BOOLEAN: return new TsBoolean(getBoolean()); case TEXT: + case BLOB: + case STRING: return new TsBinary(getBinary()); case VECTOR: return new TsVector(getVector()); @@ -203,10 +211,12 @@ public void init(TSDataType type) { booleanRet.add(new boolean[capacity]); break; case INT32: + case DATE: intRet = new ArrayList<>(); intRet.add(new int[capacity]); break; case INT64: + case TIMESTAMP: longRet = new ArrayList<>(); longRet.add(new long[capacity]); break; @@ -219,6 +229,8 @@ public void init(TSDataType type) { doubleRet.add(new double[capacity]); break; case TEXT: + case BLOB: + case STRING: binaryRet = new ArrayList<>(); binaryRet.add(new Binary[capacity]); break; @@ -547,9 +559,11 @@ public void putAnObject(long t, Object v) { putBoolean(t, (boolean) v); break; case INT32: + case DATE: putInt(t, (int) v); break; case INT64: + case TIMESTAMP: putLong(t, (long) v); break; case FLOAT: @@ -559,6 +573,8 @@ public void putAnObject(long t, Object v) { putDouble(t, (double) v); break; case TEXT: + case BLOB: + case STRING: putBinary(t, (Binary) v); break; case VECTOR: @@ -681,6 +697,8 @@ public void serializeData(DataOutputStream outputStream) throws IOException { } break; case TEXT: + case BLOB: + case STRING: for (int i = 0; i < length(); i++) { outputStream.writeLong(getTimeByIndex(i)); Binary binary = getBinaryByIndex(i); @@ -689,12 +707,14 @@ public void serializeData(DataOutputStream outputStream) throws IOException { } break; case INT64: + case TIMESTAMP: for (int i = 0; i < length(); i++) { outputStream.writeLong(getTimeByIndex(i)); outputStream.writeLong(getLongByIndex(i)); } break; case INT32: + case DATE: for (int i = 0; i < length(); i++) { outputStream.writeLong(getTimeByIndex(i)); outputStream.writeInt(getIntByIndex(i)); @@ -722,14 +742,18 @@ public void serializeData(DataOutputStream outputStream) throws IOException { outputStream.writeFloat(value.getFloat()); break; case TEXT: + case BLOB: + case STRING: Binary binary = value.getBinary(); outputStream.writeInt(binary.getLength()); outputStream.write(binary.getValues()); break; case INT64: + case TIMESTAMP: outputStream.writeLong(value.getLong()); break; case INT32: + case DATE: outputStream.writeInt(value.getInt()); break; default: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java index 9b168f290..1fd65740c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java @@ -55,10 +55,12 @@ public DescReadWriteBatchData(TSDataType dataType) { booleanRet.add(new boolean[capacity]); break; case INT32: + case DATE: intRet = new LinkedList<>(); intRet.add(new int[capacity]); break; case INT64: + case TIMESTAMP: longRet = new LinkedList<>(); longRet.add(new long[capacity]); break; @@ -71,6 +73,8 @@ public DescReadWriteBatchData(TSDataType dataType) { doubleRet.add(new double[capacity]); break; case TEXT: + case BLOB: + case STRING: binaryRet = new LinkedList<>(); binaryRet.add(new Binary[capacity]); break; @@ -434,6 +438,8 @@ public void serializeData(DataOutputStream outputStream) throws IOException { } break; case TEXT: + case BLOB: + case STRING: for (int i = length() - 1; i >= 0; i--) { outputStream.writeLong(getTimeByIndex(i)); Binary binary = getBinaryByIndex(i); @@ -442,12 +448,14 @@ public void serializeData(DataOutputStream outputStream) throws IOException { } break; case INT64: + case TIMESTAMP: for (int i = length() - 1; i >= 0; i--) { outputStream.writeLong(getTimeByIndex(i)); outputStream.writeLong(getLongByIndex(i)); } break; case INT32: + case DATE: for (int i = length() - 1; i >= 0; i--) { outputStream.writeLong(getTimeByIndex(i)); outputStream.writeInt(getIntByIndex(i)); @@ -475,14 +483,18 @@ public void serializeData(DataOutputStream outputStream) throws IOException { outputStream.writeFloat(value.getFloat()); break; case TEXT: + case BLOB: + case STRING: Binary binary = value.getBinary(); outputStream.writeInt(binary.getLength()); outputStream.write(binary.getValues()); break; case INT64: + case TIMESTAMP: outputStream.writeLong(value.getLong()); break; case INT32: + case DATE: outputStream.writeInt(value.getInt()); break; default: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java index 32feae4a3..aaa2f2f95 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java @@ -22,9 +22,12 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NullFieldException; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.time.LocalDate; + /** * Field is component of one {@code RowRecord} which stores a value in specific data type. The value * type of Field is primitive(int long, float, double, binary, boolean). @@ -54,15 +57,19 @@ public static Field copy(Field field) { out.setFloatV(field.getFloatV()); break; case INT64: + case TIMESTAMP: out.setLongV(field.getLongV()); break; case INT32: + case DATE: out.setIntV(field.getIntV()); break; case BOOLEAN: out.setBoolV(field.getBoolV()); break; case TEXT: + case BLOB: + case STRING: out.setBinaryV(field.getBinaryV()); break; default: @@ -143,6 +150,13 @@ public void setBinaryV(Binary binaryV) { this.binaryV = binaryV; } + public LocalDate getDateV() { + if (dataType == null) { + throw new NullFieldException(); + } + return DateUtils.parseIntToLocalDate(intV); + } + /** * get field value and convert to string. * @@ -156,14 +170,18 @@ public String getStringValue() { case BOOLEAN: return String.valueOf(boolV); case INT32: + case DATE: return String.valueOf(intV); case INT64: + case TIMESTAMP: return String.valueOf(longV); case FLOAT: return String.valueOf(floatV); case DOUBLE: return String.valueOf(doubleV); case TEXT: + case BLOB: + case STRING: return binaryV.toString(); default: throw new UnSupportedDataTypeException(dataType.toString()); @@ -185,12 +203,17 @@ public Object getObjectValue(TSDataType dataType) { case FLOAT: return getFloatV(); case INT64: + case TIMESTAMP: return getLongV(); case INT32: return getIntV(); + case DATE: + return getDateV(); case BOOLEAN: return getBoolV(); case TEXT: + case BLOB: + case STRING: return getBinaryV(); default: throw new UnSupportedDataTypeException(dataType.toString()); @@ -204,9 +227,11 @@ public static Field getField(Object value, TSDataType dataType) { Field field = new Field(dataType); switch (dataType) { case INT32: + case DATE: field.setIntV((int) value); break; case INT64: + case TIMESTAMP: field.setLongV((long) value); break; case FLOAT: @@ -219,6 +244,8 @@ public static Field getField(Object value, TSDataType dataType) { field.setBoolV((boolean) value); break; case TEXT: + case BLOB: + case STRING: field.setBinaryV((Binary) value); break; default: @@ -233,9 +260,11 @@ public static void setTsPrimitiveValue(TsPrimitiveType value, Field field) { field.setBoolV(value.getBoolean()); break; case INT32: + case DATE: field.setIntV(value.getInt()); break; case INT64: + case TIMESTAMP: field.setLongV(value.getLong()); break; case FLOAT: @@ -245,6 +274,8 @@ public static void setTsPrimitiveValue(TsPrimitiveType value, Field field) { field.setDoubleV(value.getDouble()); break; case TEXT: + case BLOB: + case STRING: field.setBinaryV(value.getBinary()); break; default: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java index d6b7ca3d7..da3e05681 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java @@ -523,11 +523,13 @@ public void update(int updateIdx, TsBlock sourceTsBlock, int sourceIndex) { sourceTsBlock.getValueColumns()[i].getBoolean(sourceIndex); break; case INT32: + case DATE: valueColumns[i].isNull()[updateIdx] = false; valueColumns[i].getInts()[updateIdx] = sourceTsBlock.getValueColumns()[i].getInt(sourceIndex); break; case INT64: + case TIMESTAMP: valueColumns[i].isNull()[updateIdx] = false; valueColumns[i].getLongs()[updateIdx] = sourceTsBlock.getValueColumns()[i].getLong(sourceIndex); @@ -543,6 +545,8 @@ public void update(int updateIdx, TsBlock sourceTsBlock, int sourceIndex) { sourceTsBlock.getValueColumns()[i].getDouble(sourceIndex); break; case TEXT: + case BLOB: + case STRING: valueColumns[i].isNull()[updateIdx] = false; valueColumns[i].getBinaries()[updateIdx] = sourceTsBlock.getValueColumns()[i].getBinary(sourceIndex); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java index 872372b78..568c222c7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java @@ -113,11 +113,13 @@ private TsBlockBuilder(int initialExpectedEntries, int maxTsBlockBytes, List types) { tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); break; case INT32: + case DATE: valueColumnBuilders[i] = new IntColumnBuilder( tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); break; case INT64: + case TIMESTAMP: valueColumnBuilders[i] = new LongColumnBuilder( tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); @@ -201,6 +207,8 @@ public void buildValueColumnBuilders(List types) { tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); break; case TEXT: + case BLOB: + case STRING: valueColumnBuilders[i] = new BinaryColumnBuilder( tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java index 748a054c3..cc727fb7c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int32ArrayColumnEncoder.java @@ -40,36 +40,38 @@ public Column readColumn(ByteBuffer input, TSDataType dataType, int positionCoun boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount); - if (TSDataType.INT32.equals(dataType)) { - int[] values = new int[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - values[i] = input.getInt(); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - values[i] = input.getInt(); + switch (dataType) { + case INT32: + case DATE: + int[] intValues = new int[positionCount]; + if (nullIndicators == null) { + for (int i = 0; i < positionCount; i++) { + intValues[i] = input.getInt(); + } + } else { + for (int i = 0; i < positionCount; i++) { + if (!nullIndicators[i]) { + intValues[i] = input.getInt(); + } } } - } - return new IntColumn(0, positionCount, nullIndicators, values); - } else if (TSDataType.FLOAT.equals(dataType)) { - float[] values = new float[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - values[i] = Float.intBitsToFloat(input.getInt()); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - values[i] = Float.intBitsToFloat(input.getInt()); + return new IntColumn(0, positionCount, nullIndicators, intValues); + case FLOAT: + float[] floatValues = new float[positionCount]; + if (nullIndicators == null) { + for (int i = 0; i < positionCount; i++) { + floatValues[i] = Float.intBitsToFloat(input.getInt()); + } + } else { + for (int i = 0; i < positionCount; i++) { + if (!nullIndicators[i]) { + floatValues[i] = Float.intBitsToFloat(input.getInt()); + } } } - } - return new FloatColumn(0, positionCount, nullIndicators, values); - } else { - throw new IllegalArgumentException("Invalid data type: " + dataType); + return new FloatColumn(0, positionCount, nullIndicators, floatValues); + default: + throw new IllegalArgumentException("Invalid data type: " + dataType); } } @@ -80,32 +82,36 @@ public void writeColumn(DataOutputStream output, Column column) throws IOExcepti TSDataType dataType = column.getDataType(); int positionCount = column.getPositionCount(); - if (TSDataType.INT32.equals(dataType)) { - if (column.mayHaveNull()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { + switch (dataType) { + case INT32: + case DATE: + if (column.mayHaveNull()) { + for (int i = 0; i < positionCount; i++) { + if (!column.isNull(i)) { + output.writeInt(column.getInt(i)); + } + } + } else { + for (int i = 0; i < positionCount; i++) { output.writeInt(column.getInt(i)); } } - } else { - for (int i = 0; i < positionCount; i++) { - output.writeInt(column.getInt(i)); - } - } - } else if (TSDataType.FLOAT.equals(dataType)) { - if (column.mayHaveNull()) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { + break; + case FLOAT: + if (column.mayHaveNull()) { + for (int i = 0; i < positionCount; i++) { + if (!column.isNull(i)) { + output.writeInt(Float.floatToIntBits(column.getFloat(i))); + } + } + } else { + for (int i = 0; i < positionCount; i++) { output.writeInt(Float.floatToIntBits(column.getFloat(i))); } } - } else { - for (int i = 0; i < positionCount; i++) { - output.writeInt(Float.floatToIntBits(column.getFloat(i))); - } - } - } else { - throw new IllegalArgumentException("Invalid data type: " + dataType); + break; + default: + throw new IllegalArgumentException("Invalid data type: " + dataType); } } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java index 51490f06a..58cfbebf9 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/Int64ArrayColumnEncoder.java @@ -61,37 +61,38 @@ public Column readColumn(ByteBuffer input, TSDataType dataType, int positionCoun // +---------------+-----------------+-------------+ boolean[] nullIndicators = ColumnEncoder.deserializeNullIndicators(input, positionCount); - - if (TSDataType.INT64.equals(dataType)) { - long[] values = new long[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - values[i] = input.getLong(); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { + switch (dataType) { + case INT64: + case TIMESTAMP: + long[] values = new long[positionCount]; + if (nullIndicators == null) { + for (int i = 0; i < positionCount; i++) { values[i] = input.getLong(); } + } else { + for (int i = 0; i < positionCount; i++) { + if (!nullIndicators[i]) { + values[i] = input.getLong(); + } + } } - } - return new LongColumn(0, positionCount, nullIndicators, values); - } else if (TSDataType.DOUBLE.equals(dataType)) { - double[] values = new double[positionCount]; - if (nullIndicators == null) { - for (int i = 0; i < positionCount; i++) { - values[i] = Double.longBitsToDouble(input.getLong()); - } - } else { - for (int i = 0; i < positionCount; i++) { - if (!nullIndicators[i]) { - values[i] = Double.longBitsToDouble(input.getLong()); + return new LongColumn(0, positionCount, nullIndicators, values); + case DOUBLE: + double[] doubleValues = new double[positionCount]; + if (nullIndicators == null) { + for (int i = 0; i < positionCount; i++) { + doubleValues[i] = Double.longBitsToDouble(input.getLong()); + } + } else { + for (int i = 0; i < positionCount; i++) { + if (!nullIndicators[i]) { + doubleValues[i] = Double.longBitsToDouble(input.getLong()); + } } } - } - return new DoubleColumn(0, positionCount, nullIndicators, values); - } else { - throw new IllegalArgumentException("Invalid data type: " + dataType); + return new DoubleColumn(0, positionCount, nullIndicators, doubleValues); + default: + throw new IllegalArgumentException("Invalid data type: " + dataType); } } @@ -102,20 +103,24 @@ public void writeColumn(DataOutputStream output, Column column) throws IOExcepti TSDataType dataType = column.getDataType(); int positionCount = column.getPositionCount(); - if (TSDataType.INT64.equals(dataType)) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - output.writeLong(column.getLong(i)); + switch (dataType) { + case INT64: + case TIMESTAMP: + for (int i = 0; i < positionCount; i++) { + if (!column.isNull(i)) { + output.writeLong(column.getLong(i)); + } } - } - } else if (TSDataType.DOUBLE.equals(dataType)) { - for (int i = 0; i < positionCount; i++) { - if (!column.isNull(i)) { - output.writeLong(Double.doubleToLongBits(column.getDouble(i))); + break; + case DOUBLE: + for (int i = 0; i < positionCount; i++) { + if (!column.isNull(i)) { + output.writeLong(Double.doubleToLongBits(column.getDouble(i))); + } } - } - } else { - throw new IllegalArgumentException("Invalid data type: " + dataType); + break; + default: + throw new IllegalArgumentException("Invalid data type: " + dataType); } } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java index be21cfa6e..f95cdb005 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java @@ -108,14 +108,18 @@ public static Column create(TSDataType dataType, int positionCount) { case BOOLEAN: return new RunLengthEncodedColumn(BooleanColumnBuilder.NULL_VALUE_BLOCK, positionCount); case INT32: + case DATE: return new RunLengthEncodedColumn(IntColumnBuilder.NULL_VALUE_BLOCK, positionCount); case INT64: + case TIMESTAMP: return new RunLengthEncodedColumn(LongColumnBuilder.NULL_VALUE_BLOCK, positionCount); case FLOAT: return new RunLengthEncodedColumn(FloatColumnBuilder.NULL_VALUE_BLOCK, positionCount); case DOUBLE: return new RunLengthEncodedColumn(DoubleColumnBuilder.NULL_VALUE_BLOCK, positionCount); case TEXT: + case BLOB: + case STRING: return new RunLengthEncodedColumn(BinaryColumnBuilder.NULL_VALUE_BLOCK, positionCount); default: throw new IllegalArgumentException("Unknown data type: " + dataType); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java index 622822809..72946ee83 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java @@ -30,8 +30,10 @@ private TypeFactory() { public static Type getType(TSDataType tsDataType) { switch (tsDataType) { case INT32: + case DATE: return IntType.getInstance(); case INT64: + case TIMESTAMP: return LongType.getInstance(); case FLOAT: return FloatType.getInstance(); @@ -40,6 +42,8 @@ public static Type getType(TSDataType tsDataType) { case BOOLEAN: return BooleanType.getInstance(); case TEXT: + case BLOB: + case STRING: return BinaryType.getInstance(); default: throw new UnsupportedOperationException( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/ValueFilterOperators.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/ValueFilterOperators.java index 3cc7e8ad9..c2b7284cf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/ValueFilterOperators.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/ValueFilterOperators.java @@ -557,6 +557,7 @@ public OperatorType getOperatorType() { private static boolean statisticsNotAvailable(Statistics statistics) { return statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN + || statistics.getType() == TSDataType.BLOB || statistics.isEmpty(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java index cdd50d9b1..02f8ffd45 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java @@ -163,9 +163,11 @@ private Field putValueToField(BatchData col) { field.setBoolV(col.getBoolean()); break; case INT32: + case DATE: field.setIntV(col.getInt()); break; case INT64: + case TIMESTAMP: field.setLongV(col.getLong()); break; case FLOAT: @@ -175,6 +177,8 @@ private Field putValueToField(BatchData col) { field.setDoubleV(col.getDouble()); break; case TEXT: + case BLOB: + case STRING: field.setBinaryV(col.getBinary()); break; case VECTOR: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java index 646c9f4ec..f505acbcb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java @@ -134,12 +134,14 @@ public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { } break; case INT32: + case DATE: int anInt = valueDecoder.readInt(valueBuffer); if (!isDeleted(timestamp) && (allSatisfy || recordFilter.satisfy(timestamp, anInt))) { pageData.putInt(timestamp, anInt); } break; case INT64: + case TIMESTAMP: long aLong = valueDecoder.readLong(valueBuffer); if (!isDeleted(timestamp) && (allSatisfy || recordFilter.satisfy(timestamp, aLong))) { pageData.putLong(timestamp, aLong); @@ -158,6 +160,8 @@ public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { } break; case TEXT: + case BLOB: + case STRING: Binary aBinary = valueDecoder.readBinary(valueBuffer); if (!isDeleted(timestamp) && (allSatisfy || recordFilter.satisfy(timestamp, aBinary))) { pageData.putBinary(timestamp, aBinary); @@ -206,6 +210,7 @@ public TsBlock getAllSatisfiedData() throws IOException { } break; case INT32: + case DATE: while (timeDecoder.hasNext(timeBuffer)) { long timestamp = timeDecoder.readLong(timeBuffer); int anInt = valueDecoder.readInt(valueBuffer); @@ -227,6 +232,7 @@ public TsBlock getAllSatisfiedData() throws IOException { } break; case INT64: + case TIMESTAMP: while (timeDecoder.hasNext(timeBuffer)) { long timestamp = timeDecoder.readLong(timeBuffer); long aLong = valueDecoder.readLong(valueBuffer); @@ -290,6 +296,8 @@ public TsBlock getAllSatisfiedData() throws IOException { } break; case TEXT: + case BLOB: + case STRING: while (timeDecoder.hasNext(timeBuffer)) { long timestamp = timeDecoder.readLong(timeBuffer); Binary aBinary = valueDecoder.readBinary(valueBuffer); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java index 833df288b..8bf137910 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java @@ -101,12 +101,14 @@ public BatchData nextBatch(long[] timeBatch, boolean ascending, Filter filter) { } break; case INT32: + case DATE: int anInt = valueDecoder.readInt(valueBuffer); if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, anInt))) { pageData.putInt(timestamp, anInt); } break; case INT64: + case TIMESTAMP: long aLong = valueDecoder.readLong(valueBuffer); if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aLong))) { pageData.putLong(timestamp, aLong); @@ -125,6 +127,8 @@ public BatchData nextBatch(long[] timeBatch, boolean ascending, Filter filter) { } break; case TEXT: + case BLOB: + case STRING: Binary aBinary = valueDecoder.readBinary(valueBuffer); if (!isDeleted(timestamp) && (filter == null || filter.satisfy(timestamp, aBinary))) { pageData.putBinary(timestamp, aBinary); @@ -150,12 +154,14 @@ public TsPrimitiveType nextValue(long timestamp, int timeIndex) { } break; case INT32: + case DATE: int anInt = valueDecoder.readInt(valueBuffer); if (!isDeleted(timestamp)) { resultValue = new TsPrimitiveType.TsInt(anInt); } break; case INT64: + case TIMESTAMP: long aLong = valueDecoder.readLong(valueBuffer); if (!isDeleted(timestamp)) { resultValue = new TsPrimitiveType.TsLong(aLong); @@ -174,6 +180,8 @@ public TsPrimitiveType nextValue(long timestamp, int timeIndex) { } break; case TEXT: + case BLOB: + case STRING: Binary aBinary = valueDecoder.readBinary(valueBuffer); if (!isDeleted(timestamp)) { resultValue = new TsPrimitiveType.TsBinary(aBinary); @@ -207,12 +215,14 @@ public TsPrimitiveType[] nextValueBatch(long[] timeBatch) { } break; case INT32: + case DATE: int anInt = valueDecoder.readInt(valueBuffer); if (!isDeleted(timeBatch[i])) { valueBatch[i] = new TsPrimitiveType.TsInt(anInt); } break; case INT64: + case TIMESTAMP: long aLong = valueDecoder.readLong(valueBuffer); if (!isDeleted(timeBatch[i])) { valueBatch[i] = new TsPrimitiveType.TsLong(aLong); @@ -231,6 +241,8 @@ public TsPrimitiveType[] nextValueBatch(long[] timeBatch) { } break; case TEXT: + case BLOB: + case STRING: Binary aBinary = valueDecoder.readBinary(valueBuffer); if (!isDeleted(timeBatch[i])) { valueBatch[i] = new TsPrimitiveType.TsBinary(aBinary); @@ -275,6 +287,7 @@ public void writeColumnBuilderWithNextBatch( } break; case INT32: + case DATE: int anInt = valueDecoder.readInt(valueBuffer); if (keepCurrentRow[i]) { if (isDeleted[i]) { @@ -285,6 +298,7 @@ public void writeColumnBuilderWithNextBatch( } break; case INT64: + case TIMESTAMP: long aLong = valueDecoder.readLong(valueBuffer); if (keepCurrentRow[i]) { if (isDeleted[i]) { @@ -315,6 +329,8 @@ public void writeColumnBuilderWithNextBatch( } break; case TEXT: + case BLOB: + case STRING: Binary aBinary = valueDecoder.readBinary(valueBuffer); if (keepCurrentRow[i]) { if (isDeleted[i]) { @@ -355,12 +371,14 @@ public void writeColumnBuilderWithNextBatch( } break; case INT32: + case DATE: int anInt = valueDecoder.readInt(valueBuffer); if (keepCurrentRow[i]) { columnBuilder.writeInt(anInt); } break; case INT64: + case TIMESTAMP: long aLong = valueDecoder.readLong(valueBuffer); if (keepCurrentRow[i]) { columnBuilder.writeLong(aLong); @@ -379,6 +397,8 @@ public void writeColumnBuilderWithNextBatch( } break; case TEXT: + case BLOB: + case STRING: Binary aBinary = valueDecoder.readBinary(valueBuffer); if (keepCurrentRow[i]) { columnBuilder.writeBinary(aBinary); @@ -417,6 +437,7 @@ public void writeColumnBuilderWithNextBatch( } break; case INT32: + case DATE: // skip useless data for (int i = 0; i < readStartIndex; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { @@ -435,6 +456,7 @@ public void writeColumnBuilderWithNextBatch( } break; case INT64: + case TIMESTAMP: // skip useless data for (int i = 0; i < readStartIndex; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { @@ -489,6 +511,8 @@ public void writeColumnBuilderWithNextBatch( } break; case TEXT: + case BLOB: + case STRING: // skip useless data for (int i = 0; i < readStartIndex; i++) { if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/BytesUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/BytesUtils.java index 40862a5dc..f38a0da19 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/BytesUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/BytesUtils.java @@ -931,4 +931,14 @@ public static short bytesToShort(byte[] b) { public static Binary valueOf(String value) { return new Binary(stringToBytes(value)); } + + public static String parseBlobByteArrayToString(byte[] input) { + StringBuilder hexString = new StringBuilder("0x"); + if (input != null) { + for (byte b : input) { + hexString.append(String.format("%02x", b)); + } + } + return hexString.toString(); + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/DateUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/DateUtils.java new file mode 100644 index 000000000..3caa46c2a --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/DateUtils.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.utils; + +import java.sql.Date; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; + +public class DateUtils { + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + public static String formatDate(int date) { + return date / 10000 + + "-" + + String.format("%02d", (date / 100) % 100) + + "-" + + String.format("%02d", date % 100); + } + + public static Integer parseDateExpressionToInt(String dateExpression) { + if (dateExpression == null || dateExpression.isEmpty()) { + throw new DateTimeParseException("Date expression is null or empty.", "", 0); + } + LocalDate date; + try { + date = LocalDate.parse(dateExpression, DATE_FORMATTER); + } catch (DateTimeParseException e) { + throw new DateTimeParseException( + "Invalid date format. Please use YYYY-MM-DD format.", dateExpression, 0); + } + if (date.getYear() < 1000) { + throw new DateTimeParseException("Year must be between 1000 and 9999.", dateExpression, 0); + } + return date.getYear() * 10000 + date.getMonthValue() * 100 + date.getDayOfMonth(); + } + + public static Integer parseDateExpressionToInt(LocalDate localDate) { + if (localDate == null) { + throw new DateTimeParseException("Date expression is null or empty.", "", 0); + } + if (localDate.getYear() < 1000) { + throw new DateTimeParseException( + "Year must be between 1000 and 9999.", localDate.format(DATE_FORMATTER), 0); + } + return localDate.getYear() * 10000 + + localDate.getMonthValue() * 100 + + localDate.getDayOfMonth(); + } + + public static Date parseIntToDate(int date) { + return new Date(date / 10000 - 1900, (date / 100) % 100 - 1, date % 100); + } + + public static LocalDate parseIntToLocalDate(int date) { + try { + return LocalDate.of(date / 10000, (date / 100) % 100, date % 100); + } catch (Exception e) { + throw new DateTimeParseException("Invalid date format.", "", 0); + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java index 71633a991..ba1b5e317 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java @@ -70,9 +70,11 @@ public static void writeWithTsRecord( DataPoint dPoint; switch (schema.getType()) { case INT64: + case TIMESTAMP: dPoint = new LongDataPoint(schema.getMeasurementId(), startValue); break; case INT32: + case DATE: dPoint = new IntDataPoint(schema.getMeasurementId(), (int) startValue); break; case DOUBLE: @@ -85,6 +87,8 @@ public static void writeWithTsRecord( dPoint = new BooleanDataPoint(schema.getMeasurementId(), true); break; case TEXT: + case BLOB: + case STRING: default: dPoint = new StringDataPoint( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index 1d596881e..f0e4055ad 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -120,9 +120,11 @@ public int write(long time, List data) throws WriteProcessException, valueChunkWriter.write(time, (boolean) point.getValue(), isNull); break; case INT32: + case DATE: valueChunkWriter.write(time, (int) point.getValue(), isNull); break; case INT64: + case TIMESTAMP: valueChunkWriter.write(time, (long) point.getValue(), isNull); break; case FLOAT: @@ -132,6 +134,8 @@ public int write(long time, List data) throws WriteProcessException, valueChunkWriter.write(time, (double) point.getValue(), isNull); break; case TEXT: + case BLOB: + case STRING: valueChunkWriter.write(time, (Binary) point.getValue(), isNull); break; default: @@ -180,9 +184,11 @@ public int write(Tablet tablet) throws WriteProcessException, IOException { valueChunkWriter.write(time, ((boolean[]) tablet.values[columnIndex])[row], isNull); break; case INT32: + case DATE: valueChunkWriter.write(time, ((int[]) tablet.values[columnIndex])[row], isNull); break; case INT64: + case TIMESTAMP: valueChunkWriter.write(time, ((long[]) tablet.values[columnIndex])[row], isNull); break; case FLOAT: @@ -192,6 +198,8 @@ public int write(Tablet tablet) throws WriteProcessException, IOException { valueChunkWriter.write(time, ((double[]) tablet.values[columnIndex])[row], isNull); break; case TEXT: + case BLOB: + case STRING: valueChunkWriter.write(time, ((Binary[]) tablet.values[columnIndex])[row], isNull); break; default: @@ -266,9 +274,11 @@ private void writeEmptyDataInOneRow(List valueChunkWriterList) valueChunkWriter.write(-1, false, true); break; case INT32: + case DATE: valueChunkWriter.write(-1, 0, true); break; case INT64: + case TIMESTAMP: valueChunkWriter.write(-1, 0L, true); break; case FLOAT: @@ -278,6 +288,8 @@ private void writeEmptyDataInOneRow(List valueChunkWriterList) valueChunkWriter.write(-1, 0.0d, true); break; case TEXT: + case BLOB: + case STRING: valueChunkWriter.write(-1, null, true); break; default: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java index 656b7852e..d5fddb837 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java @@ -198,9 +198,11 @@ public void write(long time, TsPrimitiveType[] points) { ValueChunkWriter writer = valueChunkWriterList.get(valueIndex++); switch (writer.getDataType()) { case INT64: + case TIMESTAMP: writer.write(time, point != null ? point.getLong() : Long.MAX_VALUE, point == null); break; case INT32: + case DATE: writer.write(time, point != null ? point.getInt() : Integer.MAX_VALUE, point == null); break; case FLOAT: @@ -213,6 +215,8 @@ public void write(long time, TsPrimitiveType[] points) { writer.write(time, point != null ? point.getBoolean() : false, point == null); break; case TEXT: + case BLOB: + case STRING: writer.write( time, point != null ? point.getBinary() : new Binary("".getBytes(StandardCharsets.UTF_8)), @@ -255,6 +259,8 @@ private void batchWrite( TSDataType tsDataType = chunkWriter.getDataType(); switch (tsDataType) { case TEXT: + case BLOB: + case STRING: chunkWriter.write(times, column.getBinaries(), column.isNull(), batchSize, arrayOffset); break; case DOUBLE: @@ -264,9 +270,11 @@ private void batchWrite( chunkWriter.write(times, column.getBooleans(), column.isNull(), batchSize, arrayOffset); break; case INT64: + case TIMESTAMP: chunkWriter.write(times, column.getLongs(), column.isNull(), batchSize, arrayOffset); break; case INT32: + case DATE: chunkWriter.write(times, column.getInts(), column.isNull(), batchSize, arrayOffset); break; case FLOAT: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java index 9df9b14d9..42485ac75 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java @@ -109,9 +109,11 @@ public int write(Tablet tablet) throws WriteProcessException { pointCount++; switch (tsDataType) { case INT32: + case DATE: chunkWriters.get(measurementId).write(time, ((int[]) tablet.values[column])[row]); break; case INT64: + case TIMESTAMP: chunkWriters.get(measurementId).write(time, ((long[]) tablet.values[column])[row]); break; case FLOAT: @@ -124,6 +126,8 @@ public int write(Tablet tablet) throws WriteProcessException { chunkWriters.get(measurementId).write(time, ((boolean[]) tablet.values[column])[row]); break; case TEXT: + case BLOB: + case STRING: chunkWriters.get(measurementId).write(time, ((Binary[]) tablet.values[column])[row]); break; default: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 61ab2e419..bd18fb2bf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -24,6 +24,7 @@ import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -32,6 +33,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.LocalDate; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -187,6 +189,7 @@ private void addValueOfDataType( } switch (dataType) { case TEXT: + case STRING: { Binary[] sensor = (Binary[]) values[indexOfSchema]; if (value instanceof Binary) { @@ -199,6 +202,12 @@ private void addValueOfDataType( } break; } + case BLOB: + { + Binary[] sensor = (Binary[]) values[indexOfSchema]; + sensor[rowIndex] = value != null ? (Binary) value : Binary.EMPTY_VALUE; + break; + } case FLOAT: { float[] sensor = (float[]) values[indexOfSchema]; @@ -211,7 +220,14 @@ private void addValueOfDataType( sensor[rowIndex] = value != null ? (int) value : Integer.MIN_VALUE; break; } + case DATE: + { + LocalDate[] sensor = (LocalDate[]) values[indexOfSchema]; + sensor[rowIndex] = (LocalDate) value; + break; + } case INT64: + case TIMESTAMP: { long[] sensor = (long[]) values[indexOfSchema]; sensor[rowIndex] = value != null ? (long) value : Long.MIN_VALUE; @@ -280,6 +296,7 @@ private Object createValueColumnOfDataType(TSDataType dataType) { valueColumn = new int[maxRowNumber]; break; case INT64: + case TIMESTAMP: valueColumn = new long[maxRowNumber]; break; case FLOAT: @@ -292,8 +309,13 @@ private Object createValueColumnOfDataType(TSDataType dataType) { valueColumn = new boolean[maxRowNumber]; break; case TEXT: + case STRING: + case BLOB: valueColumn = new Binary[maxRowNumber]; break; + case DATE: + valueColumn = new LocalDate[maxRowNumber]; + break; default: throw new UnSupportedDataTypeException(String.format(NOT_SUPPORT_DATATYPE, dataType)); } @@ -333,13 +355,17 @@ private int calOccupationOfOneColumn(TSDataType dataType, int columnIndex) { break; case INT32: case FLOAT: + case DATE: valueOccupation += rowSize * 4; break; case INT64: case DOUBLE: + case TIMESTAMP: valueOccupation += rowSize * 8; break; case TEXT: + case BLOB: + case STRING: valueOccupation += rowSize * 4; Binary[] binaries = (Binary[]) values[columnIndex]; for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { @@ -435,7 +461,14 @@ private void serializeColumn(TSDataType dataType, Object column, DataOutputStrea ReadWriteIOUtils.write(intValues[j], stream); } break; + case DATE: + LocalDate[] dateValues = (LocalDate[]) column; + for (int j = 0; j < rowSize; j++) { + ReadWriteIOUtils.write(DateUtils.parseDateExpressionToInt(dateValues[j]), stream); + } + break; case INT64: + case TIMESTAMP: long[] longValues = (long[]) column; for (int j = 0; j < rowSize; j++) { ReadWriteIOUtils.write(longValues[j], stream); @@ -460,6 +493,8 @@ private void serializeColumn(TSDataType dataType, Object column, DataOutputStrea } break; case TEXT: + case BLOB: + case STRING: Binary[] binaryValues = (Binary[]) column; for (int j = 0; j < rowSize; j++) { ReadWriteIOUtils.write(BytesUtils.boolToByte(binaryValues[j] != null), stream); @@ -565,7 +600,16 @@ public static Object[] readTabletValuesFromBuffer( } values[i] = intValues; break; + case DATE: + LocalDate[] dateValues = new LocalDate[rowSize]; + for (int index = 0; index < rowSize; index++) { + dateValues[index] = + DateUtils.parseIntToLocalDate(ReadWriteIOUtils.readInt(byteBuffer)); + } + values[i] = dateValues; + break; case INT64: + case TIMESTAMP: long[] longValues = new long[rowSize]; for (int index = 0; index < rowSize; index++) { longValues[index] = ReadWriteIOUtils.readLong(byteBuffer); @@ -587,6 +631,8 @@ public static Object[] readTabletValuesFromBuffer( values[i] = doubleValues; break; case TEXT: + case BLOB: + case STRING: Binary[] binaryValues = new Binary[rowSize]; for (int index = 0; index < rowSize; index++) { boolean isNotNull = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(byteBuffer)); @@ -678,7 +724,20 @@ public boolean equals(Object o) { } } break; + case DATE: + LocalDate[] thisDateValues = (LocalDate[]) values[i]; + LocalDate[] thatDateValues = (LocalDate[]) thatValues[i]; + if (thisDateValues.length < rowSize || thatDateValues.length < rowSize) { + return false; + } + for (int j = 0; j < rowSize; j++) { + if (!thisDateValues[j].equals(thatDateValues[j])) { + return false; + } + } + break; case INT64: + case TIMESTAMP: long[] thisLongValues = (long[]) values[i]; long[] thatLongValues = (long[]) thatValues[i]; if (thisLongValues.length < rowSize || thatLongValues.length < rowSize) { @@ -727,6 +786,8 @@ public boolean equals(Object o) { } break; case TEXT: + case BLOB: + case STRING: Binary[] thisBinaryValues = (Binary[]) values[i]; Binary[] thatBinaryValues = (Binary[]) thatValues[i]; if (thisBinaryValues.length < rowSize || thatBinaryValues.length < rowSize) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java index 6bfa1b69e..6f5e4a087 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java @@ -64,9 +64,11 @@ public static DataPoint getDataPoint(TSDataType dataType, String measurementId, try { switch (dataType) { case INT32: + case DATE: dataPoint = new IntDataPoint(measurementId, Integer.parseInt(value)); break; case INT64: + case TIMESTAMP: dataPoint = new LongDataPoint(measurementId, Long.parseLong(value)); break; case FLOAT: @@ -79,6 +81,8 @@ public static DataPoint getDataPoint(TSDataType dataType, String measurementId, dataPoint = new BooleanDataPoint(measurementId, Boolean.parseBoolean(value)); break; case TEXT: + case BLOB: + case STRING: dataPoint = new StringDataPoint(measurementId, new Binary(value, TSFileConfig.STRING_CHARSET)); break; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/DateUtilsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/DateUtilsTest.java new file mode 100644 index 000000000..6154a3a91 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/DateUtilsTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.utils; + +import org.junit.Test; + +import java.time.LocalDate; +import java.time.format.DateTimeParseException; +import java.util.Date; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class DateUtilsTest { + + @Test + public void testFormatDate() { + int date = 20230514; + String formattedDate = DateUtils.formatDate(date); + assertEquals("2023-05-14", formattedDate); + } + + @Test + public void testParseDateExpressionToInt_ValidDate() { + String dateExpression = "2023-05-14"; + int dateInt = DateUtils.parseDateExpressionToInt(dateExpression); + assertEquals(20230514, dateInt); + } + + @Test + public void testParseDateExpressionToInt_InvalidDate() { + String dateExpression = "2023-14-05"; + assertThrows( + DateTimeParseException.class, + () -> { + DateUtils.parseDateExpressionToInt(dateExpression); + }); + } + + @Test + public void testParseDateExpressionToInt_NullOrEmpty() { + assertThrows( + DateTimeParseException.class, + () -> { + DateUtils.parseDateExpressionToInt((String) null); + }); + assertThrows( + DateTimeParseException.class, + () -> { + DateUtils.parseDateExpressionToInt(""); + }); + } + + @Test + public void testParseDateExpressionToInt_ValidLocalDate() { + LocalDate localDate = LocalDate.of(2023, 5, 14); + int dateInt = DateUtils.parseDateExpressionToInt(localDate); + assertEquals(20230514, dateInt); + } + + @Test + public void testParseDateExpressionToInt_NullLocalDate() { + assertThrows( + DateTimeParseException.class, + () -> { + DateUtils.parseDateExpressionToInt((LocalDate) null); + }); + } + + @Test + public void testParseIntToDate() { + int date = 20230514; + Date parsedDate = DateUtils.parseIntToDate(date); + assertEquals(2023 - 1900, parsedDate.getYear()); + assertEquals(4, parsedDate.getMonth()); // Date month is 0-based + assertEquals(14, parsedDate.getDate()); + } + + @Test + public void testParseIntToLocalDate() { + int date = 20230514; + LocalDate localDate = DateUtils.parseIntToLocalDate(date); + assertEquals(2023, localDate.getYear()); + assertEquals(5, localDate.getMonthValue()); + assertEquals(14, localDate.getDayOfMonth()); + } + + @Test + public void testParseIntToLocalDate_InvalidDate() { + int date = 20231405; + assertThrows( + DateTimeParseException.class, + () -> { + DateUtils.parseIntToLocalDate(date); + }); + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java index 53fc4ffe7..41e72110d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/RecordUtils.java @@ -85,9 +85,11 @@ public static TSRecord parseSimpleTupleRecord(String str, Schema schema) { try { switch (type) { case INT32: + case DATE: ret.addTuple(new IntDataPoint(measurementId, Integer.parseInt(value))); break; case INT64: + case TIMESTAMP: ret.addTuple(new LongDataPoint(measurementId, Long.parseLong(value))); break; case FLOAT: @@ -100,6 +102,8 @@ public static TSRecord parseSimpleTupleRecord(String str, Schema schema) { ret.addTuple(new BooleanDataPoint(measurementId, Boolean.parseBoolean(value))); break; case TEXT: + case BLOB: + case STRING: ret.addTuple(new StringDataPoint(measurementId, BytesUtils.valueOf(items[i + 1]))); break; default: