Fix float encoder overflow when float value itself over int range

.github/workflows/code-coverage.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Generate code coverage reports
         run: |
           sudo apt-get install lcov
-          ./mvnw -B -P with-java,with-cpp,with-python,with-code-coverage clean verify
+          ./mvnw -B -P with-java,with-cpp,with-code-coverage clean verify
           lcov --capture --directory cpp/target/build/test --output-file cpp/target/build/test/coverage.info
           lcov --remove cpp/target/build/test/coverage.info '*/tsfile/cpp/test/*' --output-file cpp/target/build/test/coverage_filtered.info
           genhtml cpp/target/build/test/coverage_filtered.info --output-directory cpp/target/build/test/coverage_report

java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatDecoder.java

@@ -40,15 +40,16 @@
 public class FloatDecoder extends Decoder {
 
   private static final Logger logger = LoggerFactory.getLogger(FloatDecoder.class);
-  private Decoder decoder;
+  private final Decoder decoder;
 
   /** maxPointValue = 10^(maxPointNumber). maxPointNumber can be read from the stream. */
   private double maxPointValue;
 
   /** flag that indicates whether we have read maxPointNumber and calculated maxPointValue. */
   private boolean isMaxPointNumberRead;
 
-  private BitMap useMaxPointNumber;
+  private BitMap isUnderflowInfo;
+  private BitMap valueItselfOverflowInfo;
   private int position = 0;
 
   public FloatDecoder(TSEncoding encodingType, TSDataType dataType) {
@@ -97,6 +98,10 @@ public FloatDecoder(TSEncoding encodingType, TSDataType dataType) {
   public float readFloat(ByteBuffer buffer) {
     readMaxPointValue(buffer);
     int value = decoder.readInt(buffer);
+    if (valueItselfOverflowInfo != null && valueItselfOverflowInfo.isMarked(position)) {
+      position++;
+      return Float.intBitsToFloat(value);
+    }
     double result = value / getMaxPointValue();
     position++;
     return (float) result;
@@ -106,16 +111,20 @@ public float readFloat(ByteBuffer buffer) {
   public double readDouble(ByteBuffer buffer) {
     readMaxPointValue(buffer);
     long value = decoder.readLong(buffer);
+    if (valueItselfOverflowInfo != null && valueItselfOverflowInfo.isMarked(position)) {
+      position++;
+      return Double.longBitsToDouble(value);
+    }
     double result = value / getMaxPointValue();
     position++;
     return result;
   }
 
   private double getMaxPointValue() {
-    if (useMaxPointNumber == null) {
+    if (isUnderflowInfo == null) {
       return maxPointValue;
     } else {
-      return useMaxPointNumber.isMarked(position) ? maxPointValue : 1;
+      return isUnderflowInfo.isMarked(position) ? maxPointValue : 1;
     }
   }
 
@@ -126,7 +135,17 @@ private void readMaxPointValue(ByteBuffer buffer) {
         int size = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
         byte[] tmp = new byte[size / 8 + 1];
         buffer.get(tmp, 0, size / 8 + 1);
-        useMaxPointNumber = new BitMap(size, tmp);
+        isUnderflowInfo = new BitMap(size, tmp);
+        maxPointNumber = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+        maxPointValue = Math.pow(10, maxPointNumber);
+      } else if (maxPointNumber == Integer.MAX_VALUE - 1) {
+        int size = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+        byte[] tmp = new byte[size / 8 + 1];
+        buffer.get(tmp, 0, size / 8 + 1);
+        isUnderflowInfo = new BitMap(size, tmp);
+        tmp = new byte[size / 8 + 1];
+        buffer.get(tmp, 0, size / 8 + 1);
+        valueItselfOverflowInfo = new BitMap(size, tmp);
         maxPointNumber = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
         maxPointValue = Math.pow(10, maxPointNumber);
       } else if (maxPointNumber <= 0) {

java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatEncoder.java

@@ -41,7 +41,7 @@
  */
 public class FloatEncoder extends Encoder {
 
-  private Encoder encoder;
+  private final Encoder encoder;
 
   /** number for accuracy of decimal places. */
   private int maxPointNumber;
@@ -52,14 +52,17 @@ public class FloatEncoder extends Encoder {
   /** flag to check whether maxPointNumber is saved in the stream. */
   private boolean isMaxPointNumberSaved;
 
-  private final List<Boolean> useMaxPointNumber;
+  // value * maxPointValue not overflow -> True
+  // value * maxPointValue overflow -> False
+  // value itself overflow -> null
+  private final List<Boolean> underflowFlags;
 
   public FloatEncoder(TSEncoding encodingType, TSDataType dataType, int maxPointNumber) {
     super(encodingType);
     this.maxPointNumber = maxPointNumber;
     calculateMaxPointNum();
     isMaxPointNumberSaved = false;
-    useMaxPointNumber = new ArrayList<>();
+    underflowFlags = new ArrayList<>();
     if (encodingType == TSEncoding.RLE) {
       if (dataType == TSDataType.FLOAT) {
         encoder = new IntRleEncoder();
@@ -118,52 +121,86 @@ private void calculateMaxPointNum() {
 
   private int convertFloatToInt(float value) {
     if (value * maxPointValue > Integer.MAX_VALUE || value * maxPointValue < Integer.MIN_VALUE) {
-      useMaxPointNumber.add(false);
-      return Math.round(value);
+      if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) {
+        underflowFlags.add(null);
+        return Float.floatToIntBits(value);
+      } else {
+        underflowFlags.add(false);
+        return Math.round(value);
+      }
     } else {
-      useMaxPointNumber.add(true);
-      return (int) Math.round(value * maxPointValue);
+      if (Float.isNaN(value)) {
+        underflowFlags.add(null);
+        return Float.floatToIntBits(value);
+      } else {
+        underflowFlags.add(true);
+        return (int) Math.round(value * maxPointValue);
+      }
     }
   }
 
   private long convertDoubleToLong(double value) {
     if (value * maxPointValue > Long.MAX_VALUE || value * maxPointValue < Long.MIN_VALUE) {
-      useMaxPointNumber.add(false);
-      return Math.round(value);
+      if (value > Long.MAX_VALUE || value < Long.MIN_VALUE) {
+        underflowFlags.add(null);
+        return Double.doubleToLongBits(value);
+      } else {
+        underflowFlags.add(false);
+        return Math.round(value);
+      }
     } else {
-      useMaxPointNumber.add(true);
-      return Math.round(value * maxPointValue);
+      if (Double.isNaN(value)) {
+        underflowFlags.add(null);
+        return Double.doubleToLongBits(value);
+      } else {
+        underflowFlags.add(true);
+        return Math.round(value * maxPointValue);
+      }
     }
   }
 
   @Override
   public void flush(ByteArrayOutputStream out) throws IOException {
     encoder.flush(out);
-    if (pointsNotUseMaxPointNumber()) {
+    if (hasOverflow()) {
       byte[] ba = out.toByteArray();
       out.reset();
-      ReadWriteForEncodingUtils.writeUnsignedVarInt(Integer.MAX_VALUE, out);
-      BitMap bitMap = new BitMap(useMaxPointNumber.size());
-      for (int i = 0; i < useMaxPointNumber.size(); i++) {
-        if (useMaxPointNumber.get(i)) {
-          bitMap.mark(i);
+      BitMap bitMapOfValueItselfOverflowInfo = null;
+      BitMap bitMapOfUnderflowInfo = new BitMap(underflowFlags.size());
+      for (int i = 0; i < underflowFlags.size(); i++) {
+        if (underflowFlags.get(i) == null) {
+          if (bitMapOfValueItselfOverflowInfo == null) {
+            bitMapOfValueItselfOverflowInfo = new BitMap(underflowFlags.size());
+          }
+          bitMapOfValueItselfOverflowInfo.mark(i);
+        } else if (underflowFlags.get(i)) {
+          bitMapOfUnderflowInfo.mark(i);
         }
       }
-      ReadWriteForEncodingUtils.writeUnsignedVarInt(useMaxPointNumber.size(), out);
-      out.write(bitMap.getByteArray());
+      if (bitMapOfValueItselfOverflowInfo != null) {
+        // flag of value itself contains
+        ReadWriteForEncodingUtils.writeUnsignedVarInt(Integer.MAX_VALUE - 1, out);
+      } else {
+        ReadWriteForEncodingUtils.writeUnsignedVarInt(Integer.MAX_VALUE, out);
+      }
+      ReadWriteForEncodingUtils.writeUnsignedVarInt(underflowFlags.size(), out);
+      out.write(bitMapOfUnderflowInfo.getByteArray());
+      if (bitMapOfValueItselfOverflowInfo != null) {
+        out.write(bitMapOfValueItselfOverflowInfo.getByteArray());
+      }
       out.write(ba);
     }
     reset();
   }
 
   private void reset() {
     isMaxPointNumberSaved = false;
-    useMaxPointNumber.clear();
+    underflowFlags.clear();
   }
 
-  private boolean pointsNotUseMaxPointNumber() {
-    for (boolean info : useMaxPointNumber) {
-      if (!info) {
+  private Boolean hasOverflow() {
+    for (Boolean flag : underflowFlags) {
+      if (flag == null || !flag) {
         return true;
       }
     }

java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/FloatDecoderTest.java

@@ -34,7 +34,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.apache.tsfile.utils.EncodingUtils.roundWithGivenPrecision;
 import static org.junit.Assert.assertEquals;
 
 public class FloatDecoderTest {
@@ -208,32 +207,44 @@ private void testDoubleLength(
   public void testBigFloat() throws Exception {
     float a = 0.333F;
     float b = 6.5536403E8F;
+    float c = 3.123456768E20F;
+    float d = Float.NaN;
     Encoder encoder = new FloatEncoder(TSEncoding.TS_2DIFF, TSDataType.FLOAT, 2);
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     encoder.encode(a, baos);
     encoder.encode(b, baos);
+    encoder.encode(c, baos);
+    encoder.encode(d, baos);
     encoder.flush(baos);
 
     ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
     Decoder decoder = new FloatDecoder(TSEncoding.TS_2DIFF, TSDataType.FLOAT);
-    assertEquals(roundWithGivenPrecision(a, 2), decoder.readFloat(buffer), delta);
-    assertEquals(roundWithGivenPrecision(b, 2), decoder.readFloat(buffer), delta);
+    assertEquals(a, decoder.readFloat(buffer), 0.01);
+    assertEquals(b, decoder.readFloat(buffer), 0.01);
+    assertEquals(c, decoder.readFloat(buffer), 0.01);
+    assertEquals(d, decoder.readFloat(buffer), 0.01);
   }
 
   @Test
   public void testBigDouble() throws Exception {
     double a = 0.333;
     double b = 9.223372036854E18;
+    double c = 9.223372036854E100;
+    double d = Double.NaN;
     Encoder encoder = new FloatEncoder(TSEncoding.RLE, TSDataType.DOUBLE, 2);
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     encoder.encode(a, baos);
     encoder.encode(b, baos);
+    encoder.encode(c, baos);
+    encoder.encode(d, baos);
     encoder.flush(baos);
 
     ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray());
     Decoder decoder = new FloatDecoder(TSEncoding.RLE, TSDataType.DOUBLE);
-    assertEquals(roundWithGivenPrecision(a, 2), decoder.readDouble(buffer), delta);
-    assertEquals(roundWithGivenPrecision(b, 2), decoder.readDouble(buffer), delta);
+    assertEquals(a, decoder.readDouble(buffer), 0.01);
+    assertEquals(b, decoder.readDouble(buffer), 0.01);
+    assertEquals(c, decoder.readDouble(buffer), 0.01);
+    assertEquals(d, decoder.readDouble(buffer), 0.01);
   }
 
   // private void testDecimalLenght(TSEncoding encoding, List<Double> valueList,