Fix float RLBE encoding loss of precision

cpp/pom.xml

@@ -217,5 +217,11 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>with-code-coverage</id>
+            <properties>
+                <coverage.enabled>ON</coverage.enabled>
+            </properties>
+        </profile>
     </profiles>
 </project>

java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleRLBEDecoder.java

@@ -19,180 +19,17 @@
 
 package org.apache.tsfile.encoding.decoder;
 
-import org.apache.tsfile.file.metadata.enums.TSEncoding;
-
 import java.nio.ByteBuffer;
 
-public class DoubleRLBEDecoder extends Decoder {
+public class DoubleRLBEDecoder extends LongRLBEDecoder {
+
   /** constructor of DoubleRLBEDecoder */
   public DoubleRLBEDecoder() {
-    super(TSEncoding.RLBE);
-    numberLeftInBuffer = 0;
-    byteBuffer = 0;
-  }
-
-  /** how many numbers expected to decode in following block */
-  private int blocksize;
-
-  /** origin values are stored in data */
-  private double[] data;
-
-  /** whether the first value is decoded */
-  private int writeindex = -1;
-
-  /** the pointer of current last value */
-  private int readindex = -1;
-
-  /** fibonacci values are stored in fibonacci */
-  private long[] fibonacci;
-
-  /** read bits from input stream to byteBuffer and get bit from byteBuffer */
-  private byte byteBuffer;
-
-  /** valid bits in byteBuffer */
-  private int numberLeftInBuffer;
-
-  /**
-   * read the header of a block, determine the size of block and malloc space values
-   *
-   * @param buffer inputstream buffer
-   */
-  private void readhead(ByteBuffer buffer) {
-    for (int i = 0; i <= writeindex; i++) {
-      data[i] = 0;
-    }
-    writeindex = -1;
-    readindex = -1;
-    clearBuffer(buffer);
-    readblocksize(buffer);
-    data = new double[blocksize * 2 + 1];
-    fibonacci = new long[blocksize * 2 + 1];
-    for (int i = 0; i < blocksize * 2; i++) {
-      data[i] = 0;
-      fibonacci[i] = 0;
-    }
-    fibonacci[0] = 1;
-    fibonacci[1] = 1;
-  }
-
-  /**
-   * read a block from inputstream buffer
-   *
-   * @param buffer inputstream buffer
-   */
-  private void readT(ByteBuffer buffer) {
-    // read the header of the block
-    readhead(buffer);
-    while (writeindex < blocksize - 1) {
-      int seglength = 0;
-      long runlength = 0;
-      // read first 7 bits: length of each binary words.
-      for (int j = 6; j >= 0; j--) {
-        seglength |= (readbit(buffer) << j);
-      }
-
-      // generate repeat time of rle on delta
-      int now = readbit(buffer);
-      int next = readbit(buffer);
-
-      int j = 1;
-      while (true) {
-        if (j > 1) fibonacci[j] = fibonacci[j - 1] + fibonacci[j - 2];
-        if (now == 1) runlength += fibonacci[j];
-        // when now and next are both 1, the 1 of next is the symbol of ending of fibonacci code
-        if (now == 1 && next == 1) break;
-        j++;
-        now = next;
-        next = readbit(buffer);
-      }
-      // read the delta value one by one
-      for (long i = 1; i <= runlength; i++) {
-
-        long readlongtemp = 0;
-        for (int k = seglength - 1; k >= 0; k--) {
-          readlongtemp += ((long) readbit(buffer) << k);
-        }
-        if (seglength == 64) readlongtemp -= ((long) 1 << 63);
-        double readdoubletemp = Double.longBitsToDouble(readlongtemp);
-        if (writeindex == -1) {
-          data[++writeindex] = readdoubletemp;
-        } else {
-          ++writeindex;
-          data[writeindex] = data[writeindex - 1] + readdoubletemp;
-        }
-      }
-    }
+    super();
   }
 
   @Override
   public double readDouble(ByteBuffer buffer) {
-    if (readindex < writeindex) {
-      return data[++readindex];
-    } else {
-      readT(buffer);
-      return data[++readindex];
-    }
-  }
-
-  @Override
-  public boolean hasNext(ByteBuffer buffer) {
-    return (buffer.remaining() > 0 || readindex < writeindex);
-  }
-
-  @Override
-  public void reset() {
-    // do nothing
-  }
-
-  /**
-   * get a bit from byteBuffer, when there is no bit in byteBuffer, get new 8 bits from inputstream
-   * buffer
-   *
-   * @param buffer inputstream buffer
-   * @return the top bit of byteBuffer
-   */
-  private int readbit(ByteBuffer buffer) {
-    if (numberLeftInBuffer == 0) {
-      loadBuffer(buffer);
-      numberLeftInBuffer = 8;
-    }
-    int top = ((byteBuffer >> 7) & 1);
-    byteBuffer <<= 1;
-    numberLeftInBuffer--;
-    return top;
-  }
-
-  /**
-   * get 8 bits from inputstream buffer to byteBuffer
-   *
-   * @param buffer inputstream buffer
-   */
-  private void loadBuffer(ByteBuffer buffer) {
-    byteBuffer = buffer.get();
-  }
-
-  /**
-   * clear all remaining bits in byteBuffer
-   *
-   * @param buffer inputstream buffer
-   */
-  private void clearBuffer(ByteBuffer buffer) {
-    while (numberLeftInBuffer > 0) {
-      readbit(buffer);
-    }
-  }
-
-  /**
-   * read the first integer of the block: blocksize
-   *
-   * @param buffer inputstream buffer
-   */
-  private void readblocksize(ByteBuffer buffer) {
-    blocksize = 0;
-    for (int i = 31; i >= 0; i--) {
-      if (readbit(buffer) == 1) {
-        blocksize |= (1 << i);
-      }
-    }
+    return Double.longBitsToDouble(super.readLong(buffer));
   }
 }

java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatRLBEDecoder.java

@@ -19,179 +19,17 @@
 
 package org.apache.tsfile.encoding.decoder;
 
-import org.apache.tsfile.file.metadata.enums.TSEncoding;
-
 import java.nio.ByteBuffer;
 
-public class FloatRLBEDecoder extends Decoder {
+public class FloatRLBEDecoder extends IntRLBEDecoder {
+
   /** constructor of FloatRLBEDecoder */
   public FloatRLBEDecoder() {
-    super(TSEncoding.RLBE);
-    numberLeftInBuffer = 0;
-    byteBuffer = 0;
-  }
-
-  /** how many numbers expected to decode in following block */
-  private int blocksize;
-
-  /** origin values are stored in data */
-  private float[] data;
-
-  /** whether the first value is decoded */
-  private int writeindex = -1;
-
-  /** the pointer of current last value */
-  private int readindex = -1;
-
-  /** fibonacci values are stored in fibonacci */
-  private int[] fibonacci;
-
-  /** read bits from input stream to byteBuffer and get bit from byteBuffer */
-  private byte byteBuffer;
-
-  /** valid bits in byteBuffer */
-  private int numberLeftInBuffer;
-
-  /**
-   * read the header of a block, determine the size of block and malloc space values
-   *
-   * @param buffer inputstream buffer
-   */
-  private void readhead(ByteBuffer buffer) {
-    for (int i = 0; i <= writeindex; i++) {
-      data[i] = 0;
-    }
-    writeindex = -1;
-    readindex = -1;
-    clearBuffer(buffer);
-    readblocksize(buffer);
-    data = new float[blocksize * 2 + 1];
-    fibonacci = new int[blocksize * 2 + 1];
-    for (int i = 0; i < blocksize * 2; i++) {
-      data[i] = 0;
-      fibonacci[i] = 0;
-    }
-    fibonacci[0] = 1;
-    fibonacci[1] = 1;
-  }
-
-  /**
-   * read a block from inputstream buffer
-   *
-   * @param buffer inputstream buffer
-   */
-  private void readT(ByteBuffer buffer) {
-    // read the header of the block
-    readhead(buffer);
-    while (writeindex < blocksize - 1) {
-      int seglength = 0, runlength = 0;
-      // read first 6 bits: length of each binary words.
-      for (int j = 5; j >= 0; j--) {
-        seglength |= (readbit(buffer) << j);
-      }
-
-      // generate repeat time of rle on delta
-      int now = readbit(buffer);
-      int next = readbit(buffer);
-
-      int j = 1;
-      while (true) {
-        if (j > 1) fibonacci[j] = fibonacci[j - 1] + fibonacci[j - 2];
-        if (now == 1) runlength += fibonacci[j];
-        // when now and next are both 1, the 1 of next is the symbol of ending of fibonacci code
-        if (now == 1 && next == 1) break;
-        j++;
-        now = next;
-        next = readbit(buffer);
-      }
-      // read the delta value one by one
-      for (int i = 1; i <= runlength; i++) {
-
-        int readinttemp = 0;
-        for (int k = seglength - 1; k >= 0; k--) {
-          readinttemp += (readbit(buffer) << k);
-        }
-        if (seglength == 32) readinttemp -= (1 << 31);
-        float readfloattemp = Float.intBitsToFloat(readinttemp);
-        if (writeindex == -1) {
-          data[++writeindex] = readfloattemp;
-        } else {
-          ++writeindex;
-          data[writeindex] = data[writeindex - 1] + readfloattemp;
-        }
-      }
-    }
+    super();
   }
 
   @Override
   public float readFloat(ByteBuffer buffer) {
-    if (readindex < writeindex) {
-      return data[++readindex];
-    } else {
-      readT(buffer);
-      return data[++readindex];
-    }
-  }
-
-  @Override
-  public boolean hasNext(ByteBuffer buffer) {
-    return (buffer.remaining() > 0 || readindex < writeindex);
-  }
-
-  @Override
-  public void reset() {
-    // do nothing
-  }
-
-  /**
-   * get a bit from byteBuffer, when there is no bit in byteBuffer, get new 8 bits from inputstream
-   * buffer
-   *
-   * @param buffer inputstream buffer
-   * @return the top bit of byteBuffer
-   */
-  private int readbit(ByteBuffer buffer) {
-    if (numberLeftInBuffer == 0) {
-      loadBuffer(buffer);
-      numberLeftInBuffer = 8;
-    }
-    int top = ((byteBuffer >> 7) & 1);
-    byteBuffer <<= 1;
-    numberLeftInBuffer--;
-    return top;
-  }
-
-  /**
-   * get 8 bits from inputstream buffer to byteBuffer
-   *
-   * @param buffer inputstream buffer
-   */
-  private void loadBuffer(ByteBuffer buffer) {
-    byteBuffer = buffer.get();
-  }
-
-  /**
-   * clear all remaining bits in byteBuffer
-   *
-   * @param buffer inputstream buffer
-   */
-  private void clearBuffer(ByteBuffer buffer) {
-    while (numberLeftInBuffer > 0) {
-      readbit(buffer);
-    }
-  }
-
-  /**
-   * read the first integer of the block: blocksize
-   *
-   * @param buffer inputstream buffer
-   */
-  private void readblocksize(ByteBuffer buffer) {
-    blocksize = 0;
-    for (int i = 31; i >= 0; i--) {
-      if (readbit(buffer) == 1) {
-        blocksize |= (1 << i);
-      }
-    }
+    return Float.intBitsToFloat(super.readInt(buffer));
   }
 }