diff --git a/common-api/pom.xml b/common-api/pom.xml index 6bcbde63b..db29c0769 100644 --- a/common-api/pom.xml +++ b/common-api/pom.xml @@ -28,12 +28,18 @@ common-api TsFile: Common API - - - org.openjdk.jol - jol-core - - + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + get-jar-with-dependencies diff --git a/common-api/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnBuilderStatus.java b/common-api/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnBuilderStatus.java index afe8fa8e0..287b11658 100644 --- a/common-api/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnBuilderStatus.java +++ b/common-api/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/ColumnBuilderStatus.java @@ -21,8 +21,6 @@ import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus; -import org.openjdk.jol.info.ClassLayout; - import java.lang.reflect.Field; import java.lang.reflect.Modifier; diff --git a/common-api/src/main/java/org/apache/iotdb/tsfile/utils/Constants.java b/common-api/src/main/java/org/apache/iotdb/tsfile/utils/Constants.java new file mode 100644 index 000000000..fad8af193 --- /dev/null +++ b/common-api/src/main/java/org/apache/iotdb/tsfile/utils/Constants.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tsfile.utils; + +import java.util.StringTokenizer; + +/** + * This class is copied from apache lucene, version 8.4.0. Estimates the size(memory representation) + * of Java objects. Constants.java + * Some useful constants. + */ +public final class Constants { + + private Constants() {} // can't construct + + /** JVM vendor info. */ + public static final String JVM_VENDOR = System.getProperty("java.vm.vendor"); + + public static final String JVM_VERSION = System.getProperty("java.vm.version"); + public static final String JVM_NAME = System.getProperty("java.vm.name"); + public static final String JVM_SPEC_VERSION = System.getProperty("java.specification.version"); + + /** The value of System.getProperty("java.version"). * */ + public static final String JAVA_VERSION = System.getProperty("java.version"); + + /** The value of System.getProperty("os.name"). * */ + public static final String OS_NAME = System.getProperty("os.name"); + /** True iff running on Linux. */ + public static final boolean LINUX = OS_NAME.startsWith("Linux"); + /** True iff running on Windows. */ + public static final boolean WINDOWS = OS_NAME.startsWith("Windows"); + /** True iff running on SunOS. */ + public static final boolean SUN_OS = OS_NAME.startsWith("SunOS"); + /** True iff running on Mac OS X */ + public static final boolean MAC_OS_X = OS_NAME.startsWith("Mac OS X"); + /** True iff running on FreeBSD */ + public static final boolean FREE_BSD = OS_NAME.startsWith("FreeBSD"); + + public static final String OS_ARCH = System.getProperty("os.arch"); + public static final String OS_VERSION = System.getProperty("os.version"); + public static final String JAVA_VENDOR = System.getProperty("java.vendor"); + + private static final int JVM_MAJOR_VERSION; + private static final int JVM_MINOR_VERSION; + + /** True iff running on a 64bit JVM */ + public static final boolean JRE_IS_64BIT; + + static { + final StringTokenizer st = new StringTokenizer(JVM_SPEC_VERSION, "."); + JVM_MAJOR_VERSION = Integer.parseInt(st.nextToken()); + if (st.hasMoreTokens()) { + JVM_MINOR_VERSION = Integer.parseInt(st.nextToken()); + } else { + JVM_MINOR_VERSION = 0; + } + boolean is64Bit = false; + String datamodel = null; + try { + datamodel = System.getProperty("sun.arch.data.model"); + if (datamodel != null) { + is64Bit = datamodel.contains("64"); + } + } catch (SecurityException ex) { + } + if (datamodel == null) { + if (OS_ARCH != null && OS_ARCH.contains("64")) { + is64Bit = true; + } else { + is64Bit = false; + } + } + JRE_IS_64BIT = is64Bit; + } + + public static final boolean JRE_IS_MINIMUM_JAVA8 = + JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 8); + public static final boolean JRE_IS_MINIMUM_JAVA9 = + JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 9); + public static final boolean JRE_IS_MINIMUM_JAVA11 = + JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 11); +} diff --git a/common-api/src/main/java/org/apache/iotdb/tsfile/utils/RamUsageEstimator.java b/common-api/src/main/java/org/apache/iotdb/tsfile/utils/RamUsageEstimator.java new file mode 100644 index 000000000..94841e1b6 --- /dev/null +++ b/common-api/src/main/java/org/apache/iotdb/tsfile/utils/RamUsageEstimator.java @@ -0,0 +1,588 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tsfile.utils; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.Collection; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Locale; +import java.util.Map; + +import static sun.misc.Unsafe.ARRAY_BOOLEAN_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_BOOLEAN_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_BYTE_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_CHAR_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_CHAR_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_DOUBLE_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_DOUBLE_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_FLOAT_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_FLOAT_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_INT_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_INT_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_LONG_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_LONG_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_OBJECT_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_OBJECT_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_SHORT_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_SHORT_INDEX_SCALE; + +/** + * This class is copied from apache lucene, version 8.4.0. Estimates the size(memory representation) + * of Java objects. RamUsageEstimator.java + * Estimates the size (memory representation) of Java objects. + * + *

This class uses assumptions that were discovered for the Hotspot virtual machine. If you use a + * non-OpenJDK/Oracle-based JVM, the measurements may be slightly wrong. + * + * @see #shallowSizeOf(Object) + * @see #shallowSizeOfInstance(Class) + */ +public final class RamUsageEstimator { + + /** One kilobyte bytes. */ + public static final long ONE_KB = 1024; + + /** One megabyte bytes. */ + public static final long ONE_MB = ONE_KB * ONE_KB; + + /** One gigabyte bytes. */ + public static final long ONE_GB = ONE_KB * ONE_MB; + + /** No instantiation. */ + private RamUsageEstimator() {} + + /** True, iff compressed references (oops) are enabled by this JVM */ + public static final boolean COMPRESSED_REFS_ENABLED; + + /** Number of bytes this JVM uses to represent an object reference. */ + public static final int NUM_BYTES_OBJECT_REF; + + /** Number of bytes to represent an object header (no fields, no alignments). */ + public static final int NUM_BYTES_OBJECT_HEADER; + + /** Number of bytes to represent an array header (no content, but with alignments). */ + public static final int NUM_BYTES_ARRAY_HEADER; + + /** + * A constant specifying the object alignment boundary inside the JVM. Objects will always take a + * full multiple of this constant, possibly wasting some space. + */ + public static final int NUM_BYTES_OBJECT_ALIGNMENT; + + /** + * Approximate memory usage that we assign to all unknown queries - this maps roughly to a + * BooleanQuery with a couple term clauses. + */ + public static final int QUERY_DEFAULT_RAM_BYTES_USED = 1024; + + /** + * Approximate memory usage that we assign to all unknown objects - this maps roughly to a few + * primitive fields and a couple short String-s. + */ + public static final int UNKNOWN_DEFAULT_RAM_BYTES_USED = 256; + + /** Sizes of primitive classes. */ + public static final Map, Integer> primitiveSizes; + + static { + Map, Integer> primitiveSizesMap = new IdentityHashMap<>(); + primitiveSizesMap.put(boolean.class, 1); + primitiveSizesMap.put(byte.class, 1); + primitiveSizesMap.put(char.class, Character.BYTES); + primitiveSizesMap.put(short.class, Short.BYTES); + primitiveSizesMap.put(int.class, Integer.BYTES); + primitiveSizesMap.put(float.class, Float.BYTES); + primitiveSizesMap.put(double.class, Double.BYTES); + primitiveSizesMap.put(long.class, Long.BYTES); + + primitiveSizes = Collections.unmodifiableMap(primitiveSizesMap); + } + + /** JVMs typically cache small longs. This tries to find out what the range is. */ + static final long LONG_CACHE_MIN_VALUE, LONG_CACHE_MAX_VALUE; + + static final int LONG_SIZE, STRING_SIZE; + + /** For testing only */ + static final boolean JVM_IS_HOTSPOT_64BIT; + + static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory"; + static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean"; + + // Initialize constants and try to collect information about the JVM internals. + static { + if (Constants.JRE_IS_64BIT) { + // Try to get compressed oops and object alignment (the default seems to be 8 on Hotspot); + // (this only works on 64 bit, on 32 bits the alignment and reference size is fixed): + boolean compressedOops = false; + int objectAlignment = 8; + boolean isHotspot = false; + try { + final Class beanClazz = Class.forName(HOTSPOT_BEAN_CLASS); + // we use reflection for this, because the management factory is not part + // of Java 8's compact profile: + final Object hotSpotBean = + Class.forName(MANAGEMENT_FACTORY_CLASS) + .getMethod("getPlatformMXBean", Class.class) + .invoke(null, beanClazz); + if (hotSpotBean != null) { + isHotspot = true; + final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class); + try { + final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "UseCompressedOops"); + compressedOops = + Boolean.parseBoolean( + vmOption.getClass().getMethod("getValue").invoke(vmOption).toString()); + } catch (ReflectiveOperationException | RuntimeException e) { + isHotspot = false; + } + try { + final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes"); + objectAlignment = + Integer.parseInt( + vmOption.getClass().getMethod("getValue").invoke(vmOption).toString()); + } catch (ReflectiveOperationException | RuntimeException e) { + isHotspot = false; + } + } + } catch (ReflectiveOperationException | RuntimeException e) { + isHotspot = false; + } + JVM_IS_HOTSPOT_64BIT = isHotspot; + COMPRESSED_REFS_ENABLED = compressedOops; + NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment; + // reference size is 4, if we have compressed oops: + NUM_BYTES_OBJECT_REF = COMPRESSED_REFS_ENABLED ? 4 : 8; + // "best guess" based on reference size: + NUM_BYTES_OBJECT_HEADER = 8 + NUM_BYTES_OBJECT_REF; + // array header is NUM_BYTES_OBJECT_HEADER + NUM_BYTES_INT, but aligned (object alignment): + NUM_BYTES_ARRAY_HEADER = (int) alignObjectSize(NUM_BYTES_OBJECT_HEADER + Integer.BYTES); + } else { + JVM_IS_HOTSPOT_64BIT = false; + COMPRESSED_REFS_ENABLED = false; + NUM_BYTES_OBJECT_ALIGNMENT = 8; + NUM_BYTES_OBJECT_REF = 4; + NUM_BYTES_OBJECT_HEADER = 8; + // For 32 bit JVMs, no extra alignment of array header: + NUM_BYTES_ARRAY_HEADER = NUM_BYTES_OBJECT_HEADER + Integer.BYTES; + } + + // get min/max value of cached Long class instances: + long longCacheMinValue = 0; + while (longCacheMinValue > Long.MIN_VALUE + && Long.valueOf(longCacheMinValue - 1) == Long.valueOf(longCacheMinValue - 1)) { + longCacheMinValue -= 1; + } + long longCacheMaxValue = -1; + while (longCacheMaxValue < Long.MAX_VALUE + && Long.valueOf(longCacheMaxValue + 1) == Long.valueOf(longCacheMaxValue + 1)) { + longCacheMaxValue += 1; + } + LONG_CACHE_MIN_VALUE = longCacheMinValue; + LONG_CACHE_MAX_VALUE = longCacheMaxValue; + LONG_SIZE = (int) shallowSizeOfInstance(Long.class); + STRING_SIZE = (int) shallowSizeOfInstance(String.class); + } + + /** Approximate memory usage that we assign to a Hashtable / HashMap entry. */ + public static final long HASHTABLE_RAM_BYTES_PER_ENTRY = + 2 + * NUM_BYTES_OBJECT_REF // key + value + * 2; // hash tables need to be oversized to avoid collisions, assume 2x capacity + + /** Approximate memory usage that we assign to a LinkedHashMap entry. */ + public static final long LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY = + HASHTABLE_RAM_BYTES_PER_ENTRY + 2 * NUM_BYTES_OBJECT_REF; // previous & next references + + /** Aligns an object size to be the next multiple of {@link #NUM_BYTES_OBJECT_ALIGNMENT}. */ + public static long alignObjectSize(long size) { + size += NUM_BYTES_OBJECT_ALIGNMENT - 1L; + return size - (size % NUM_BYTES_OBJECT_ALIGNMENT); + } + + /** + * Return the size of the provided {@link Long} object, returning 0 if it is cached by the JVM and + * its shallow size otherwise. + */ + public static long sizeOf(Long value) { + if (value >= LONG_CACHE_MIN_VALUE && value <= LONG_CACHE_MAX_VALUE) { + return 0; + } + return LONG_SIZE; + } + + /** Returns the size in bytes of the byte[] object. */ + public static long sizeOf(byte[] arr) { + return arr == null ? 0 : alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length); + } + + /** Returns the size in bytes of the boolean[] object. */ + public static long sizeOf(boolean[] arr) { + return arr == null ? 0 : alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length); + } + + /** Returns the size in bytes of the char[] object. */ + public static long sizeOf(char[] arr) { + return arr == null + ? 0 + : alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long) Character.BYTES * arr.length); + } + + /** Returns the size in bytes of the short[] object. */ + public static long sizeOf(short[] arr) { + return arr == null + ? 0 + : alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long) Short.BYTES * arr.length); + } + + /** Returns the size in bytes of the int[] object. */ + public static long sizeOf(int[] arr) { + return arr == null + ? 0 + : alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long) Integer.BYTES * arr.length); + } + + /** Returns the size in bytes of the float[] object. */ + public static long sizeOf(float[] arr) { + return arr == null + ? 0 + : alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long) Float.BYTES * arr.length); + } + + /** Returns the size in bytes of the long[] object. */ + public static long sizeOf(long[] arr) { + return arr == null + ? 0 + : alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long) Long.BYTES * arr.length); + } + + /** Returns the size in bytes of the double[] object. */ + public static long sizeOf(double[] arr) { + return arr == null + ? 0 + : alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long) Double.BYTES * arr.length); + } + + /** Returns the size in bytes of the String[] object. */ + public static long sizeOf(String[] arr) { + long size = shallowSizeOf(arr); + for (String s : arr) { + if (s == null) { + continue; + } + size += sizeOf(s); + } + return size; + } + + /** Recurse only into immediate descendants. */ + public static final int MAX_DEPTH = 1; + + /** + * Returns the size in bytes of a Map object, including sizes of its keys and values, supplying + * {@link #UNKNOWN_DEFAULT_RAM_BYTES_USED} when object type is not well known. This method + * recurses up to {@link #MAX_DEPTH}. + */ + public static long sizeOfMap(Map map) { + return sizeOfMap(map, 0, UNKNOWN_DEFAULT_RAM_BYTES_USED); + } + + /** + * Returns the size in bytes of a Map object, including sizes of its keys and values, supplying + * default object size when object type is not well known. This method recurses up to {@link + * #MAX_DEPTH}. + */ + public static long sizeOfMap(Map map, long defSize) { + return sizeOfMap(map, 0, defSize); + } + + private static long sizeOfMap(Map map, int depth, long defSize) { + if (map == null) { + return 0; + } + long size = shallowSizeOf(map); + if (depth > MAX_DEPTH) { + return size; + } + long sizeOfEntry = -1; + for (Map.Entry entry : map.entrySet()) { + if (sizeOfEntry == -1) { + sizeOfEntry = shallowSizeOf(entry); + } + size += sizeOfEntry; + size += sizeOfObject(entry.getKey(), depth, defSize); + size += sizeOfObject(entry.getValue(), depth, defSize); + } + return alignObjectSize(size); + } + + /** + * Returns the size in bytes of a Collection object, including sizes of its values, supplying + * {@link #UNKNOWN_DEFAULT_RAM_BYTES_USED} when object type is not well known. This method + * recurses up to {@link #MAX_DEPTH}. + */ + public static long sizeOfCollection(Collection collection) { + return sizeOfCollection(collection, 0, UNKNOWN_DEFAULT_RAM_BYTES_USED); + } + + /** + * Returns the size in bytes of a Collection object, including sizes of its values, supplying + * default object size when object type is not well known. This method recurses up to {@link + * #MAX_DEPTH}. + */ + public static long sizeOfCollection(Collection collection, long defSize) { + return sizeOfCollection(collection, 0, defSize); + } + + private static long sizeOfCollection(Collection collection, int depth, long defSize) { + if (collection == null) { + return 0; + } + long size = shallowSizeOf(collection); + if (depth > MAX_DEPTH) { + return size; + } + // assume array-backed collection and add per-object references + size += NUM_BYTES_ARRAY_HEADER + (long) collection.size() * NUM_BYTES_OBJECT_REF; + for (Object o : collection) { + size += sizeOfObject(o, depth, defSize); + } + return alignObjectSize(size); + } + + /** + * Best effort attempt to estimate the size in bytes of an undetermined object. Known types will + * be estimated according to their formulas, and all other object sizes will be estimated as + * {@link #UNKNOWN_DEFAULT_RAM_BYTES_USED}. + */ + public static long sizeOfObject(Object o) { + return sizeOfObject(o, 0, UNKNOWN_DEFAULT_RAM_BYTES_USED); + } + + /** + * Best effort attempt to estimate the size in bytes of an undetermined object. Known types will + * be estimated according to their formulas, and all other object sizes will be estimated using + * {@link #shallowSizeOf(Object)}, or using the supplied defSize parameter if its + * value is greater than 0. + */ + public static long sizeOfObject(Object o, long defSize) { + return sizeOfObject(o, 0, defSize); + } + + private static long sizeOfObject(Object o, int depth, long defSize) { + if (o == null) { + return 0; + } + long size; + if (o instanceof String) { + size = sizeOf((String) o); + } else if (o instanceof boolean[]) { + size = sizeOf((boolean[]) o); + } else if (o instanceof byte[]) { + size = sizeOf((byte[]) o); + } else if (o instanceof char[]) { + size = sizeOf((char[]) o); + } else if (o instanceof double[]) { + size = sizeOf((double[]) o); + } else if (o instanceof float[]) { + size = sizeOf((float[]) o); + } else if (o instanceof int[]) { + size = sizeOf((int[]) o); + } else if (o instanceof Long) { + size = sizeOf((Long) o); + } else if (o instanceof long[]) { + size = sizeOf((long[]) o); + } else if (o instanceof short[]) { + size = sizeOf((short[]) o); + } else if (o instanceof String[]) { + size = sizeOf((String[]) o); + } else if (o instanceof Map) { + size = sizeOfMap((Map) o, ++depth, defSize); + } else if (o instanceof Collection) { + size = sizeOfCollection((Collection) o, ++depth, defSize); + } else { + if (defSize > 0) { + size = defSize; + } else { + size = shallowSizeOf(o); + } + } + return size; + } + + /** Returns the size in bytes of the String object. */ + public static long sizeOf(String s) { + if (s == null) { + return 0; + } + // may not be true in Java 9+ and CompactStrings - but we have no way to determine this + + // char[] + hashCode + long size = STRING_SIZE + (long) NUM_BYTES_ARRAY_HEADER + (long) Character.BYTES * s.length(); + return alignObjectSize(size); + } + + /** Returns the shallow size in bytes of the Object[] object. */ + // Use this method instead of #shallowSizeOf(Object) to avoid costly reflection + public static long shallowSizeOf(Object[] arr) { + return arr == null + ? 0 + : alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_OBJECT_REF * arr.length); + } + + /** + * Estimates a "shallow" memory usage of the given object. For arrays, this will be the memory + * taken by array storage (no subreferences will be followed). For objects, this will be the + * memory taken by the fields. + * + *

JVM object alignments are also applied. + */ + public static long shallowSizeOf(Object obj) { + if (obj == null) return 0; + final Class clz = obj.getClass(); + if (clz.isArray()) { + return shallowSizeOfArray(obj); + } else { + return shallowSizeOfInstance(clz); + } + } + + /** + * Returns the shallow instance size in bytes an instance of the given class would occupy. This + * works with all conventional classes and primitive types, but not with arrays (the size then + * depends on the number of elements and varies from object to object). + * + * @see #shallowSizeOf(Object) + * @throws IllegalArgumentException if {@code clazz} is an array class. + */ + public static long shallowSizeOfInstance(Class clazz) { + if (clazz.isArray()) + throw new IllegalArgumentException("This method does not work with array classes."); + if (clazz.isPrimitive()) return primitiveSizes.get(clazz); + + long size = NUM_BYTES_OBJECT_HEADER; + + // Walk type hierarchy + for (; clazz != null; clazz = clazz.getSuperclass()) { + final Field[] fields = + AccessController.doPrivileged((PrivilegedAction) clazz::getDeclaredFields); + for (Field f : fields) { + if (!Modifier.isStatic(f.getModifiers())) { + size = adjustForField(size, f); + } + } + } + return alignObjectSize(size); + } + + /** Return shallow size of any array. */ + private static long shallowSizeOfArray(Object array) { + long size = NUM_BYTES_ARRAY_HEADER; + final int len = Array.getLength(array); + if (len > 0) { + Class arrayElementClazz = array.getClass().getComponentType(); + if (arrayElementClazz.isPrimitive()) { + size += (long) len * primitiveSizes.get(arrayElementClazz); + } else { + size += (long) NUM_BYTES_OBJECT_REF * len; + } + } + return alignObjectSize(size); + } + + /** + * This method returns the maximum representation size of an object. sizeSoFar is the + * object's size measured so far. f is the field being probed. + * + *

The returned offset will be the maximum of whatever was measured so far and f + * field's offset and representation size (unaligned). + */ + static long adjustForField(long sizeSoFar, final Field f) { + final Class type = f.getType(); + final int fsize = type.isPrimitive() ? primitiveSizes.get(type) : NUM_BYTES_OBJECT_REF; + // TODO: No alignments based on field type/ subclass fields alignments? + return sizeSoFar + fsize; + } + + /** Returns size in human-readable units (GB, MB, KB or bytes). */ + public static String humanReadableUnits(long bytes) { + return humanReadableUnits( + bytes, new DecimalFormat("0.#", DecimalFormatSymbols.getInstance(Locale.ROOT))); + } + + /** Returns size in human-readable units (GB, MB, KB or bytes). */ + public static String humanReadableUnits(long bytes, DecimalFormat df) { + if (bytes / ONE_GB > 0) { + return df.format((float) bytes / ONE_GB) + " GB"; + } else if (bytes / ONE_MB > 0) { + return df.format((float) bytes / ONE_MB) + " MB"; + } else if (bytes / ONE_KB > 0) { + return df.format((float) bytes / ONE_KB) + " KB"; + } else { + return bytes + " bytes"; + } + } + + public static long sizeOfBooleanArray(int length) { + return ARRAY_BOOLEAN_BASE_OFFSET + (((long) ARRAY_BOOLEAN_INDEX_SCALE) * length); + } + + public static long sizeOfByteArray(int length) { + return ARRAY_BYTE_BASE_OFFSET + (((long) ARRAY_BYTE_INDEX_SCALE) * length); + } + + public static long sizeOfShortArray(int length) { + return ARRAY_SHORT_BASE_OFFSET + (((long) ARRAY_SHORT_INDEX_SCALE) * length); + } + + public static long sizeOfCharArray(int length) { + return ARRAY_CHAR_BASE_OFFSET + (((long) ARRAY_CHAR_INDEX_SCALE) * length); + } + + public static long sizeOfIntArray(int length) { + return ARRAY_INT_BASE_OFFSET + (((long) ARRAY_INT_INDEX_SCALE) * length); + } + + public static long sizeOfLongArray(int length) { + return ARRAY_LONG_BASE_OFFSET + (((long) ARRAY_LONG_INDEX_SCALE) * length); + } + + public static long sizeOfFloatArray(int length) { + return ARRAY_FLOAT_BASE_OFFSET + (((long) ARRAY_FLOAT_INDEX_SCALE) * length); + } + + public static long sizeOfDoubleArray(int length) { + return ARRAY_DOUBLE_BASE_OFFSET + (((long) ARRAY_DOUBLE_INDEX_SCALE) * length); + } + + public static long sizeOfObjectArray(int length) { + return ARRAY_OBJECT_BASE_OFFSET + (((long) ARRAY_OBJECT_INDEX_SCALE) * length); + } +} diff --git a/pom.xml b/pom.xml index 7518642d4..bbb1fd4bd 100644 --- a/pom.xml +++ b/pom.xml @@ -62,24 +62,6 @@ lz4-java 1.8.0 - - - org.openjdk.jol - jol-core - 0.2 - - - io.airlift - slice - - 0.41 - org.tukaani xz diff --git a/tsfile/pom.xml b/tsfile/pom.xml index 801dc3c93..cf137beb5 100644 --- a/tsfile/pom.xml +++ b/tsfile/pom.xml @@ -60,14 +60,6 @@ org.lz4 lz4-java - - org.openjdk.jol - jol-core - - - io.airlift - slice - org.tukaani xz @@ -226,6 +218,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java index 8ffa19897..eb4d05b99 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java @@ -28,11 +28,10 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.reader.TsFileInput; import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.openjdk.jol.info.ClassLayout; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -40,11 +39,11 @@ public class ChunkHeader { - public static final int INSTANCE_SIZE = - ClassLayout.parseClass(ChunkHeader.class).instanceSize() - + ClassLayout.parseClass(TSDataType.class).instanceSize() - + ClassLayout.parseClass(CompressionType.class).instanceSize() - + ClassLayout.parseClass(TSEncoding.class).instanceSize(); + public static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ChunkHeader.class) + + RamUsageEstimator.shallowSizeOfInstance(TSDataType.class) + + RamUsageEstimator.shallowSizeOfInstance(CompressionType.class) + + RamUsageEstimator.shallowSizeOfInstance(TSEncoding.class); /** * 1 means this chunk has more than one page, so each page has its own page statistic. 5 means diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java index efaaedb10..21643c5fe 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java @@ -26,8 +26,6 @@ import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.openjdk.jol.info.ClassLayout; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -43,10 +41,10 @@ /** Metadata of one chunk. */ public class ChunkMetadata implements IChunkMetadata { - private static final int INSTANCE_SIZE = - ClassLayout.parseClass(ChunkMetadata.class).instanceSize() - + ClassLayout.parseClass(TSDataType.class).instanceSize() - + ClassLayout.parseClass(ArrayList.class).instanceSize(); + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ChunkMetadata.class) + + RamUsageEstimator.shallowSizeOfInstance(TSDataType.class) + + RamUsageEstimator.shallowSizeOfInstance(ArrayList.class); private String measurementUid; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java index d14408ab7..d5d141d8c 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java @@ -24,11 +24,10 @@ import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader; import org.apache.iotdb.tsfile.read.reader.TsFileInput; import org.apache.iotdb.tsfile.utils.PublicBAOS; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.openjdk.jol.info.ClassLayout; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -39,17 +38,18 @@ import java.util.Optional; import java.util.Set; -import static io.airlift.slice.SizeOf.sizeOfCharArray; -import static io.airlift.slice.SizeOf.sizeOfObjectArray; import static org.apache.iotdb.tsfile.utils.Preconditions.checkArgument; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfCharArray; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfObjectArray; public class TimeseriesMetadata implements ITimeSeriesMetadata { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(TimeseriesMetadata.class).instanceSize() - + ClassLayout.parseClass(String.class).instanceSize() - + ClassLayout.parseClass(TSDataType.class).instanceSize() - + ClassLayout.parseClass(ArrayList.class).instanceSize(); + (int) + (RamUsageEstimator.shallowSizeOfInstance(TimeseriesMetadata.class) + + RamUsageEstimator.shallowSizeOfInstance(String.class) + + RamUsageEstimator.shallowSizeOfInstance(TSDataType.class) + + RamUsageEstimator.shallowSizeOfInstance(ArrayList.class)); /** * 0 means this time series has only one chunk, no need to save the statistic again in chunk diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java index 1e27147aa..c359afee9 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java @@ -23,24 +23,23 @@ import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.openjdk.jol.info.ClassLayout; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Objects; -import static io.airlift.slice.SizeOf.sizeOfCharArray; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfCharArray; /** Statistics for string type. */ public class BinaryStatistics extends Statistics { - public static final int INSTANCE_SIZE = - ClassLayout.parseClass(BinaryStatistics.class).instanceSize() - + 2 * ClassLayout.parseClass(Binary.class).instanceSize(); + public static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(BinaryStatistics.class) + + 2 * RamUsageEstimator.shallowSizeOfInstance(Binary.class); private static final Binary EMPTY_VALUE = new Binary("", TSFileConfig.STRING_CHARSET); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java index a386905fa..663fa186c 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java @@ -21,10 +21,9 @@ import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.openjdk.jol.info.ClassLayout; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -33,8 +32,8 @@ public class BooleanStatistics extends Statistics { - public static final int INSTANCE_SIZE = - ClassLayout.parseClass(BooleanStatistics.class).instanceSize(); + public static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(BooleanStatistics.class); private boolean firstValue; private boolean lastValue; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java index 5eee38757..8ebc25100 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java @@ -21,10 +21,9 @@ import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.openjdk.jol.info.ClassLayout; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -33,8 +32,8 @@ public class DoubleStatistics extends Statistics { - public static final int INSTANCE_SIZE = - ClassLayout.parseClass(DoubleStatistics.class).instanceSize(); + public static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(DoubleStatistics.class); private double minValue; private double maxValue; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java index 0daa74d1f..31f1c8979 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java @@ -20,10 +20,9 @@ import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.openjdk.jol.info.ClassLayout; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -33,8 +32,8 @@ /** Statistics for float type. */ public class FloatStatistics extends Statistics { - public static final int INSTANCE_SIZE = - ClassLayout.parseClass(FloatStatistics.class).instanceSize(); + public static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(FloatStatistics.class); private float minValue; private float maxValue; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java index a7f4cb008..b4fd18ba1 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java @@ -21,10 +21,9 @@ import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.openjdk.jol.info.ClassLayout; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -34,8 +33,8 @@ /** Statistics for int type. */ public class IntegerStatistics extends Statistics { - public static final int INSTANCE_SIZE = - ClassLayout.parseClass(IntegerStatistics.class).instanceSize(); + public static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(IntegerStatistics.class); private int minValue; private int maxValue; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java index 4bb358c8b..8798dfdd3 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java @@ -21,10 +21,9 @@ import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.openjdk.jol.info.ClassLayout; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -33,8 +32,8 @@ public class LongStatistics extends Statistics { - public static final int INSTANCE_SIZE = - ClassLayout.parseClass(LongStatistics.class).instanceSize(); + public static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(LongStatistics.class); private long minValue; private long maxValue; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java index c17f518d0..61224a9df 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java @@ -90,7 +90,7 @@ public static Statistics getStatsByType(TSDataType type) } } - public static int getSizeByType(TSDataType type) { + public static long getSizeByType(TSDataType type) { switch (type) { case INT32: return IntegerStatistics.INSTANCE_SIZE; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/TimeStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/TimeStatistics.java index f5b4f795d..c6d31bab9 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/TimeStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/TimeStatistics.java @@ -21,8 +21,7 @@ import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; - -import org.openjdk.jol.info.ClassLayout; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import java.io.IOException; import java.io.InputStream; @@ -31,8 +30,8 @@ public class TimeStatistics extends Statistics { - public static final int INSTANCE_SIZE = - ClassLayout.parseClass(TimeStatistics.class).instanceSize(); + public static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TimeStatistics.class); private static final String TIME = "Time"; private static final String UPDATE_STATS = "update stats"; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java index e0b07a332..99b9ace3a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java @@ -23,23 +23,22 @@ import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.utils.PublicBAOS; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; -import org.openjdk.jol.info.ClassLayout; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import static io.airlift.slice.SizeOf.sizeOfByteArray; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfByteArray; /** used in query. */ public class Chunk { - private static final int INSTANCE_SIZE = - ClassLayout.parseClass(Chunk.class).instanceSize() + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(Chunk.class) + ChunkHeader.INSTANCE_SIZE - + ClassLayout.parseClass(ByteBuffer.class).instanceSize(); + + RamUsageEstimator.shallowSizeOfInstance(ByteBuffer.class); private final ChunkHeader chunkHeader; private ByteBuffer chunkData; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlock.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlock.java index ae5cbf1bd..668b396c7 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlock.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlock.java @@ -25,10 +25,9 @@ import org.apache.iotdb.tsfile.read.common.block.column.Column; import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn; import org.apache.iotdb.tsfile.read.reader.IPointReader; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; import java.util.Iterator; import java.util.NoSuchElementException; @@ -43,7 +42,8 @@ */ public class TsBlock { - public static final int INSTANCE_SIZE = ClassLayout.parseClass(TsBlock.class).instanceSize(); + public static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(TsBlock.class); private static final Column[] EMPTY_COLUMNS = new Column[0]; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumn.java index 0c82f97cf..e383342f7 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumn.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumn.java @@ -21,21 +21,20 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; import java.util.Optional; -import static io.airlift.slice.SizeOf.sizeOfBooleanArray; -import static io.airlift.slice.SizeOf.sizeOfObjectArray; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfObjectArray; public class BinaryColumn implements Column { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(BinaryColumn.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(BinaryColumn.class); private final int arrayOffset; private final int positionCount; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java index 48706a317..965a59048 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java @@ -22,20 +22,20 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; -import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.Math.max; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.shallowSizeOf; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOf; public class BinaryColumnBuilder implements ColumnBuilder { - private static final int INSTANCE_SIZE = - ClassLayout.parseClass(BinaryColumnBuilder.class).instanceSize(); + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(BinaryColumnBuilder.class); private final ColumnBuilderStatus columnBuilderStatus; public static final BinaryColumn NULL_VALUE_BLOCK = @@ -150,6 +150,6 @@ private void growCapacity() { } private void updateArraysDataSize() { - arraysRetainedSizeInBytes = sizeOf(valueIsNull) + sizeOf(values); + arraysRetainedSizeInBytes = sizeOf(valueIsNull) + shallowSizeOf(values); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumn.java index 6d64b841c..daed6cf31 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumn.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumn.java @@ -20,20 +20,19 @@ package org.apache.iotdb.tsfile.read.common.block.column; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; import java.util.Optional; -import static io.airlift.slice.SizeOf.sizeOfBooleanArray; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; public class BooleanColumn implements Column { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(BooleanColumn.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(BooleanColumn.class); public static final int SIZE_IN_BYTES_PER_POSITION = Byte.BYTES + Byte.BYTES; private final int arrayOffset; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumnBuilder.java index 37958d678..e8a883b16 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumnBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BooleanColumnBuilder.java @@ -21,20 +21,19 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; -import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.Math.max; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOf; public class BooleanColumnBuilder implements ColumnBuilder { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(BooleanColumnBuilder.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(BooleanColumnBuilder.class); public static final BooleanColumn NULL_VALUE_BLOCK = new BooleanColumn(0, 1, new boolean[] {true}, new boolean[1]); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumn.java index 7324123b1..6042b4715 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumn.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumn.java @@ -20,21 +20,20 @@ package org.apache.iotdb.tsfile.read.common.block.column; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; import java.util.Optional; -import static io.airlift.slice.SizeOf.sizeOfBooleanArray; -import static io.airlift.slice.SizeOf.sizeOfDoubleArray; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfDoubleArray; public class DoubleColumn implements Column { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(DoubleColumn.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(DoubleColumn.class); public static final int SIZE_IN_BYTES_PER_POSITION = Double.BYTES + Byte.BYTES; private final int arrayOffset; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumnBuilder.java index 52de139ed..68e47def0 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumnBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/DoubleColumnBuilder.java @@ -21,20 +21,19 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; -import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.Math.max; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOf; public class DoubleColumnBuilder implements ColumnBuilder { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(DoubleColumnBuilder.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(DoubleColumnBuilder.class); public static final DoubleColumn NULL_VALUE_BLOCK = new DoubleColumn(0, 1, new boolean[] {true}, new double[1]); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumn.java index 69694acfa..003ad3fce 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumn.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumn.java @@ -20,20 +20,20 @@ package org.apache.iotdb.tsfile.read.common.block.column; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; import java.util.Optional; -import static io.airlift.slice.SizeOf.sizeOfBooleanArray; -import static io.airlift.slice.SizeOf.sizeOfFloatArray; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfFloatArray; public class FloatColumn implements Column { - private static final int INSTANCE_SIZE = ClassLayout.parseClass(FloatColumn.class).instanceSize(); + private static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(FloatColumn.class); public static final int SIZE_IN_BYTES_PER_POSITION = Float.BYTES + Byte.BYTES; private final int arrayOffset; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumnBuilder.java index 27823f729..ac64bae26 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumnBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/FloatColumnBuilder.java @@ -21,20 +21,19 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; -import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.Math.max; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOf; public class FloatColumnBuilder implements ColumnBuilder { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(FloatColumnBuilder.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(FloatColumnBuilder.class); public static final FloatColumn NULL_VALUE_BLOCK = new FloatColumn(0, 1, new boolean[] {true}, new float[1]); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumn.java index 8eee589c9..867cb37ee 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumn.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumn.java @@ -20,20 +20,20 @@ package org.apache.iotdb.tsfile.read.common.block.column; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; import java.util.Optional; -import static io.airlift.slice.SizeOf.sizeOfBooleanArray; -import static io.airlift.slice.SizeOf.sizeOfIntArray; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfIntArray; public class IntColumn implements Column { - private static final int INSTANCE_SIZE = ClassLayout.parseClass(IntColumn.class).instanceSize(); + private static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(IntColumn.class); public static final int SIZE_IN_BYTES_PER_POSITION = Integer.BYTES + Byte.BYTES; private final int arrayOffset; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumnBuilder.java index 78030eb10..a7a4253ae 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumnBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/IntColumnBuilder.java @@ -21,20 +21,19 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; -import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.Math.max; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOf; public class IntColumnBuilder implements ColumnBuilder { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(IntColumnBuilder.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(IntColumnBuilder.class); public static final IntColumn NULL_VALUE_BLOCK = new IntColumn(0, 1, new boolean[] {true}, new int[1]); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumn.java index 47dd972a2..0cbacc7bd 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumn.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumn.java @@ -20,20 +20,20 @@ package org.apache.iotdb.tsfile.read.common.block.column; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; import java.util.Optional; -import static io.airlift.slice.SizeOf.sizeOfBooleanArray; -import static io.airlift.slice.SizeOf.sizeOfLongArray; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfLongArray; public class LongColumn implements Column { - private static final int INSTANCE_SIZE = ClassLayout.parseClass(LongColumn.class).instanceSize(); + private static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(LongColumn.class); public static final int SIZE_IN_BYTES_PER_POSITION = Long.BYTES + Byte.BYTES; private final int arrayOffset; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumnBuilder.java index 76f3cc554..b93b70c19 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumnBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/LongColumnBuilder.java @@ -21,20 +21,19 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; -import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.Math.max; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOf; public class LongColumnBuilder implements ColumnBuilder { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(LongColumnBuilder.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(LongColumnBuilder.class); public static final LongColumn NULL_VALUE_BLOCK = new LongColumn(0, 1, new boolean[] {true}, new long[1]); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/NullColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/NullColumn.java index ffd70136d..dff65bc65 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/NullColumn.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/NullColumn.java @@ -20,8 +20,7 @@ package org.apache.iotdb.tsfile.read.common.block.column; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; - -import org.openjdk.jol.info.ClassLayout; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion; @@ -33,7 +32,7 @@ public class NullColumn implements Column { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(BooleanColumn.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(BooleanColumn.class); private final int positionCount; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthEncodedColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthEncodedColumn.java index 449471a70..8db220928 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthEncodedColumn.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/RunLengthEncodedColumn.java @@ -21,10 +21,9 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; -import org.openjdk.jol.info.ClassLayout; - import java.util.Arrays; import static java.lang.String.format; @@ -34,7 +33,7 @@ public class RunLengthEncodedColumn implements Column { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(RunLengthEncodedColumn.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(RunLengthEncodedColumn.class); private final Column value; private final int positionCount; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java index 89261c6a3..a18d5fcf3 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumn.java @@ -20,15 +20,15 @@ package org.apache.iotdb.tsfile.read.common.block.column; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; -import org.openjdk.jol.info.ClassLayout; - -import static io.airlift.slice.SizeOf.sizeOfLongArray; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.checkValidRegion; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfLongArray; public class TimeColumn implements Column { - private static final int INSTANCE_SIZE = ClassLayout.parseClass(LongColumn.class).instanceSize(); + private static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(LongColumn.class); public static final int SIZE_IN_BYTES_PER_POSITION = Long.BYTES; private final int arrayOffset; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumnBuilder.java index c5bca16b8..fa1988d29 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumnBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/TimeColumnBuilder.java @@ -21,19 +21,18 @@ import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; - -import org.openjdk.jol.info.ClassLayout; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; import java.util.Arrays; -import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.Math.max; import static org.apache.iotdb.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOf; public class TimeColumnBuilder implements ColumnBuilder { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(TimeColumnBuilder.class).instanceSize(); + (int) RamUsageEstimator.shallowSizeOfInstance(TimeColumnBuilder.class); private final ColumnBuilderStatus columnBuilderStatus; private boolean initialized; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BloomFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BloomFilter.java index 4b21f8487..2c0979548 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BloomFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BloomFilter.java @@ -20,18 +20,17 @@ import org.apache.iotdb.tsfile.common.conf.TSFileConfig; -import org.openjdk.jol.info.ClassLayout; - import java.util.BitSet; import java.util.Objects; -import static io.airlift.slice.SizeOf.sizeOfLongArray; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.sizeOfLongArray; public class BloomFilter { private static final int INSTANCE_SIZE = - ClassLayout.parseClass(BloomFilter.class).instanceSize() - + ClassLayout.parseClass(BitSet.class).instanceSize(); + (int) + (RamUsageEstimator.shallowSizeOfInstance(BloomFilter.class) + + RamUsageEstimator.shallowSizeOfInstance(BitSet.class)); private static final int MINIMAL_SIZE = 256; private static final int MAXIMAL_HASH_FUNCTION_SIZE = 8; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/RamUsageEstimator.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/RamUsageEstimator.java deleted file mode 100644 index 4c5400476..000000000 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/RamUsageEstimator.java +++ /dev/null @@ -1,926 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.tsfile.utils; - -import java.lang.management.ManagementFactory; -import java.lang.reflect.Array; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.EnumSet; -import java.util.IdentityHashMap; -import java.util.Iterator; -import java.util.Locale; -import java.util.Map; -import java.util.NoSuchElementException; - -/** - * This class is copied from apache lucene, version 4.6.1. Estimates the size(memory representation) - * of Java objects. - * https://github.com/apache/lucene-solr/blob/releases/lucene-solr/4.6.1/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java - * - * @lucene.internal - * @see #sizeOf(Object) - * @see #shallowSizeOf(Object) - * @see #shallowSizeOfInstance(Class) - */ -public final class RamUsageEstimator { - - /** JVM diagnostic features. */ - public enum JvmFeature { - OBJECT_REFERENCE_SIZE("Object reference size estimated using array index scale"), - ARRAY_HEADER_SIZE("Array header size estimated using array based offset"), - FIELD_OFFSETS("Shallow instance size based on field offsets"), - OBJECT_ALIGNMENT("Object alignment retrieved from HotSpotDiagnostic MX bean"); - - public final String description; - - JvmFeature(String description) { - this.description = description; - } - - @Override - public String toString() { - return super.name() + " (" + description + ")"; - } - } - - /** JVM info string for debugging and reports. */ - public static final String JVM_INFO_STRING; - - /** One kilobyte bytes. */ - public static final long ONE_KB = 1024; - - /** One megabyte bytes. */ - public static final long ONE_MB = ONE_KB * ONE_KB; - - /** One gigabyte bytes. */ - public static final long ONE_GB = ONE_KB * ONE_MB; - - /** No instantiation. */ - private RamUsageEstimator() {} - - public static final int NUM_BYTES_BOOLEAN = 1; - public static final int NUM_BYTES_BYTE = 1; - public static final int NUM_BYTES_CHAR = 2; - public static final int NUM_BYTES_SHORT = 2; - public static final int NUM_BYTES_INT = 4; - public static final int NUM_BYTES_FLOAT = 4; - public static final int NUM_BYTES_LONG = 8; - public static final int NUM_BYTES_DOUBLE = 8; - - /** Number of bytes this jvm uses to represent an object reference. */ - public static final int NUM_BYTES_OBJECT_REF; - - /** Number of bytes to represent an object header (no fields, no alignments). */ - public static final int NUM_BYTES_OBJECT_HEADER; - - /** Number of bytes to represent an array header (no content, but with alignments). */ - public static final int NUM_BYTES_ARRAY_HEADER; - - /** - * A constant specifying the object alignment boundary inside the JVM. Objects will always take a - * full multiple of this constant, possibly wasting some space. - */ - public static final int NUM_BYTES_OBJECT_ALIGNMENT; - - /** Sizes of primitive classes. */ - private static final Map, Integer> primitiveSizes; - - static { - primitiveSizes = new IdentityHashMap, Integer>(); - primitiveSizes.put(boolean.class, Integer.valueOf(NUM_BYTES_BOOLEAN)); - primitiveSizes.put(byte.class, Integer.valueOf(NUM_BYTES_BYTE)); - primitiveSizes.put(char.class, Integer.valueOf(NUM_BYTES_CHAR)); - primitiveSizes.put(short.class, Integer.valueOf(NUM_BYTES_SHORT)); - primitiveSizes.put(int.class, Integer.valueOf(NUM_BYTES_INT)); - primitiveSizes.put(float.class, Integer.valueOf(NUM_BYTES_FLOAT)); - primitiveSizes.put(double.class, Integer.valueOf(NUM_BYTES_DOUBLE)); - primitiveSizes.put(long.class, Integer.valueOf(NUM_BYTES_LONG)); - } - - /** A handle to sun.misc.Unsafe. */ - private static final Object theUnsafe; - - /** A handle to sun.misc.Unsafe#fieldOffset(Field). */ - private static final Method objectFieldOffsetMethod; - - /** All the supported "internal" JVM features detected at clinit. */ - private static final EnumSet supportedFeatures; - - /** Initialize constants and try to collect information about the JVM internals. */ - static { - // Initialize empirically measured defaults. We'll modify them to the current - // JVM settings later on if possible. - int referenceSize = Constants.JRE_IS_64BIT ? 8 : 4; - // The following is objectHeader + NUM_BYTES_INT, but aligned (object alignment) - // so on 64 bit JVMs it'll be align(16 + 4, @8) = 24. - - supportedFeatures = EnumSet.noneOf(JvmFeature.class); - - Class unsafeClass = null; - Object tempTheUnsafe = null; - try { - unsafeClass = Class.forName("sun.misc.Unsafe"); - final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe"); - unsafeField.setAccessible(true); - tempTheUnsafe = unsafeField.get(null); - } catch (Exception e) { - // Ignore. - } - theUnsafe = tempTheUnsafe; - - // get object reference size by getting scale factor of Object[] arrays: - try { - final Method arrayIndexScaleM = unsafeClass.getMethod("arrayIndexScale", Class.class); - referenceSize = ((Number) arrayIndexScaleM.invoke(theUnsafe, Object[].class)).intValue(); - supportedFeatures.add(JvmFeature.OBJECT_REFERENCE_SIZE); - } catch (Exception e) { - // ignore. - } - - // "best guess" based on reference size. We will attempt to modify - // these to exact values if there is supported infrastructure. - int objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8; - int arrayHeader = Constants.JRE_IS_64BIT ? (8 + 2 * referenceSize) : 12; - - // get the object header size: - // - first try out if the field offsets are not scaled (see warning in Unsafe docs) - // - get the object header size by getting the field offset of the first field of a dummy object - // If the scaling is byte-wise and unsafe is available, enable dynamic size measurement for - // estimateRamUsage(). - Method tempObjectFieldOffsetMethod = null; - try { - final Method objectFieldOffsetM = unsafeClass.getMethod("objectFieldOffset", Field.class); - final Field dummy1Field = DummyTwoLongObject.class.getDeclaredField("dummy1"); - final int ofs1 = ((Number) objectFieldOffsetM.invoke(theUnsafe, dummy1Field)).intValue(); - final Field dummy2Field = DummyTwoLongObject.class.getDeclaredField("dummy2"); - final int ofs2 = ((Number) objectFieldOffsetM.invoke(theUnsafe, dummy2Field)).intValue(); - if (Math.abs(ofs2 - ofs1) == NUM_BYTES_LONG) { - final Field baseField = DummyOneFieldObject.class.getDeclaredField("base"); - objectHeader = ((Number) objectFieldOffsetM.invoke(theUnsafe, baseField)).intValue(); - supportedFeatures.add(JvmFeature.FIELD_OFFSETS); - tempObjectFieldOffsetMethod = objectFieldOffsetM; - } - } catch (Exception e) { - // Ignore. - } - objectFieldOffsetMethod = tempObjectFieldOffsetMethod; - - // Get the array header size by retrieving the array base offset - // (offset of the first element of an array). - try { - final Method arrayBaseOffsetM = unsafeClass.getMethod("arrayBaseOffset", Class.class); - // we calculate that only for byte[] arrays, it's actually the same for all types: - arrayHeader = ((Number) arrayBaseOffsetM.invoke(theUnsafe, byte[].class)).intValue(); - supportedFeatures.add(JvmFeature.ARRAY_HEADER_SIZE); - } catch (Exception e) { - // Ignore. - } - - NUM_BYTES_OBJECT_REF = referenceSize; - NUM_BYTES_OBJECT_HEADER = objectHeader; - NUM_BYTES_ARRAY_HEADER = arrayHeader; - - // Try to get the object alignment (the default seems to be 8 on Hotspot, - // regardless of the architecture). - int objectAlignment = 8; - try { - final Class beanClazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean"); - // Try to get the diagnostic mxbean without calling {@link - // ManagementFactory#getPlatformMBeanServer()} - // which starts AWT thread (and shows junk in the dock) on a Mac: - Object hotSpotBean; - // Java 7+, HotSpot - try { - hotSpotBean = - ManagementFactory.class - .getMethod("getPlatformMXBean", Class.class) - .invoke(null, beanClazz); - } catch (Exception e1) { - // Java 6, HotSpot - try { - Class sunMF = Class.forName("sun.management.ManagementFactory"); - hotSpotBean = sunMF.getMethod("getDiagnosticMXBean").invoke(null); - } catch (Exception e2) { - // Last resort option is an attempt to get it from ManagementFactory's server anyway (may - // start AWT). - hotSpotBean = - ManagementFactory.newPlatformMXBeanProxy( - ManagementFactory.getPlatformMBeanServer(), - "com.sun.management:type=HotSpotDiagnostic", - beanClazz); - } - } - if (hotSpotBean != null) { - final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class); - final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes"); - objectAlignment = - Integer.parseInt(vmOption.getClass().getMethod("getValue").invoke(vmOption).toString()); - supportedFeatures.add(JvmFeature.OBJECT_ALIGNMENT); - } - } catch (Exception e) { - // Ignore. - } - - NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment; - - JVM_INFO_STRING = - "[JVM: " - + Constants.JVM_NAME - + ", " - + Constants.JVM_VERSION - + ", " - + Constants.JVM_VENDOR - + ", " - + Constants.JAVA_VENDOR - + ", " - + Constants.JAVA_VERSION - + "]"; - } - - /** Cached information about a given class. */ - private static final class ClassCache { - - public final long alignedShallowInstanceSize; - public final Field[] referenceFields; - - public ClassCache(long alignedShallowInstanceSize, Field[] referenceFields) { - this.alignedShallowInstanceSize = alignedShallowInstanceSize; - this.referenceFields = referenceFields; - } - } - - // Object with just one field to determine the object header size by getting the offset of the - // dummy field: - @SuppressWarnings("unused") - private static final class DummyOneFieldObject { - - public byte base; - } - - // Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes. - // Only then we can be sure that those are real, unscaled offsets: - @SuppressWarnings("unused") - private static final class DummyTwoLongObject { - - public long dummy1, dummy2; - } - - /** - * Returns true, if the current JVM is fully supported by {@code RamUsageEstimator}. If this - * method returns {@code false} you are maybe using a 3rd party Java VM that is not supporting - * Oracle/Sun private APIs. The memory estimates can be imprecise then (no way of detecting - * compressed references, alignments, etc.). Lucene still tries to use sensible defaults. - */ - public static boolean isSupportedJVM() { - return supportedFeatures.size() == JvmFeature.values().length; - } - - /** Aligns an object size to be the next multiple of {@link #NUM_BYTES_OBJECT_ALIGNMENT}. */ - public static long alignObjectSize(long size) { - size += (long) NUM_BYTES_OBJECT_ALIGNMENT - 1L; - return size - (size % NUM_BYTES_OBJECT_ALIGNMENT); - } - - /** Returns the size in bytes of the byte[] object. */ - public static long sizeOf(byte[] arr) { - return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length); - } - - /** Returns the size in bytes of the boolean[] object. */ - public static long sizeOf(boolean[] arr) { - return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length); - } - - /** Returns the size in bytes of the char[] object. */ - public static long sizeOf(char[] arr) { - return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_CHAR * arr.length); - } - - /** Returns the size in bytes of the short[] object. */ - public static long sizeOf(short[] arr) { - return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_SHORT * arr.length); - } - - /** Returns the size in bytes of the int[] object. */ - public static long sizeOf(int[] arr) { - return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_INT * arr.length); - } - - /** Returns the size in bytes of the float[] object. */ - public static long sizeOf(float[] arr) { - return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_FLOAT * arr.length); - } - - /** Returns the size in bytes of the long[] object. */ - public static long sizeOf(long[] arr) { - return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_LONG * arr.length); - } - - /** Returns the size in bytes of the double[] object. */ - public static long sizeOf(double[] arr) { - return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_DOUBLE * arr.length); - } - - /** - * Estimates the RAM usage by the given object. It will walk the object tree and sum up all - * referenced objects. - * - *

Resource Usage: This method internally uses a set of every object seen during - * traversals so it does allocate memory (it isn't side-effect free). After the method exits, this - * memory should be GCed. - */ - public static long sizeOf(Object obj) { - return measureObjectSize(obj); - } - - /** - * Estimates a "shallow" memory usage of the given object. For arrays, this will be the memory - * taken by array storage (no subreferences will be followed). For objects, this will be the - * memory taken by the fields. - * - *

JVM object alignments are also applied. - */ - public static long shallowSizeOf(Object obj) { - if (obj == null) { - return 0; - } - final Class clz = obj.getClass(); - if (clz.isArray()) { - return shallowSizeOfArray(obj); - } else { - return shallowSizeOfInstance(clz); - } - } - - /** - * Returns the shallow instance size in bytes an instance of the given class would occupy. This - * works with all conventional classes and primitive types, but not with arrays (the size then - * depends on the number of elements and varies from object to object). - * - * @throws IllegalArgumentException if {@code clazz} is an array class. - * @see #shallowSizeOf(Object) - */ - public static long shallowSizeOfInstance(Class clazz) { - if (clazz.isArray()) { - throw new IllegalArgumentException("This method does not work with array classes."); - } - if (clazz.isPrimitive()) { - return primitiveSizes.get(clazz); - } - - long size = NUM_BYTES_OBJECT_HEADER; - - // Walk type hierarchy - for (; clazz != null; clazz = clazz.getSuperclass()) { - final Field[] fields = clazz.getDeclaredFields(); - for (Field f : fields) { - if (!Modifier.isStatic(f.getModifiers())) { - size = adjustForField(size, f); - } - } - } - return alignObjectSize(size); - } - - /** Return shallow size of any array. */ - private static long shallowSizeOfArray(Object array) { - long size = NUM_BYTES_ARRAY_HEADER; - final int len = Array.getLength(array); - if (len > 0) { - Class arrayElementClazz = array.getClass().getComponentType(); - if (arrayElementClazz.isPrimitive()) { - size += (long) len * primitiveSizes.get(arrayElementClazz); - } else { - size += (long) NUM_BYTES_OBJECT_REF * len; - } - } - return alignObjectSize(size); - } - - /* - * Non-recursive version of object descend. This consumes more memory than recursive in-depth - * traversal but prevents stack overflows on long chains of objects - * or complex graphs (a max. recursion depth on my machine was ~5000 objects linked in a chain - * so not too much). - */ - @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning - private static long measureObjectSize(Object root) { - // Objects seen so far. - final IdentityHashSet seen = new IdentityHashSet(); - // Class cache with reference Field and precalculated shallow size. - final IdentityHashMap, ClassCache> classCache = - new IdentityHashMap, ClassCache>(); - // Stack of objects pending traversal. Recursion caused stack overflows. - final ArrayList stack = new ArrayList(); - stack.add(root); - - long totalSize = 0; - while (!stack.isEmpty()) { - final Object ob = stack.remove(stack.size() - 1); - - if (ob == null || seen.contains(ob)) { - continue; - } - seen.add(ob); - - final Class obClazz = ob.getClass(); - if (obClazz.isArray()) { - /* - * Consider an array, possibly of primitive types. Push any of its references to - * the processing stack and accumulate this array's shallow size. - */ - long size = NUM_BYTES_ARRAY_HEADER; - final int len = Array.getLength(ob); - if (len > 0) { - Class componentClazz = obClazz.getComponentType(); - if (componentClazz.isPrimitive()) { - size += (long) len * primitiveSizes.get(componentClazz); - } else { - size += (long) NUM_BYTES_OBJECT_REF * len; - - // Push refs for traversal later. - for (int i = len; --i >= 0; ) { - final Object o = Array.get(ob, i); - if (o != null && !seen.contains(o)) { - stack.add(o); - } - } - } - } - totalSize += alignObjectSize(size); - } else { - /* - * Consider an object. Push any references it has to the processing stack - * and accumulate this object's shallow size. - */ - try { - ClassCache cachedInfo = classCache.get(obClazz); - if (cachedInfo == null) { - classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz)); - } - - for (Field f : cachedInfo.referenceFields) { - // Fast path to eliminate redundancies. - final Object o = f.get(ob); - if (o != null && !seen.contains(o)) { - stack.add(o); - } - } - - totalSize += cachedInfo.alignedShallowInstanceSize; - } catch (IllegalAccessException e) { - // this should never happen as we enabled setAccessible(). - throw new RuntimeException("Reflective field access failed?", e); - } - } - } - - // Help the GC (?). - seen.clear(); - stack.clear(); - classCache.clear(); - - return totalSize; - } - - /** Create a cached information about shallow size and reference fields for a given class. */ - private static ClassCache createCacheEntry(final Class clazz) { - long shallowInstanceSize = NUM_BYTES_OBJECT_HEADER; - final ArrayList referenceFields = new ArrayList(32); - for (Class c = clazz; c != null; c = c.getSuperclass()) { - final Field[] fields = c.getDeclaredFields(); - for (final Field f : fields) { - if (!Modifier.isStatic(f.getModifiers())) { - shallowInstanceSize = adjustForField(shallowInstanceSize, f); - - if (!f.getType().isPrimitive()) { - f.setAccessible(true); - referenceFields.add(f); - } - } - } - } - - ClassCache cachedInfo = - new ClassCache(alignObjectSize(shallowInstanceSize), referenceFields.toArray(new Field[0])); - return cachedInfo; - } - - /** - * This method returns the maximum representation size of an object. sizeSoFar is the - * object's size measured so far. f is the field being probed. - * - *

The returned offset will be the maximum of whatever was measured so far and f - * field's offset and representation size (unaligned). - */ - private static long adjustForField(long sizeSoFar, final Field f) { - final Class type = f.getType(); - final int fsize = type.isPrimitive() ? primitiveSizes.get(type) : NUM_BYTES_OBJECT_REF; - if (objectFieldOffsetMethod != null) { - try { - final long offsetPlusSize = - ((Number) objectFieldOffsetMethod.invoke(theUnsafe, f)).longValue() + fsize; - return Math.max(sizeSoFar, offsetPlusSize); - } catch (IllegalAccessException ex) { - throw new RuntimeException("Access problem with sun.misc.Unsafe", ex); - } catch (InvocationTargetException ite) { - final Throwable cause = ite.getCause(); - if (cause instanceof RuntimeException) { - throw (RuntimeException) cause; - } - if (cause instanceof Error) { - throw (Error) cause; - } - // this should never happen (Unsafe does not declare - // checked Exceptions for this method), but who knows? - throw new RuntimeException( - "Call to Unsafe's objectFieldOffset() throwed " - + "checked Exception when accessing field " - + f.getDeclaringClass().getName() - + "#" - + f.getName(), - cause); - } - } else { - // TODO: No alignments based on field type/ subclass fields alignments? - return sizeSoFar + fsize; - } - } - - /** Return the set of unsupported JVM features that improve the estimation. */ - public static EnumSet getUnsupportedFeatures() { - EnumSet unsupported = EnumSet.allOf(JvmFeature.class); - unsupported.removeAll(supportedFeatures); - return unsupported; - } - - /** Return the set of supported JVM features that improve the estimation. */ - public static EnumSet getSupportedFeatures() { - return EnumSet.copyOf(supportedFeatures); - } - - /** Returns size in human-readable units (GB, MB, KB or bytes). */ - public static String humanReadableUnits(long bytes) { - return humanReadableUnits( - bytes, new DecimalFormat("0.#", DecimalFormatSymbols.getInstance(Locale.ROOT))); - } - - /** Returns size in human-readable units (GB, MB, KB or bytes). */ - public static String humanReadableUnits(long bytes, DecimalFormat df) { - if (bytes / ONE_GB > 0) { - return df.format((float) bytes / ONE_GB) + " GB"; - } else if (bytes / ONE_MB > 0) { - return df.format((float) bytes / ONE_MB) + " MB"; - } else if (bytes / ONE_KB > 0) { - return df.format((float) bytes / ONE_KB) + " KB"; - } else { - return bytes + " bytes"; - } - } - - /** - * Return a human-readable size of a given object. - * - * @see #sizeOf(Object) - * @see #humanReadableUnits(long) - */ - public static String humanSizeOf(Object object) { - return humanReadableUnits(sizeOf(object)); - } - - /** - * An identity hash set implemented using open addressing. No null keys are allowed. - * - *

TODO: If this is useful outside this class, make it public - needs some work - */ - static final class IdentityHashSet implements Iterable { - - /** Default load factor. */ - public static final float DEFAULT_LOAD_FACTOR = 0.75f; - - /** Minimum capacity for the set. */ - public static final int MIN_CAPACITY = 4; - - /** All of set entries. Always of power of two length. */ - public Object[] keys; - - /** Cached number of assigned slots. */ - public int assigned; - - /** - * The load factor for this set (fraction of allocated or deleted slots before the buffers must - * be rehashed or reallocated). - */ - public final float loadFactor; - - /** Cached capacity threshold at which we must resize the buffers. */ - private int resizeThreshold; - - /** - * Creates a hash set with the default capacity of 16. load factor of {@value - * #DEFAULT_LOAD_FACTOR}. ` - */ - public IdentityHashSet() { - this(16, DEFAULT_LOAD_FACTOR); - } - - /** Creates a hash set with the given capacity, load factor of {@value #DEFAULT_LOAD_FACTOR}. */ - public IdentityHashSet(int initialCapacity) { - this(initialCapacity, DEFAULT_LOAD_FACTOR); - } - - /** Creates a hash set with the given capacity and load factor. */ - public IdentityHashSet(int initialCapacity, float loadFactor) { - initialCapacity = Math.max(MIN_CAPACITY, initialCapacity); - - assert initialCapacity > 0 - : "Initial capacity must be between (0, " + Integer.MAX_VALUE + "]."; - assert loadFactor > 0 && loadFactor < 1 : "Load factor must be between (0, 1)."; - this.loadFactor = loadFactor; - allocateBuffers(roundCapacity(initialCapacity)); - } - - /** Adds a reference to the set. Null keys are not allowed. */ - public boolean add(KType e) { - assert e != null : "Null keys not allowed."; - - if (assigned >= resizeThreshold) { - expandAndRehash(); - } - - final int mask = keys.length - 1; - int slot = rehash(e) & mask; - Object existing; - while ((existing = keys[slot]) != null) { - if (e == existing) { - return false; // already found. - } - slot = (slot + 1) & mask; - } - assigned++; - keys[slot] = e; - return true; - } - - /** Checks if the set contains a given ref. */ - public boolean contains(KType e) { - final int mask = keys.length - 1; - int slot = rehash(e) & mask; - Object existing; - while ((existing = keys[slot]) != null) { - if (e == existing) { - return true; - } - slot = (slot + 1) & mask; - } - return false; - } - - /** - * Rehash via MurmurHash. - * - *

The implementation is based on the finalization step from Austin Appleby's - * MurmurHash3. - * - * @see "http://sites.google.com/site/murmurhash/" - */ - private static int rehash(Object o) { - int k = System.identityHashCode(o); - k ^= k >>> 16; - k *= 0x85ebca6b; - k ^= k >>> 13; - k *= 0xc2b2ae35; - k ^= k >>> 16; - return k; - } - - /** - * Expand the internal storage buffers (capacity) or rehash current keys and values if there are - * a lot of deleted slots. - */ - private void expandAndRehash() { - final Object[] oldKeys = this.keys; - - assert assigned >= resizeThreshold; - allocateBuffers(nextCapacity(keys.length)); - - /* - * Rehash all assigned slots from the old hash table. - */ - final int mask = keys.length - 1; - for (int i = 0; i < oldKeys.length; i++) { - final Object key = oldKeys[i]; - if (key != null) { - int slot = rehash(key) & mask; - while (keys[slot] != null) { - slot = (slot + 1) & mask; - } - keys[slot] = key; - } - } - Arrays.fill(oldKeys, null); - } - - /** - * Allocate internal buffers for a given capacity. - * - * @param capacity New capacity (must be a power of two). - */ - private void allocateBuffers(int capacity) { - this.keys = new Object[capacity]; - this.resizeThreshold = (int) (capacity * DEFAULT_LOAD_FACTOR); - } - - /** Return the next possible capacity, counting from the current buffers' size. */ - protected int nextCapacity(int current) { - assert current > 0 && Long.bitCount(current) == 1 : "Capacity must be a power of two."; - assert ((current << 1) > 0) : "Maximum capacity exceeded (" + (0x80000000 >>> 1) + ")."; - - if (current < MIN_CAPACITY / 2) { - current = MIN_CAPACITY / 2; - } - return current << 1; - } - - /** Round the capacity to the next allowed value. */ - protected int roundCapacity(int requestedCapacity) { - // Maximum positive integer that is a power of two. - if (requestedCapacity > (0x80000000 >>> 1)) { - return (0x80000000 >>> 1); - } - - int capacity = MIN_CAPACITY; - while (capacity < requestedCapacity) { - capacity <<= 1; - } - - return capacity; - } - - public void clear() { - assigned = 0; - Arrays.fill(keys, null); - } - - public int size() { - return assigned; - } - - public boolean isEmpty() { - return size() == 0; - } - - @Override - public Iterator iterator() { - return new Iterator() { - int pos = -1; - Object nextElement = fetchNext(); - - @Override - public boolean hasNext() { - return nextElement != null; - } - - @SuppressWarnings("unchecked") - @Override - public KType next() { - Object r = this.nextElement; - if (r == null) { - throw new NoSuchElementException(); - } - this.nextElement = fetchNext(); - return (KType) r; - } - - private Object fetchNext() { - pos++; - while (pos < keys.length && keys[pos] == null) { - pos++; - } - - return (pos >= keys.length ? null : keys[pos]); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - } - - /** Some useful constants. */ - static class Constants { - - private Constants() {} // can't construct - - /** JVM vendor info. */ - public static final String JVM_VENDOR = System.getProperty("java.vm.vendor"); - - public static final String JVM_VERSION = System.getProperty("java.vm.version"); - public static final String JVM_NAME = System.getProperty("java.vm.name"); - - /** The value of System.getProperty("java.version"). */ - public static final String JAVA_VERSION = System.getProperty("java.version"); - - /** The value of System.getProperty("os.name"). */ - public static final String OS_NAME = System.getProperty("os.name"); - /** True iff running on Linux. */ - public static final boolean LINUX = OS_NAME.startsWith("Linux"); - /** True iff running on Windows. */ - public static final boolean WINDOWS = OS_NAME.startsWith("Windows"); - /** True iff running on SunOS. */ - public static final boolean SUN_OS = OS_NAME.startsWith("SunOS"); - /** True iff running on Mac OS X */ - public static final boolean MAC_OS_X = OS_NAME.startsWith("Mac OS X"); - /** True iff running on FreeBSD */ - public static final boolean FREE_BSD = OS_NAME.startsWith("FreeBSD"); - - public static final String OS_ARCH = System.getProperty("os.arch"); - public static final String OS_VERSION = System.getProperty("os.version"); - public static final String JAVA_VENDOR = System.getProperty("java.vendor"); - - /** @deprecated With Lucene 4.0, we are always on Java 6 */ - @Deprecated - public static final boolean JRE_IS_MINIMUM_JAVA6 = - Boolean.TRUE; // prevent inlining in foreign class files - - public static final boolean JRE_IS_MINIMUM_JAVA7; - public static final boolean JRE_IS_MINIMUM_JAVA8; - - /** True iff running on a 64bit JVM */ - public static final boolean JRE_IS_64BIT; - - static { - boolean is64Bit = false; - try { - final Class unsafeClass = Class.forName("sun.misc.Unsafe"); - final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe"); - unsafeField.setAccessible(true); - final Object unsafe = unsafeField.get(null); - final int addressSize = - ((Number) unsafeClass.getMethod("addressSize").invoke(unsafe)).intValue(); - // System.out.println("Address size: " + addressSize); - is64Bit = addressSize >= 8; - } catch (Exception e) { - final String x = System.getProperty("sun.arch.data.model"); - if (x != null) { - is64Bit = x.indexOf("64") != -1; - } else { - if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) { - is64Bit = true; - } else { - is64Bit = false; - } - } - } - JRE_IS_64BIT = is64Bit; - - // this method only exists in Java 7: - boolean v7 = true; - try { - Throwable.class.getMethod("getSuppressed"); - } catch (NoSuchMethodException nsme) { - v7 = false; - } - JRE_IS_MINIMUM_JAVA7 = v7; - - if (JRE_IS_MINIMUM_JAVA7) { - // this method only exists in Java 8: - boolean v8 = true; - try { - Collections.class.getMethod("emptySortedSet"); - } catch (NoSuchMethodException nsme) { - v8 = false; - } - JRE_IS_MINIMUM_JAVA8 = v8; - } else { - JRE_IS_MINIMUM_JAVA8 = false; - } - } - } -}