Skip to content

Commit afd6ec4

Browse files
authored
Modify tablet usage (#358)
* modify tablet usage * modify tablet * modify tablet * modify tablet * revert pom * add ut * modify exception message
1 parent d05cc65 commit afd6ec4

13 files changed

Lines changed: 201 additions & 115 deletions

File tree

java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,6 @@ private static void writeNonAlignedWithTablet(TsFileWriter tsFileWriter)
133133
measurementSchemas.add(new MeasurementSchema(SENSOR_1, TSDataType.INT64, TSEncoding.RLE));
134134
measurementSchemas.add(new MeasurementSchema(SENSOR_2, TSDataType.INT64, TSEncoding.RLE));
135135
Tablet tablet = new Tablet(DEVICE_2, measurementSchemas);
136-
long[] timestamps = tablet.timestamps;
137-
Object[] values = tablet.values;
138136
int rowNum = 100;
139137
int sensorNum = measurementSchemas.size();
140138
long timestamp = 1;
@@ -143,8 +141,7 @@ private static void writeNonAlignedWithTablet(TsFileWriter tsFileWriter)
143141
int row = tablet.getRowSize();
144142
tablet.addTimestamp(row, timestamp++);
145143
for (int i = 0; i < sensorNum; i++) {
146-
long[] sensor = (long[]) values[i];
147-
sensor[row] = value;
144+
tablet.addValue(row, i, value);
148145
}
149146
// write
150147
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {

java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,12 +88,11 @@ private static void writeWithTablet(
8888
long startValue)
8989
throws IOException, WriteProcessException {
9090
Tablet tablet = new Tablet(deviceId, schemas);
91-
long[] timestamps = tablet.timestamps;
9291
long sensorNum = schemas.size();
9392

9493
for (long r = 0; r < rowNum; r++, startValue++) {
9594
int row = tablet.getRowSize();
96-
timestamps[row] = startTime++;
95+
tablet.addTimestamp(row, startTime++);
9796
for (int i = 0; i < sensorNum; i++) {
9897
tablet.addValue(
9998
schemas.get(i).getMeasurementName(),

java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/AbstractResultSet.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ protected Field getNonNullField(int columnIndex) {
168168
}
169169

170170
protected Field getField(int columnIndex) {
171-
if (columnIndex > this.columnNameToColumnIndexMap.size()) {
171+
if (columnIndex > this.columnNameToColumnIndexMap.size() || columnIndex <= 0) {
172172
throw new IndexOutOfBoundsException("column index " + columnIndex + " out of bound");
173173
}
174174
Field field;

java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileGeneratorUtils.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -117,16 +117,13 @@ public static void writeWithTablet(
117117
boolean isAligned)
118118
throws IOException, WriteProcessException {
119119
Tablet tablet = new Tablet(deviceId, schemas);
120-
long[] timestamps = tablet.timestamps;
121-
Object[] values = tablet.values;
122120
long sensorNum = schemas.size();
123121

124122
for (long r = 0; r < rowNum; r++, startValue++) {
125123
int row = tablet.getRowSize();
126124
tablet.addTimestamp(row, startTime++);
127125
for (int i = 0; i < sensorNum; i++) {
128-
long[] sensor = (long[]) values[i];
129-
sensor[row] = startValue;
126+
tablet.addValue(row, i, startValue);
130127
}
131128
// write
132129
if (tablet.getRowSize() == tablet.getMaxRowNumber()) {

java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex)
233233
// TODO: changing to a column-first style by calculating the remaining page space of each
234234
// column firsts
235235
for (int row = startRowIndex; row < endRowIndex; row++) {
236-
long time = tablet.timestamps[row];
236+
long time = tablet.getTimestamps()[row];
237237
checkIsHistoryData(time);
238238
for (int columnIndex = 0; columnIndex < tablet.getSchemas().size(); columnIndex++) {
239239
if (tablet.getColumnTypes() != null
@@ -242,42 +242,43 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex)
242242
}
243243

244244
boolean isNull =
245-
tablet.bitMaps != null
246-
&& tablet.bitMaps[columnIndex] != null
247-
&& tablet.bitMaps[columnIndex].isMarked(row);
245+
tablet.getBitMaps() != null
246+
&& tablet.getBitMaps()[columnIndex] != null
247+
&& tablet.getBitMaps()[columnIndex].isMarked(row);
248248
// check isNull by bitMap in tablet
249249
ValueChunkWriter valueChunkWriter =
250250
tryToAddSeriesWriterInternal(measurementSchemas.get(columnIndex));
251251
switch (measurementSchemas.get(columnIndex).getType()) {
252252
case BOOLEAN:
253-
valueChunkWriter.write(time, ((boolean[]) tablet.values[columnIndex])[row], isNull);
253+
valueChunkWriter.write(
254+
time, ((boolean[]) tablet.getValues()[columnIndex])[row], isNull);
254255
break;
255256
case INT32:
256-
valueChunkWriter.write(time, ((int[]) tablet.values[columnIndex])[row], isNull);
257+
valueChunkWriter.write(time, ((int[]) tablet.getValues()[columnIndex])[row], isNull);
257258
break;
258259
case DATE:
259260
valueChunkWriter.write(
260261
time,
261262
isNull
262263
? 0
263264
: DateUtils.parseDateExpressionToInt(
264-
((LocalDate[]) tablet.values[columnIndex])[row]),
265+
((LocalDate[]) tablet.getValues()[columnIndex])[row]),
265266
isNull);
266267
break;
267268
case INT64:
268269
case TIMESTAMP:
269-
valueChunkWriter.write(time, ((long[]) tablet.values[columnIndex])[row], isNull);
270+
valueChunkWriter.write(time, ((long[]) tablet.getValues()[columnIndex])[row], isNull);
270271
break;
271272
case FLOAT:
272-
valueChunkWriter.write(time, ((float[]) tablet.values[columnIndex])[row], isNull);
273+
valueChunkWriter.write(time, ((float[]) tablet.getValues()[columnIndex])[row], isNull);
273274
break;
274275
case DOUBLE:
275-
valueChunkWriter.write(time, ((double[]) tablet.values[columnIndex])[row], isNull);
276+
valueChunkWriter.write(time, ((double[]) tablet.getValues()[columnIndex])[row], isNull);
276277
break;
277278
case TEXT:
278279
case BLOB:
279280
case STRING:
280-
valueChunkWriter.write(time, ((Binary[]) tablet.values[columnIndex])[row], isNull);
281+
valueChunkWriter.write(time, ((Binary[]) tablet.getValues()[columnIndex])[row], isNull);
281282
break;
282283
default:
283284
throw new UnSupportedDataTypeException(

java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -121,42 +121,51 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex)
121121
pointCount = 0;
122122
for (int row = startRowIndex; row < endRowIndex; row++) {
123123
// check isNull in tablet
124-
if (tablet.bitMaps != null
125-
&& tablet.bitMaps[column] != null
126-
&& tablet.bitMaps[column].isMarked(row)) {
124+
if (tablet.getBitMaps() != null
125+
&& tablet.getBitMaps()[column] != null
126+
&& tablet.getBitMaps()[column].isMarked(row)) {
127127
continue;
128128
}
129-
long time = tablet.timestamps[row];
129+
long time = tablet.getTimestamps()[row];
130130
checkIsHistoryData(measurementId, time);
131131
pointCount++;
132132
switch (tsDataType) {
133133
case INT32:
134-
chunkWriters.get(measurementId).write(time, ((int[]) tablet.values[column])[row]);
134+
chunkWriters.get(measurementId).write(time, ((int[]) tablet.getValues()[column])[row]);
135135
break;
136136
case DATE:
137137
chunkWriters
138138
.get(measurementId)
139139
.write(
140140
time,
141-
DateUtils.parseDateExpressionToInt(((LocalDate[]) tablet.values[column])[row]));
141+
DateUtils.parseDateExpressionToInt(
142+
((LocalDate[]) tablet.getValues()[column])[row]));
142143
break;
143144
case INT64:
144145
case TIMESTAMP:
145-
chunkWriters.get(measurementId).write(time, ((long[]) tablet.values[column])[row]);
146+
chunkWriters.get(measurementId).write(time, ((long[]) tablet.getValues()[column])[row]);
146147
break;
147148
case FLOAT:
148-
chunkWriters.get(measurementId).write(time, ((float[]) tablet.values[column])[row]);
149+
chunkWriters
150+
.get(measurementId)
151+
.write(time, ((float[]) tablet.getValues()[column])[row]);
149152
break;
150153
case DOUBLE:
151-
chunkWriters.get(measurementId).write(time, ((double[]) tablet.values[column])[row]);
154+
chunkWriters
155+
.get(measurementId)
156+
.write(time, ((double[]) tablet.getValues()[column])[row]);
152157
break;
153158
case BOOLEAN:
154-
chunkWriters.get(measurementId).write(time, ((boolean[]) tablet.values[column])[row]);
159+
chunkWriters
160+
.get(measurementId)
161+
.write(time, ((boolean[]) tablet.getValues()[column])[row]);
155162
break;
156163
case TEXT:
157164
case BLOB:
158165
case STRING:
159-
chunkWriters.get(measurementId).write(time, ((Binary[]) tablet.values[column])[row]);
166+
chunkWriters
167+
.get(measurementId)
168+
.write(time, ((Binary[]) tablet.getValues()[column])[row]);
160169
break;
161170
default:
162171
throw new UnSupportedDataTypeException(

java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java

Lines changed: 82 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -80,14 +80,11 @@ public class Tablet {
8080
/** MeasurementId->indexOf({@link MeasurementSchema}) */
8181
private final Map<String, Integer> measurementIndex;
8282

83-
/** Timestamps in this {@link Tablet} */
84-
public long[] timestamps;
83+
private long[] timestamps;
8584

86-
/** Each object is a primitive type array, which represents values of one measurement */
87-
public Object[] values;
85+
private Object[] values;
8886

89-
/** Each {@link BitMap} represents the existence of each value in the current column. */
90-
public BitMap[] bitMaps;
87+
private BitMap[] bitMaps;
9188

9289
/**
9390
* For compatibility with the usage of directly modifying Tablet content through public fields.
@@ -292,6 +289,7 @@ public void initBitMaps() {
292289
public void addTimestamp(int rowIndex, long timestamp) {
293290
timestamps[rowIndex] = timestamp;
294291
this.rowSize = Math.max(this.rowSize, rowIndex + 1);
292+
initBitMapsWithApiUsage();
295293
}
296294

297295
public void addValue(final String measurementId, final int rowIndex, final Object value) {
@@ -413,6 +411,10 @@ public void addValue(int rowIndex, String measurement, int val) {
413411

414412
@TsFileApi
415413
public void addValue(int rowIndex, int columnIndex, int val) {
414+
if (!(values[columnIndex] instanceof int[])) {
415+
throw new IllegalArgumentException(
416+
"The data type of column index " + columnIndex + " is not INT32");
417+
}
416418
final int[] sensor = (int[]) values[columnIndex];
417419
sensor[rowIndex] = val;
418420
updateBitMap(rowIndex, columnIndex, false);
@@ -426,6 +428,10 @@ public void addValue(int rowIndex, String measurement, long val) {
426428

427429
@TsFileApi
428430
public void addValue(int rowIndex, int columnIndex, long val) {
431+
if (!(values[columnIndex] instanceof long[])) {
432+
throw new IllegalArgumentException(
433+
"The data type of column index " + columnIndex + " is not INT64/TIMESTAMP");
434+
}
429435
final long[] sensor = (long[]) values[columnIndex];
430436
sensor[rowIndex] = val;
431437
updateBitMap(rowIndex, columnIndex, false);
@@ -439,6 +445,10 @@ public void addValue(int rowIndex, String measurement, float val) {
439445

440446
@TsFileApi
441447
public void addValue(int rowIndex, int columnIndex, float val) {
448+
if (!(values[columnIndex] instanceof float[])) {
449+
throw new IllegalArgumentException(
450+
"The data type of column index " + columnIndex + " is not FLOAT");
451+
}
442452
final float[] sensor = (float[]) values[columnIndex];
443453
sensor[rowIndex] = val;
444454
updateBitMap(rowIndex, columnIndex, false);
@@ -452,6 +462,10 @@ public void addValue(int rowIndex, String measurement, double val) {
452462

453463
@TsFileApi
454464
public void addValue(int rowIndex, int columnIndex, double val) {
465+
if (!(values[columnIndex] instanceof double[])) {
466+
throw new IllegalArgumentException(
467+
"The data type of column index " + columnIndex + " is not DOUBLE");
468+
}
455469
final double[] sensor = (double[]) values[columnIndex];
456470
sensor[rowIndex] = val;
457471
updateBitMap(rowIndex, columnIndex, false);
@@ -465,6 +479,10 @@ public void addValue(int rowIndex, String measurement, boolean val) {
465479

466480
@TsFileApi
467481
public void addValue(int rowIndex, int columnIndex, boolean val) {
482+
if (!(values[columnIndex] instanceof boolean[])) {
483+
throw new IllegalArgumentException(
484+
"The data type of column index " + columnIndex + " is not BOOLEAN");
485+
}
468486
final boolean[] sensor = (boolean[]) values[columnIndex];
469487
sensor[rowIndex] = val;
470488
updateBitMap(rowIndex, columnIndex, false);
@@ -478,6 +496,10 @@ public void addValue(int rowIndex, String measurement, String val) {
478496

479497
@TsFileApi
480498
public void addValue(int rowIndex, int columnIndex, String val) {
499+
if (!(values[columnIndex] instanceof Binary[])) {
500+
throw new IllegalArgumentException(
501+
"The data type of column index " + columnIndex + " is not TEXT/STRING/BLOB");
502+
}
481503
final Binary[] sensor = (Binary[]) values[columnIndex];
482504
sensor[rowIndex] = new Binary(val, TSFileConfig.STRING_CHARSET);
483505
updateBitMap(rowIndex, columnIndex, false);
@@ -491,6 +513,10 @@ public void addValue(int rowIndex, String measurement, byte[] val) {
491513

492514
@TsFileApi
493515
public void addValue(int rowIndex, int columnIndex, byte[] val) {
516+
if (!(values[columnIndex] instanceof Binary[])) {
517+
throw new IllegalArgumentException(
518+
"The data type of column index " + columnIndex + " is not TEXT/STRING/BLOB");
519+
}
494520
final Binary[] sensor = (Binary[]) values[columnIndex];
495521
sensor[rowIndex] = new Binary(val);
496522
updateBitMap(rowIndex, columnIndex, false);
@@ -504,6 +530,10 @@ public void addValue(int rowIndex, String measurement, LocalDate val) {
504530

505531
@TsFileApi
506532
public void addValue(int rowIndex, int columnIndex, LocalDate val) {
533+
if (!(values[columnIndex] instanceof LocalDate[])) {
534+
throw new IllegalArgumentException(
535+
"The data type of column index " + columnIndex + " is not DATE");
536+
}
507537
final LocalDate[] sensor = (LocalDate[]) values[columnIndex];
508538
sensor[rowIndex] = val;
509539
updateBitMap(rowIndex, columnIndex, false);
@@ -521,6 +551,15 @@ private int getColumnIndexByMeasurement(String measurement) {
521551
}
522552

523553
private void updateBitMap(int rowIndex, int columnIndex, boolean mark) {
554+
initBitMapsWithApiUsage();
555+
if (mark) {
556+
bitMaps[columnIndex].mark(rowIndex);
557+
} else {
558+
bitMaps[columnIndex].unmark(rowIndex);
559+
}
560+
}
561+
562+
private void initBitMapsWithApiUsage() {
524563
if (bitMaps == null) {
525564
initBitMaps();
526565
}
@@ -530,11 +569,6 @@ private void updateBitMap(int rowIndex, int columnIndex, boolean mark) {
530569
bitMap.markAll();
531570
}
532571
}
533-
if (mark) {
534-
bitMaps[columnIndex].mark(rowIndex);
535-
} else {
536-
bitMaps[columnIndex].unmark(rowIndex);
537-
}
538572
}
539573

540574
public List<IMeasurementSchema> getSchemas() {
@@ -1195,6 +1229,34 @@ public void setRowSize(int rowSize) {
11951229
this.rowSize = rowSize;
11961230
}
11971231

1232+
public long getTimestamp(int i) {
1233+
return timestamps[i];
1234+
}
1235+
1236+
public long[] getTimestamps() {
1237+
return timestamps;
1238+
}
1239+
1240+
public void setTimestamps(long[] timestamps) {
1241+
this.timestamps = timestamps;
1242+
}
1243+
1244+
public Object[] getValues() {
1245+
return values;
1246+
}
1247+
1248+
public void setValues(Object[] values) {
1249+
this.values = values;
1250+
}
1251+
1252+
public BitMap[] getBitMaps() {
1253+
return bitMaps;
1254+
}
1255+
1256+
public void setBitMaps(BitMap[] bitMaps) {
1257+
this.bitMaps = bitMaps;
1258+
}
1259+
11981260
public enum ColumnCategory {
11991261
TAG,
12001262
FIELD,
@@ -1247,4 +1309,13 @@ public void setTableName(String tableName) {
12471309
public List<ColumnCategory> getColumnTypes() {
12481310
return columnCategories;
12491311
}
1312+
1313+
public boolean isSorted() {
1314+
for (int i = 1; i < rowSize; i++) {
1315+
if (timestamps[i] < timestamps[i - 1]) {
1316+
return false;
1317+
}
1318+
}
1319+
return true;
1320+
}
12501321
}

0 commit comments

Comments
 (0)