diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java index 54f4fe2c51fe0..c99a119d61ea3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java @@ -324,7 +324,7 @@ private void prepareTypeConversionTest( generateMeasurementSchemas(); // Generate createTimeSeries in sender and receiver - String uuid = "bcdedit"; + String uuid = "bcdedit" + Long.toHexString(System.nanoTime()); for (Pair pair : measurementSchemas) { createTimeSeries(uuid, pair.left.getMeasurementName(), pair.left.getType().name(), senderEnv); createTimeSeries( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java index 6903504129385..fa99fa73a6ca3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java @@ -54,6 +54,7 @@ import java.util.Map; import java.util.Objects; import java.util.function.BiConsumer; +import java.util.function.IntPredicate; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -115,7 +116,9 @@ public void markAsNeedToReport() { //////////////////////////// parse //////////////////////////// protected void parse(final InsertRowNode insertRowNode) { - final int originColumnSize = insertRowNode.getMeasurements().length; + final String[] originColumnNameStringList = insertRowNode.getMeasurements(); + final int originColumnSize = + originColumnNameStringList == null ? 0 : originColumnNameStringList.length; final Integer[] originColumnIndex2FilteredColumnIndexMapperList = new Integer[originColumnSize]; // The full path is always cached when device path is deserialized @@ -127,14 +130,25 @@ protected void parse(final InsertRowNode insertRowNode) { final List rowIndexList = generateRowIndexList(originTimestampColumn); this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray(); + final MeasurementSchema[] originMeasurementSchemaList = insertRowNode.getMeasurementSchemas(); + final TsTableColumnCategory[] originColumnCategories = insertRowNode.getColumnCategories(); + final TSDataType[] originValueDataTypes = insertRowNode.getDataTypes(); + final Object[] originValues = insertRowNode.getValues(); + generateColumnIndexMapper( - insertRowNode.getMeasurements(), originColumnIndex2FilteredColumnIndexMapperList); + originColumnNameStringList, originColumnIndex2FilteredColumnIndexMapperList); final int filteredColumnSize = - Arrays.stream(originColumnIndex2FilteredColumnIndexMapperList) - .filter(Objects::nonNull) - .toArray() - .length; + compactColumnIndexMapper( + originColumnIndex2FilteredColumnIndexMapperList, + i -> + isValidOriginColumn( + originColumnNameStringList, + originMeasurementSchemaList, + originValueDataTypes, + i) + && originValues != null + && i < originValues.length); this.measurementSchemaList = new MeasurementSchema[filteredColumnSize]; this.columnNameStringList = new String[filteredColumnSize]; @@ -143,19 +157,15 @@ protected void parse(final InsertRowNode insertRowNode) { this.valueColumns = new Object[filteredColumnSize]; this.nullValueColumnBitmaps = new BitMap[filteredColumnSize]; - final MeasurementSchema[] originMeasurementSchemaList = insertRowNode.getMeasurementSchemas(); - final String[] originColumnNameStringList = insertRowNode.getMeasurements(); - final TsTableColumnCategory[] originColumnCategories = insertRowNode.getColumnCategories(); - final TSDataType[] originValueDataTypes = insertRowNode.getDataTypes(); - final Object[] originValues = insertRowNode.getValues(); - for (int i = 0; i < originColumnIndex2FilteredColumnIndexMapperList.length; i++) { if (originColumnIndex2FilteredColumnIndexMapperList[i] != null) { final int filteredColumnIndex = originColumnIndex2FilteredColumnIndexMapperList[i]; this.measurementSchemaList[filteredColumnIndex] = originMeasurementSchemaList[i]; this.columnNameStringList[filteredColumnIndex] = originColumnNameStringList[i]; this.valueColumnTypes[filteredColumnIndex] = - originColumnCategories != null && originColumnCategories[i] != null + originColumnCategories != null + && i < originColumnCategories.length + && originColumnCategories[i] != null ? originColumnCategories[i].toTsFileColumnType() : ColumnCategory.FIELD; this.valueColumnDataTypes[filteredColumnIndex] = originValueDataTypes[i]; @@ -190,7 +200,9 @@ protected void parse(final InsertRowNode insertRowNode) { } protected void parse(final InsertTabletNode insertTabletNode) throws IllegalPathException { - final int originColumnSize = insertTabletNode.getMeasurements().length; + final String[] originColumnNameStringList = insertTabletNode.getMeasurements(); + final int originColumnSize = + originColumnNameStringList == null ? 0 : originColumnNameStringList.length; final Integer[] originColumnIndex2FilteredColumnIndexMapperList = new Integer[originColumnSize]; // The full path is always cached when device path is deserialized @@ -202,14 +214,28 @@ protected void parse(final InsertTabletNode insertTabletNode) throws IllegalPath final List rowIndexList = generateRowIndexList(originTimestampColumn); this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray(); + final MeasurementSchema[] originMeasurementSchemaList = + insertTabletNode.getMeasurementSchemas(); + final TsTableColumnCategory[] originColumnCategories = insertTabletNode.getColumnCategories(); + final TSDataType[] originValueColumnDataTypes = insertTabletNode.getDataTypes(); + final Object[] originValueColumns = insertTabletNode.getColumns(); + final BitMap[] originBitMapList = insertTabletNode.getBitMaps(); + generateColumnIndexMapper( - insertTabletNode.getMeasurements(), originColumnIndex2FilteredColumnIndexMapperList); + originColumnNameStringList, originColumnIndex2FilteredColumnIndexMapperList); final int filteredColumnSize = - Arrays.stream(originColumnIndex2FilteredColumnIndexMapperList) - .filter(Objects::nonNull) - .toArray() - .length; + compactColumnIndexMapper( + originColumnIndex2FilteredColumnIndexMapperList, + i -> + isValidOriginColumn( + originColumnNameStringList, + originMeasurementSchemaList, + originValueColumnDataTypes, + i) + && originValueColumns != null + && i < originValueColumns.length + && originValueColumns[i] != null); this.measurementSchemaList = new MeasurementSchema[filteredColumnSize]; this.columnNameStringList = new String[filteredColumnSize]; @@ -218,21 +244,15 @@ protected void parse(final InsertTabletNode insertTabletNode) throws IllegalPath this.valueColumns = new Object[filteredColumnSize]; this.nullValueColumnBitmaps = new BitMap[filteredColumnSize]; - final MeasurementSchema[] originMeasurementSchemaList = - insertTabletNode.getMeasurementSchemas(); - final String[] originColumnNameStringList = insertTabletNode.getMeasurements(); - final TsTableColumnCategory[] originColumnCategories = insertTabletNode.getColumnCategories(); - final TSDataType[] originValueColumnDataTypes = insertTabletNode.getDataTypes(); - final Object[] originValueColumns = insertTabletNode.getColumns(); - final BitMap[] originBitMapList = insertTabletNode.getBitMaps(); - for (int i = 0; i < originColumnIndex2FilteredColumnIndexMapperList.length; i++) { if (originColumnIndex2FilteredColumnIndexMapperList[i] != null) { final int filteredColumnIndex = originColumnIndex2FilteredColumnIndexMapperList[i]; this.measurementSchemaList[filteredColumnIndex] = originMeasurementSchemaList[i]; this.columnNameStringList[filteredColumnIndex] = originColumnNameStringList[i]; this.valueColumnTypes[filteredColumnIndex] = - originColumnCategories != null && originColumnCategories[i] != null + originColumnCategories != null + && i < originColumnCategories.length + && originColumnCategories[i] != null ? originColumnCategories[i].toTsFileColumnType() : ColumnCategory.FIELD; this.valueColumnDataTypes[filteredColumnIndex] = originValueColumnDataTypes[i]; @@ -298,10 +318,9 @@ protected void parse(final Tablet tablet, final boolean isAligned) { originMeasurementList, originColumnIndex2FilteredColumnIndexMapperList); final int filteredColumnSize = - Arrays.stream(originColumnIndex2FilteredColumnIndexMapperList) - .filter(Objects::nonNull) - .toArray() - .length; + compactColumnIndexMapper( + originColumnIndex2FilteredColumnIndexMapperList, + i -> isValidOriginColumn(originMeasurementSchemaList, i)); this.measurementSchemaList = new MeasurementSchema[filteredColumnSize]; this.columnNameStringList = new String[filteredColumnSize]; @@ -373,6 +392,46 @@ protected abstract void generateColumnIndexMapper( final String[] originMeasurementList, final Integer[] originColumnIndex2FilteredColumnIndexMapperList); + private static int compactColumnIndexMapper( + final Integer[] originColumnIndex2FilteredColumnIndexMapperList, + final IntPredicate columnValidator) { + int filteredCount = 0; + for (int i = 0; i < originColumnIndex2FilteredColumnIndexMapperList.length; i++) { + if (originColumnIndex2FilteredColumnIndexMapperList[i] != null && columnValidator.test(i)) { + originColumnIndex2FilteredColumnIndexMapperList[i] = filteredCount++; + } else { + originColumnIndex2FilteredColumnIndexMapperList[i] = null; + } + } + return filteredCount; + } + + private static boolean isValidOriginColumn( + final String[] originColumnNameStringList, + final MeasurementSchema[] originMeasurementSchemaList, + final TSDataType[] originValueDataTypes, + final int index) { + return originColumnNameStringList != null + && index < originColumnNameStringList.length + && originColumnNameStringList[index] != null + && originMeasurementSchemaList != null + && index < originMeasurementSchemaList.length + && originMeasurementSchemaList[index] != null + && originMeasurementSchemaList[index].getType() != null + && originValueDataTypes != null + && index < originValueDataTypes.length + && originValueDataTypes[index] != null; + } + + private static boolean isValidOriginColumn( + final List originMeasurementSchemaList, final int index) { + return originMeasurementSchemaList != null + && index < originMeasurementSchemaList.size() + && originMeasurementSchemaList.get(index) != null + && originMeasurementSchemaList.get(index).getMeasurementName() != null + && originMeasurementSchemaList.get(index).getType() != null; + } + private List generateRowIndexList(final long[] originTimestampColumn) { final int rowCount = originTimestampColumn.length; if (Objects.isNull(sourceEvent) || !sourceEvent.shouldParseTime()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java index c3db7954b40be..5f70750864b9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java @@ -107,7 +107,13 @@ protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { @Override public void transferType(ZoneId zoneId) throws QueryProcessException { + if (measurementSchemas == null) { + return; + } for (int i = 0; i < measurementSchemas.length; i++) { + if (!isColumnPresent(i) || dataTypes == null || i >= dataTypes.length) { + continue; + } // null when time series doesn't exist if (measurementSchemas[i] == null) { if (!IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) { @@ -126,6 +132,9 @@ public void transferType(ZoneId zoneId) throws QueryProcessException { // parse string value to specific type dataTypes[i] = measurementSchemas[i].getType(); + if (values == null || i >= values.length || values[i] == null) { + continue; + } try { values[i] = ValueConverter.parse(values[i].toString(), dataTypes[i]); } catch (Exception e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index 677b404c217b8..54655704c1c53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -548,6 +548,9 @@ private static long sizeOfBinary(final Binary binary) { public static long sizeOfColumns( final Object[] columns, final MeasurementSchema[] measurementSchemas) { + if (Objects.isNull(columns)) { + return 0L; + } // Directly calculate if measurementSchemas are absent if (Objects.isNull(measurementSchemas)) { return RamUsageEstimator.shallowSizeOf(columns) @@ -559,7 +562,10 @@ public static long sizeOfColumns( RamUsageEstimator.alignObjectSize( NUM_BYTES_ARRAY_HEADER + NUM_BYTES_OBJECT_REF * columns.length); for (int i = 0; i < columns.length; i++) { - if (measurementSchemas[i] == null || measurementSchemas[i].getType() == null) { + if (columns[i] == null + || i >= measurementSchemas.length + || measurementSchemas[i] == null + || measurementSchemas[i].getType() == null) { continue; } switch (measurementSchemas[i].getType()) { @@ -611,6 +617,9 @@ private static long getNumBytesUnknownObject(final Object obj) { public static long sizeOfValues( final Object[] values, final MeasurementSchema[] measurementSchemas) { + if (Objects.isNull(values)) { + return 0L; + } // Directly calculate if measurementSchemas are absent if (Objects.isNull(measurementSchemas)) { return RamUsageEstimator.shallowSizeOf(values) @@ -622,7 +631,9 @@ public static long sizeOfValues( RamUsageEstimator.alignObjectSize( NUM_BYTES_ARRAY_HEADER + NUM_BYTES_OBJECT_REF * values.length); for (int i = 0; i < values.length; i++) { - if (measurementSchemas[i] == null || measurementSchemas[i].getType() == null) { + if (i >= measurementSchemas.length + || measurementSchemas[i] == null + || measurementSchemas[i].getType() == null) { size += NUM_BYTES_OBJECT_HEADER; continue; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java index 773d40e99d1f7..d7be9f548f542 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverter.java @@ -395,7 +395,7 @@ private static Pair readValuesFromBufferWithMemory( for (int i = 0; i < columns; i++) { final boolean isValueColumnsNotNull = BytesUtils.byteToBool(ReadWriteIOUtils.readByte(byteBuffer)); - if (isValueColumnsNotNull && types[i] == null) { + if (types[i] == null) { continue; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/NormalSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/NormalSchemaFetcher.java index d7d5275ef5567..04206a7f2083f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/NormalSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/NormalSchemaFetcher.java @@ -358,7 +358,9 @@ void processNormalTimeSeries( schemaComputationWithAutoCreationList.stream() .map( o -> { - TSDataType[] dataTypes = new TSDataType[o.getMeasurements().length]; + final String[] measurements = o.getMeasurements(); + TSDataType[] dataTypes = + new TSDataType[measurements == null ? 0 : measurements.length]; for (int i = 0, length = dataTypes.length; i < length; i++) { dataTypes[i] = o.getDataType(i); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java index da5f2325acdbd..0855dbeb63a04 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java @@ -174,6 +174,7 @@ public MeasurementSchema[] getMeasurementSchemas() { public void setMeasurementSchemas(MeasurementSchema[] measurementSchemas) { this.measurementSchemas = measurementSchemas; + measurementColumnCnt = -1; } public String[] getMeasurements() { @@ -189,13 +190,23 @@ public int measureColumnCnt() { } public boolean isValidMeasurement(int i) { - return measurementSchemas != null + return isValidMeasurement(i, true); + } + + public boolean isValidMeasurement(int i, boolean countFieldOnly) { + return measurements != null + && i >= 0 + && i < measurements.length + && measurements[i] != null + && measurementSchemas != null + && i < measurementSchemas.length && measurementSchemas[i] != null - && (columnCategories == null || columnCategories[i] == TsTableColumnCategory.FIELD); + && (!countFieldOnly || isFieldMeasurement(i)); } public void setMeasurements(String[] measurements) { this.measurements = measurements; + measurementColumnCnt = -1; } public TSDataType[] getDataTypes() { @@ -217,7 +228,7 @@ public int getMeasurementColumnCnt() { } public TSDataType getDataType(int index) { - return dataTypes[index]; + return dataTypes == null || index < 0 || index >= dataTypes.length ? null : dataTypes[index]; } public void setDataTypes(TSDataType[] dataTypes) { @@ -327,8 +338,14 @@ public void markFailedMeasurement(int index) { } public boolean hasValidMeasurements() { - for (Object o : measurements) { - if (o != null) { + if (measurements == null) { + return false; + } + for (int i = 0; i < measurements.length; i++) { + if (measurements[i] != null + && (columnCategories == null + || i < columnCategories.length + && columnCategories[i] == TsTableColumnCategory.FIELD)) { return true; } } @@ -345,8 +362,18 @@ public int getFailedMeasurementNumber() { protected int getValidMeasurementNumber() { int validMeasurementNumber = 0; - for (String measurement : measurements) { - if (measurement != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (measurements[i] != null) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + + public int getValidMeasurementNumber(boolean countFieldOnly) { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (isValidMeasurement(i, countFieldOnly)) { validMeasurementNumber++; } } @@ -354,15 +381,24 @@ protected int getValidMeasurementNumber() { } public boolean isMeasurementFailed(int index) { - return measurements[index] == null; + return measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null; + } + + protected boolean isWritableFieldMeasurement(int index) { + return !isMeasurementFailed(index) && isFieldMeasurement(index); + } + + public boolean isFieldMeasurement(int index) { + return columnCategories == null + || index < columnCategories.length + && columnCategories[index] == TsTableColumnCategory.FIELD; } public boolean allMeasurementFailed() { - if (measurements != null) { - return failedMeasurementNumber - >= measurements.length - (tagColumnIndices == null ? 0 : tagColumnIndices.size()); - } - return true; + return measurements == null || !hasValidMeasurements(); } // endregion @@ -418,10 +454,12 @@ public TsTableColumnCategory[] getColumnCategories() { public void setColumnCategories(TsTableColumnCategory[] columnCategories) { this.columnCategories = columnCategories; + measurementColumnCnt = -1; + tagColumnIndices = null; if (columnCategories != null) { tagColumnIndices = new ArrayList<>(); for (int i = 0; i < columnCategories.length; i++) { - if (columnCategories[i].equals(TsTableColumnCategory.TAG)) { + if (columnCategories[i] == TsTableColumnCategory.TAG) { tagColumnIndices.add(i); } } @@ -442,7 +480,9 @@ public String[] getRawMeasurements() { MeasurementSchema[] measurementSchemas = getMeasurementSchemas(); String[] rawMeasurements = new String[measurements.length]; for (int i = 0; i < measurements.length; i++) { - if (measurementSchemas[i] != null) { + if (measurementSchemas != null + && i < measurementSchemas.length + && measurementSchemas[i] != null) { // get raw measurement rather than alias rawMeasurements[i] = measurementSchemas[i].getMeasurementName(); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index fcb96cf3f0795..73463864c0660 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -184,9 +184,13 @@ public List getOutputColumnNames() { @Override public TSDataType[] getDataTypes() { if (isNeedInferType) { - TSDataType[] predictedDataTypes = new TSDataType[dataTypes.length]; - for (int i = 0; i < dataTypes.length; i++) { - predictedDataTypes[i] = TypeInferenceUtils.getPredictedDataType(values[i], true); + TSDataType[] predictedDataTypes = + new TSDataType + [dataTypes == null ? (values == null ? 0 : values.length) : dataTypes.length]; + for (int i = 0; i < predictedDataTypes.length; i++) { + predictedDataTypes[i] = + TypeInferenceUtils.getPredictedDataType( + values != null && i < values.length ? values[i] : null, true); } return predictedDataTypes; } @@ -197,9 +201,10 @@ public TSDataType[] getDataTypes() { @Override public TSDataType getDataType(int index) { if (isNeedInferType) { - return TypeInferenceUtils.getPredictedDataType(values[index], true); + return TypeInferenceUtils.getPredictedDataType( + values != null && index >= 0 && index < values.length ? values[index] : null, true); } else { - return dataTypes[index]; + return getDataTypeIfPresent(index); } } @@ -234,12 +239,96 @@ public List getTimePartitionSlots() { @Override public void markFailedMeasurement(int index) { - if (measurements[index] == null) { + if (measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null) { return; } measurements[index] = null; - dataTypes[index] = null; - values[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + if (values != null && index < values.length) { + values[index] = null; + } + measurementColumnCnt = -1; + } + + @Override + protected int getValidMeasurementNumber() { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + + @Override + public int getValidMeasurementNumber(boolean countFieldOnly) { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (values != null && i < values.length && isValidMeasurement(i, countFieldOnly)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + + protected int getValidMeasurementNumberForWAL() { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + + @Override + protected int serializeMeasurementSchemasSize() { + int byteLen = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { + byteLen += WALWriteUtils.sizeToWrite(measurementSchemas[i]); + } + } + return byteLen; + } + + @Override + protected void serializeMeasurementSchemasToWAL(IWALByteBufferView buffer) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { + WALWriteUtils.write(measurementSchemas[i], buffer); + } + } + } + + protected boolean shouldSerializeMeasurement(final int index) { + return measurements != null + && index >= 0 + && index < measurements.length + && measurements[index] != null + && values != null + && index < values.length + && (measurementSchemas == null + || index < measurementSchemas.length && measurementSchemas[index] != null) + && (values[index] == null || isNeedInferType || getDataTypeIfPresent(index) != null); + } + + protected boolean shouldSerializeMeasurementToWAL(final int index) { + return shouldSerializeMeasurement(index) + && measurementSchemas != null + && index < measurementSchemas.length + && measurementSchemas[index] != null + && (values[index] == null || !isNeedInferType && getDataTypeIfPresent(index) != null); + } + + private TSDataType getDataTypeIfPresent(final int index) { + return dataTypes != null && index >= 0 && index < dataTypes.length ? dataTypes[index] : null; } @Override @@ -268,7 +357,7 @@ void subSerialize(DataOutputStream stream) throws IOException { /** Serialize measurements and values, ignoring failed time series. */ void serializeMeasurementsAndValues(ByteBuffer buffer) { - ReadWriteIOUtils.write(measurements.length - getFailedMeasurementNumber(), buffer); + ReadWriteIOUtils.write(getValidMeasurementNumber(), buffer); serializeMeasurementsOrSchemas(buffer); putDataTypesAndValues(buffer); ReadWriteIOUtils.write((byte) (isNeedInferType ? 1 : 0), buffer); @@ -282,7 +371,7 @@ void serializeMeasurementsAndValues(ByteBuffer buffer) { * @throws IOException - If an I/O error occurs. */ void serializeMeasurementsAndValues(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(measurements.length - getFailedMeasurementNumber(), stream); + ReadWriteIOUtils.write(getValidMeasurementNumber(), stream); serializeMeasurementsOrSchemas(stream); putDataTypesAndValues(stream); ReadWriteIOUtils.write((byte) (isNeedInferType ? 1 : 0), stream); @@ -292,9 +381,9 @@ void serializeMeasurementsAndValues(DataOutputStream stream) throws IOException /** Serialize measurements or measurement schemas, ignoring failed time series. */ private void serializeMeasurementsOrSchemas(ByteBuffer buffer) { ReadWriteIOUtils.write((byte) (measurementSchemas != null ? 1 : 0), buffer); - for (int i = 0; i < measurements.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurement(i)) { continue; } // serialize measurement schemas when exist @@ -314,9 +403,9 @@ private void serializeMeasurementsOrSchemas(ByteBuffer buffer) { */ private void serializeMeasurementsOrSchemas(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write((byte) (measurementSchemas != null ? 1 : 0), stream); - for (int i = 0; i < measurements.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurement(i)) { continue; } // serialize measurement schemas when exist @@ -335,17 +424,18 @@ private void serializeMeasurementsOrSchemas(DataOutputStream stream) throws IOEx * @throws UnSupportedDataTypeException - If meets unsupported data type. */ private void putDataTypesAndValues(ByteBuffer buffer) { - for (int i = 0; i < values.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; values != null && i < values.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurement(i)) { continue; } + final TSDataType dataType = getDataTypeIfPresent(i); // serialize null value if (values[i] == null) { ReadWriteIOUtils.write( - dataTypes[i] == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, buffer); - if (dataTypes[i] != null) { - ReadWriteIOUtils.write(dataTypes[i], buffer); + dataType == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, buffer); + if (dataType != null) { + ReadWriteIOUtils.write(dataType, buffer); } continue; } @@ -355,8 +445,8 @@ private void putDataTypesAndValues(ByteBuffer buffer) { ReadWriteIOUtils.write(TYPE_RAW_STRING, buffer); ReadWriteIOUtils.write(values[i].toString(), buffer); } else { - ReadWriteIOUtils.write(dataTypes[i], buffer); - switch (dataTypes[i]) { + ReadWriteIOUtils.write(dataType, buffer); + switch (dataType) { case BOOLEAN: ReadWriteIOUtils.write((Boolean) values[i], buffer); break; @@ -381,7 +471,7 @@ private void putDataTypesAndValues(ByteBuffer buffer) { ReadWriteIOUtils.write((Binary) values[i], buffer); break; default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); + throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); } } } @@ -395,17 +485,18 @@ private void putDataTypesAndValues(ByteBuffer buffer) { * @throws UnSupportedDataTypeException - If meets unsupported data type. */ private void putDataTypesAndValues(DataOutputStream stream) throws IOException { - for (int i = 0; i < values.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; values != null && i < values.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurement(i)) { continue; } + final TSDataType dataType = getDataTypeIfPresent(i); // serialize null value if (values[i] == null) { ReadWriteIOUtils.write( - dataTypes[i] == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, stream); - if (dataTypes[i] != null) { - ReadWriteIOUtils.write(dataTypes[i], stream); + dataType == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, stream); + if (dataType != null) { + ReadWriteIOUtils.write(dataType, stream); } continue; } @@ -415,8 +506,8 @@ private void putDataTypesAndValues(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(TYPE_RAW_STRING, stream); ReadWriteIOUtils.write(values[i].toString(), stream); } else { - ReadWriteIOUtils.write(dataTypes[i], stream); - switch (dataTypes[i]) { + ReadWriteIOUtils.write(dataType, stream); + switch (dataType) { case BOOLEAN: ReadWriteIOUtils.write((Boolean) values[i], stream); break; @@ -441,7 +532,7 @@ private void putDataTypesAndValues(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write((Binary) values[i], stream); break; default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); + throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); } } } @@ -567,21 +658,22 @@ private int serializeMeasurementsAndValuesSize() { size += serializeMeasurementSchemasSize(); // putValues - for (int i = 0; i < values.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; values != null && i < values.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurementToWAL(i)) { continue; } + final TSDataType dataType = getDataTypeIfPresent(i); // serialize null value if (values[i] == null) { size += Byte.BYTES; - if (dataTypes[i] != null) { + if (dataType != null) { size += Byte.BYTES; } continue; } size += Byte.BYTES; - switch (dataTypes[i]) { + switch (dataType) { case BOOLEAN: size += Byte.BYTES; break; @@ -606,7 +698,7 @@ private int serializeMeasurementsAndValuesSize() { size += ReadWriteIOUtils.sizeToWrite((Binary) values[i]); break; default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); + throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); } } @@ -637,7 +729,7 @@ protected void subSerialize(IWALByteBufferView buffer) { /** Serialize measurements and values, ignoring failed time series. */ private void serializeMeasurementsAndValues(IWALByteBufferView buffer) { - buffer.putInt(measurements.length - getFailedMeasurementNumber()); + buffer.putInt(getValidMeasurementNumberForWAL()); serializeMeasurementSchemasToWAL(buffer); putDataTypesAndValues(buffer); buffer.put((byte) (isAligned ? 1 : 0)); @@ -650,22 +742,23 @@ private void serializeMeasurementsAndValues(IWALByteBufferView buffer) { * @throws UnSupportedDataTypeException - If meets unsupported data type. */ private void putDataTypesAndValues(IWALByteBufferView buffer) { - for (int i = 0; i < values.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; values != null && i < values.length; i++) { + // ignore failed partial insert and incomplete columns + if (!shouldSerializeMeasurementToWAL(i)) { continue; } + final TSDataType dataType = getDataTypeIfPresent(i); // serialize null value if (values[i] == null) { WALWriteUtils.write( - dataTypes[i] == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, buffer); - if (dataTypes[i] != null) { - WALWriteUtils.write(dataTypes[i], buffer); + dataType == null ? TYPE_NULL_WITHOUT_TYPE : TYPE_NULL_WITH_TYPE, buffer); + if (dataType != null) { + WALWriteUtils.write(dataType, buffer); } continue; } - WALWriteUtils.write(dataTypes[i], buffer); - switch (dataTypes[i]) { + WALWriteUtils.write(dataType, buffer); + switch (dataType) { case BOOLEAN: WALWriteUtils.write((Boolean) values[i], buffer); break; @@ -690,7 +783,7 @@ private void putDataTypesAndValues(IWALByteBufferView buffer) { WALWriteUtils.write((Binary) values[i], buffer); break; default: - throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataTypes[i]); + throw new UnSupportedDataTypeException(UNSUPPORTED_DATA_TYPE + dataType); } } } @@ -910,15 +1003,26 @@ public R accept(IPlanVisitor visitor, C context) { } public TimeValuePair composeTimeValuePair(int columnIndex) { - if (columnIndex >= values.length - || Objects.isNull(dataTypes[columnIndex]) - || dataTypes[columnIndex] == TSDataType.OBJECT) { + if (!canComposeTimeValuePair(columnIndex)) { return null; } Object value = values[columnIndex]; - return Objects.nonNull(value) - ? new TimeValuePair(time, TsPrimitiveType.getByType(dataTypes[columnIndex], value)) - : null; + return new TimeValuePair(time, TsPrimitiveType.getByType(dataTypes[columnIndex], value)); + } + + private boolean canComposeTimeValuePair(final int columnIndex) { + return measurements != null + && columnIndex >= 0 + && columnIndex < measurements.length + && values != null + && columnIndex < values.length + && values[columnIndex] != null + && dataTypes != null + && columnIndex < dataTypes.length + && dataTypes[columnIndex] != null + && dataTypes[columnIndex] != TSDataType.OBJECT + && (columnCategories == null || columnIndex < columnCategories.length) + && isWritableFieldMeasurement(columnIndex); } public void updateLastCache(String databaseName) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java index 457ab81dbca86..ccc4ca810d848 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java @@ -240,7 +240,10 @@ private void storeMeasurementsAndDataType() { for (InsertRowNode insertRowNode : insertRowNodeList) { String[] measurements = insertRowNode.getMeasurements(); TSDataType[] dataTypes = insertRowNode.getDataTypes(); - for (int i = 0; i < measurements.length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (measurements[i] == null || dataTypes == null || i >= dataTypes.length) { + continue; + } if (!measurementSet.contains(measurements[i])) { measurementList.add(measurements[i]); dataTypeList.add(dataTypes[i]); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 98d0eca98b0c3..becd347cfee8b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -332,8 +332,9 @@ protected List doSplit(Map> spli protected InsertTabletNode getEmptySplit(int count) { long[] subTimes = new long[count]; - Object[] values = initTabletValues(dataTypes.length, count, dataTypes); - BitMap[] newBitMaps = initBitmapsForSplit(dataTypes.length, count); + final int columnSize = getColumnArrayLength(); + Object[] values = initTabletValuesForSplit(columnSize, count, dataTypes); + BitMap[] newBitMaps = initBitmapsForSplit(columnSize, count); return new InsertTabletNode( getPlanNodeId(), targetPath, @@ -368,14 +369,12 @@ protected WritePlanNode generateOneSplit(Map.Entry getTimePartitionSlots() { } protected Object[] initTabletValues(int columnSize, int rowSize, TSDataType[] dataTypes) { + return initTabletValues(columnSize, rowSize, dataTypes, false); + } + + private Object[] initTabletValues( + int columnSize, int rowSize, TSDataType[] dataTypes, boolean forSplit) { Object[] values = new Object[columnSize]; for (int i = 0; i < values.length; i++) { - if (dataTypes[i] != null) { - switch (dataTypes[i]) { - case TEXT: - case BLOB: - case STRING: - case OBJECT: - values[i] = new Binary[rowSize]; - break; - case FLOAT: - values[i] = new float[rowSize]; - break; - case INT32: - case DATE: - values[i] = new int[rowSize]; - break; - case TIMESTAMP: - case INT64: - values[i] = new long[rowSize]; - break; - case DOUBLE: - values[i] = new double[rowSize]; - break; - case BOOLEAN: - values[i] = new boolean[rowSize]; - break; - } + if (dataTypes == null + || i >= dataTypes.length + || dataTypes[i] == null + || forSplit && !hasColumnForSplit(i)) { + continue; + } + switch (dataTypes[i]) { + case TEXT: + case BLOB: + case STRING: + case OBJECT: + values[i] = new Binary[rowSize]; + break; + case FLOAT: + values[i] = new float[rowSize]; + break; + case INT32: + case DATE: + values[i] = new int[rowSize]; + break; + case TIMESTAMP: + case INT64: + values[i] = new long[rowSize]; + break; + case DOUBLE: + values[i] = new double[rowSize]; + break; + case BOOLEAN: + values[i] = new boolean[rowSize]; + break; } } return values; } + protected Object[] initTabletValuesForSplit(int columnSize, int rowSize, TSDataType[] dataTypes) { + return initTabletValues(columnSize, rowSize, dataTypes, true); + } + protected BitMap[] initBitmaps(int columnSize, int rowSize) { BitMap[] bitMaps = new BitMap[columnSize]; for (int i = 0; i < columnSize; i++) { @@ -455,7 +467,8 @@ protected BitMap[] initBitmapsForSplit(int columnSize, int rowSize) { final BitMap[] splitBitMaps = new BitMap[columnSize]; boolean hasBitMap = false; for (int i = 0; i < columnSize && i < this.bitMaps.length; ++i) { - if (this.bitMaps[i] != null + if (hasColumnForSplit(i) + && this.bitMaps[i] != null && !this.bitMaps[i].isAllUnmarked(Math.min(sourceRowCount, this.bitMaps[i].getSize()))) { splitBitMaps[i] = new BitMap(rowSize); hasBitMap = true; @@ -464,14 +477,57 @@ protected BitMap[] initBitmapsForSplit(int columnSize, int rowSize) { return hasBitMap ? splitBitMaps : null; } + protected int getColumnArrayLength() { + int length = 0; + if (measurements != null) { + length = Math.max(length, measurements.length); + } + if (measurementSchemas != null) { + length = Math.max(length, measurementSchemas.length); + } + if (dataTypes != null) { + length = Math.max(length, dataTypes.length); + } + if (columns != null) { + length = Math.max(length, columns.length); + } + if (bitMaps != null) { + length = Math.max(length, bitMaps.length); + } + if (columnCategories != null) { + length = Math.max(length, columnCategories.length); + } + return length; + } + + protected boolean hasColumnForSplit(int index) { + return dataTypes != null + && index < dataTypes.length + && dataTypes[index] != null + && columns != null + && index < columns.length + && columns[index] != null + && (measurements == null || index < measurements.length && measurements[index] != null) + && (measurementSchemas == null + || index < measurementSchemas.length && measurementSchemas[index] != null); + } + @Override public void markFailedMeasurement(int index) { - if (measurements[index] == null) { + if (measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null) { return; } measurements[index] = null; - dataTypes[index] = null; - columns[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + if (columns != null && index < columns.length) { + columns[index] = null; + } + measurementColumnCnt = -1; } @Override @@ -513,12 +569,12 @@ void subSerialize(DataOutputStream stream) throws IOException { /** Serialize measurements or measurement schemas, ignoring failed time series */ private void writeMeasurementsOrSchemas(ByteBuffer buffer) { - ReadWriteIOUtils.write(measurements.length - getFailedMeasurementNumber(), buffer); + ReadWriteIOUtils.write(getValidMeasurementNumber(), buffer); ReadWriteIOUtils.write((byte) (measurementSchemas != null ? 1 : 0), buffer); - for (int i = 0; i < measurements.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } // serialize measurement schemas when exist @@ -532,12 +588,12 @@ private void writeMeasurementsOrSchemas(ByteBuffer buffer) { /** Serialize measurements or measurement schemas, ignoring failed time series */ private void writeMeasurementsOrSchemas(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(measurements.length - getFailedMeasurementNumber(), stream); + ReadWriteIOUtils.write(getValidMeasurementNumber(), stream); ReadWriteIOUtils.write((byte) (measurementSchemas != null ? 1 : 0), stream); - for (int i = 0; i < measurements.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } // serialize measurement schemas when exist @@ -551,9 +607,9 @@ private void writeMeasurementsOrSchemas(DataOutputStream stream) throws IOExcept /** Serialize data types, ignoring failed time series */ private void writeDataTypes(ByteBuffer buffer) { - for (int i = 0; i < dataTypes.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } dataTypes[i].serializeTo(buffer); @@ -562,9 +618,9 @@ private void writeDataTypes(ByteBuffer buffer) { /** Serialize data types, ignoring failed time series */ private void writeDataTypes(DataOutputStream stream) throws IOException { - for (int i = 0; i < dataTypes.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } dataTypes[i].serializeTo(stream); @@ -589,17 +645,18 @@ private void writeTimes(final DataOutputStream stream) throws IOException { private void writeBitMaps(ByteBuffer buffer) { ReadWriteIOUtils.write(BytesUtils.boolToByte(bitMaps != null), buffer); if (bitMaps != null) { - for (int i = 0; i < bitMaps.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } - if (bitMaps[i] == null) { + final BitMap bitMap = getBitMapIfPresent(i); + if (bitMap == null) { ReadWriteIOUtils.write(BytesUtils.boolToByte(false), buffer); } else { ReadWriteIOUtils.write(BytesUtils.boolToByte(true), buffer); - buffer.put(bitMaps[i].getByteArray(), 0, BitMap.getSizeOfBytes(rowCount)); + buffer.put(bitMap.getByteArray(), 0, BitMap.getSizeOfBytes(rowCount)); } } } @@ -609,17 +666,18 @@ private void writeBitMaps(ByteBuffer buffer) { private void writeBitMaps(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(BytesUtils.boolToByte(bitMaps != null), stream); if (bitMaps != null) { - for (int i = 0; i < bitMaps.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } - if (bitMaps[i] == null) { + final BitMap bitMap = getBitMapIfPresent(i); + if (bitMap == null) { ReadWriteIOUtils.write(BytesUtils.boolToByte(false), stream); } else { ReadWriteIOUtils.write(BytesUtils.boolToByte(true), stream); - stream.write(bitMaps[i].getByteArray(), 0, BitMap.getSizeOfBytes(rowCount)); + stream.write(bitMap.getByteArray(), 0, BitMap.getSizeOfBytes(rowCount)); } } } @@ -627,9 +685,9 @@ private void writeBitMaps(DataOutputStream stream) throws IOException { /** Serialize values, ignoring failed time series */ private void writeValues(ByteBuffer buffer) { - for (int i = 0; i < columns.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; columns != null && i < columns.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } serializeColumn(dataTypes[i], columns[i], buffer); @@ -638,9 +696,9 @@ private void writeValues(ByteBuffer buffer) { /** Serialize values, ignoring failed time series */ private void writeValues(DataOutputStream stream) throws IOException { - for (int i = 0; i < columns.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; columns != null && i < columns.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurement(i)) { continue; } serializeColumn(dataTypes[i], columns[i], stream); @@ -843,19 +901,20 @@ private int subSerializeSizeByRange(List rangeList) { // bitmaps size size += Byte.BYTES; if (bitMaps != null) { - for (int i = 0; i < bitMaps.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurementToWAL(i)) { continue; } size += Byte.BYTES; - if (bitMaps[i] != null) { + final BitMap bitMap = getBitMapIfPresent(i); + if (bitMap != null) { BitMap partBitMap = new BitMap(rowNumInRange); int copiedLength = 0; for (int[] range : rangeList) { int len = range[1] - range[0]; - BitMap.copyOfRange(bitMaps[i], range[0], partBitMap, copiedLength, len); + BitMap.copyOfRange(bitMap, range[0], partBitMap, copiedLength, len); copiedLength += len; } size += partBitMap.getByteArray().length; @@ -863,11 +922,12 @@ private int subSerializeSizeByRange(List rangeList) { } } // values size - for (int i = 0; i < dataTypes.length; i++) { - if (columns[i] != null) { - for (int[] range : rangeList) { - size += getColumnSize(dataTypes[i], columns[i], range[0], range[1]); - } + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (!shouldSerializeMeasurementToWAL(i)) { + continue; + } + for (int[] range : rangeList) { + size += getColumnSize(dataTypes[i], columns[i], range[0], range[1]); } } // isAlign @@ -958,7 +1018,7 @@ void subSerialize(IWALByteBufferView buffer, List rangeList, long encoded /** Serialize measurement schemas, ignoring failed time series */ protected void writeMeasurementSchemas(IWALByteBufferView buffer) { - buffer.putInt(measurements.length - getFailedMeasurementNumber()); + buffer.putInt(getValidMeasurementNumberForWAL()); serializeMeasurementSchemasToWAL(buffer); } @@ -975,13 +1035,14 @@ protected void writeTimes(IWALByteBufferView buffer, List rangeList, int protected void writeBitMaps(IWALByteBufferView buffer, List rangeList, int rowNumInRange) { buffer.put(BytesUtils.boolToByte(bitMaps != null)); if (bitMaps != null) { - for (int i = 0; i < bitMaps.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurementToWAL(i)) { continue; } - if (bitMaps[i] == null) { + final BitMap bitMap = getBitMapIfPresent(i); + if (bitMap == null) { buffer.put(BytesUtils.boolToByte(false)); } else { buffer.put(BytesUtils.boolToByte(true)); @@ -989,7 +1050,7 @@ protected void writeBitMaps(IWALByteBufferView buffer, List rangeList, in int copiedLength = 0; for (int[] startEnd : rangeList) { int len = startEnd[1] - startEnd[0]; - BitMap.copyOfRange(bitMaps[i], startEnd[0], partBitMap, copiedLength, len); + BitMap.copyOfRange(bitMap, startEnd[0], partBitMap, copiedLength, len); copiedLength += len; } buffer.put(partBitMap.getByteArray()); @@ -1000,9 +1061,9 @@ protected void writeBitMaps(IWALByteBufferView buffer, List rangeList, in /** Serialize values, ignoring failed time series */ protected void writeValues(IWALByteBufferView buffer, List rangeList) { - for (int i = 0; i < columns.length; i++) { - // ignore failed partial insert - if (measurements[i] == null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + // ignore failed partial insert and null columns + if (!shouldSerializeMeasurementToWAL(i)) { continue; } for (int[] startEnd : rangeList) { @@ -1011,6 +1072,82 @@ protected void writeValues(IWALByteBufferView buffer, List rangeList) { } } + @Override + protected int getValidMeasurementNumber() { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + + protected int getValidMeasurementNumberForWAL() { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + + @Override + public int getValidMeasurementNumber(boolean countFieldOnly) { + int validMeasurementNumber = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (isValidMeasurement(i, countFieldOnly) + && columns != null + && i < columns.length + && columns[i] != null) { + validMeasurementNumber++; + } + } + return validMeasurementNumber; + } + + @Override + protected int serializeMeasurementSchemasSize() { + int byteLen = 0; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { + byteLen += WALWriteUtils.sizeToWrite(measurementSchemas[i]); + } + } + return byteLen; + } + + @Override + protected void serializeMeasurementSchemasToWAL(IWALByteBufferView buffer) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { + WALWriteUtils.write(measurementSchemas[i], buffer); + } + } + } + + protected boolean shouldSerializeMeasurement(int index) { + return measurements != null + && index < measurements.length + && measurements[index] != null + && (measurementSchemas == null + || index < measurementSchemas.length && measurementSchemas[index] != null) + && dataTypes != null + && index < dataTypes.length + && dataTypes[index] != null + && columns != null + && index < columns.length + && columns[index] != null; + } + + protected boolean shouldSerializeMeasurementToWAL(int index) { + return shouldSerializeMeasurement(index) + && measurementSchemas != null + && index < measurementSchemas.length + && measurementSchemas[index] != null; + } + private void serializeColumn( TSDataType dataType, Object column, IWALByteBufferView buffer, int start, int end) { switch (dataType) { @@ -1183,8 +1320,9 @@ private boolean equals(Object[] columns) { } for (int i = 0; i < columns.length; i++) { - if (dataTypes[i] != null) { - switch (dataTypes[i]) { + final TSDataType dataType = getDataType(i); + if (dataType != null) { + switch (dataType) { case INT32: case DATE: if (!Arrays.equals((int[]) this.columns[i], (int[]) columns[i])) { @@ -1221,10 +1359,9 @@ private boolean equals(Object[] columns) { } break; default: - throw new UnSupportedDataTypeException( - String.format(DATATYPE_UNSUPPORTED, dataTypes[i])); + throw new UnSupportedDataTypeException(String.format(DATATYPE_UNSUPPORTED, dataType)); } - } else if (!columns[i].equals(columns)) { + } else if (!Objects.equals(this.columns[i], columns[i])) { return false; } } @@ -1239,14 +1376,14 @@ public R accept(IPlanVisitor visitor, C context) { public TimeValuePair composeLastTimeValuePair( int measurementIndex, int startOffset, int endOffset) { - if (measurementIndex >= columns.length || Objects.isNull(dataTypes[measurementIndex])) { + if (!canComposeLastTimeValuePair(measurementIndex)) { return null; } // get non-null value int lastIdx = Math.min(endOffset - 1, rowCount - 1); - if (bitMaps != null && bitMaps[measurementIndex] != null) { - BitMap bitMap = bitMaps[measurementIndex]; + final BitMap bitMap = getBitMapIfPresent(measurementIndex); + if (bitMap != null) { while (lastIdx >= startOffset) { if (!bitMap.isMarked(lastIdx)) { break; @@ -1272,11 +1409,11 @@ protected TimeValuePair composeLastTimeValuePair( if (results == null) { return composeLastTimeValuePair(measurementIndex, startOffset, endOffset); } - if (measurementIndex >= columns.length || Objects.isNull(dataTypes[measurementIndex])) { + if (!canComposeLastTimeValuePair(measurementIndex)) { return null; } - final BitMap bitMap = bitMaps == null ? null : bitMaps[measurementIndex]; + final BitMap bitMap = getBitMapIfPresent(measurementIndex); int lastIdx = Math.min(endOffset - 1, rowCount - 1); while (lastIdx >= startOffset) { if (results[lastIdx] != null @@ -1293,6 +1430,26 @@ protected TimeValuePair composeLastTimeValuePair( return lastIdx < startOffset ? null : composeTimeValuePair(measurementIndex, lastIdx); } + private boolean canComposeLastTimeValuePair(final int measurementIndex) { + return measurements != null + && measurementIndex >= 0 + && measurementIndex < measurements.length + && columns != null + && measurementIndex < columns.length + && columns[measurementIndex] != null + && dataTypes != null + && measurementIndex < dataTypes.length + && dataTypes[measurementIndex] != null + && (columnCategories == null || measurementIndex < columnCategories.length) + && isWritableFieldMeasurement(measurementIndex); + } + + private BitMap getBitMapIfPresent(final int measurementIndex) { + return bitMaps != null && measurementIndex >= 0 && measurementIndex < bitMaps.length + ? bitMaps[measurementIndex] + : null; + } + private TimeValuePair composeTimeValuePair(final int measurementIndex, final int rowIndex) { TsPrimitiveType value; switch (dataTypes[measurementIndex]) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java index 93838ffaf8b49..d6ae162c6d04f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowNode.java @@ -43,6 +43,8 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; public class RelationalInsertRowNode extends InsertRowNode { @@ -91,12 +93,16 @@ public RelationalInsertRowNode( @Override public IDeviceID getDeviceID() { if (deviceID == null) { - String[] deviceIdSegments = new String[tagColumnIndices.size() + 1]; + final List presentTagColumnIndices = getPresentTagColumnIndices(); + String[] deviceIdSegments = new String[presentTagColumnIndices.size() + 1]; deviceIdSegments[0] = this.getTableName(); - for (int i = 0; i < tagColumnIndices.size(); i++) { - final Integer columnIndex = tagColumnIndices.get(i); - deviceIdSegments[i + 1] = - getValues()[columnIndex] != null ? getValues()[columnIndex].toString() : null; + for (int i = 0; i < presentTagColumnIndices.size(); i++) { + final Integer columnIndex = presentTagColumnIndices.get(i); + final Object value = + getValues() != null && columnIndex < getValues().length + ? getValues()[columnIndex] + : null; + deviceIdSegments[i + 1] = value != null ? value.toString() : null; } deviceID = Factory.DEFAULT_FACTORY.create(deviceIdSegments); } @@ -104,6 +110,22 @@ public IDeviceID getDeviceID() { return deviceID; } + private List getPresentTagColumnIndices() { + final List presentTagColumnIndices = new ArrayList<>(); + final Object[] values = getValues(); + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG + && measurements != null + && i < measurements.length + && measurements[i] != null + && values != null + && i < values.length) { + presentTagColumnIndices.add(i); + } + } + return presentTagColumnIndices; + } + @Override public R accept(IPlanVisitor visitor, C context) { return ((PlanVisitor) visitor).visitRelationalInsertRow(this, context); @@ -178,11 +200,28 @@ protected static RelationalInsertRowNode subDeserializeFromWAL(ByteBuffer buffer return insertNode; } + @Override + protected boolean shouldSerializeMeasurement(final int index) { + return super.shouldSerializeMeasurement(index) && hasColumnCategory(index); + } + + @Override + protected boolean shouldSerializeMeasurementToWAL(final int index) { + return super.shouldSerializeMeasurementToWAL(index) && hasColumnCategory(index); + } + + private boolean hasColumnCategory(final int index) { + return columnCategories != null + && index >= 0 + && index < columnCategories.length + && columnCategories[index] != null; + } + @Override void subSerialize(ByteBuffer buffer) { super.subSerialize(buffer); - for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { columnCategories[i].serialize(buffer); } } @@ -191,8 +230,8 @@ void subSerialize(ByteBuffer buffer) { @Override void subSerialize(DataOutputStream stream) throws IOException { super.subSerialize(stream); - for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { columnCategories[i].serialize(stream); } } @@ -201,8 +240,8 @@ void subSerialize(DataOutputStream stream) throws IOException { @Override protected void subSerialize(IWALByteBufferView buffer) { super.subSerialize(buffer); - for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { buffer.put(columnCategories[i].getCategory()); } } @@ -219,7 +258,7 @@ public void subDeserialize(ByteBuffer buffer) { @Override protected int subSerializeSize() { - return super.subSerializeSize() + getValidMeasurementNumber() * Byte.BYTES; + return super.subSerializeSize() + getValidMeasurementNumberForWAL() * Byte.BYTES; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java index 7d7e1e035159a..c76e7f327dd27 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.IPlanVisitor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; @@ -62,12 +63,14 @@ public IDeviceID getDeviceID(int rowIdx) { deviceIDs = new IDeviceID[getInsertRowNodeList().size()]; } if (deviceIDs[rowIdx] == null) { - String[] deviceIdSegments = new String[tagColumnIndices.size() + 1]; - deviceIdSegments[0] = this.getTableName(); - for (int i = 0; i < tagColumnIndices.size(); i++) { - final Integer columnIndex = tagColumnIndices.get(i); - deviceIdSegments[i + 1] = - ((Object[]) getInsertRowNodeList().get(i).getValues()[columnIndex])[rowIdx].toString(); + final InsertRowNode insertRowNode = getInsertRowNodeList().get(rowIdx); + final List currentTagColumnIndices = getTagColumnIndices(insertRowNode); + String[] deviceIdSegments = new String[currentTagColumnIndices.size() + 1]; + final String tableName = insertRowNode.getTableName(); + deviceIdSegments[0] = tableName != null ? tableName : this.getTableName(); + for (int i = 0; i < currentTagColumnIndices.size(); i++) { + final Object idSegment = getValue(insertRowNode, currentTagColumnIndices.get(i)); + deviceIdSegments[i + 1] = idSegment != null ? idSegment.toString() : null; } deviceIDs[rowIdx] = Factory.DEFAULT_FACTORY.create(deviceIdSegments); } @@ -75,6 +78,36 @@ public IDeviceID getDeviceID(int rowIdx) { return deviceIDs[rowIdx]; } + private List getTagColumnIndices(final InsertRowNode insertRowNode) { + final TsTableColumnCategory[] columnCategories = insertRowNode.getColumnCategories(); + final List currentTagColumnIndices = new ArrayList<>(); + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG + && isTagColumnPresent(insertRowNode, i)) { + currentTagColumnIndices.add(i); + } + } + return currentTagColumnIndices; + } + + private boolean isTagColumnPresent(final InsertRowNode insertRowNode, final int columnIndex) { + final String[] measurements = insertRowNode.getMeasurements(); + final Object[] values = insertRowNode.getValues(); + return measurements != null + && columnIndex >= 0 + && columnIndex < measurements.length + && measurements[columnIndex] != null + && values != null + && columnIndex < values.length; + } + + private Object getValue(final InsertRowNode insertRowNode, final int columnIndex) { + final Object[] values = insertRowNode.getValues(); + return values != null && columnIndex >= 0 && columnIndex < values.length + ? values[columnIndex] + : null; + } + @Override public R accept(IPlanVisitor visitor, C context) { return ((PlanVisitor) visitor).visitRelationalInsertRows(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index 839c3db5a7cf9..b842cfc5db718 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -118,7 +118,9 @@ public void setSingleDevice() { public List getObjectColumns() { List objectColumns = new ArrayList<>(); - for (int i = 0; i < columns.length; i++) { + for (int i = 0; + columns != null && dataTypes != null && i < columns.length && i < dataTypes.length; + i++) { if (dataTypes[i] == TSDataType.OBJECT) { objectColumns.add((Binary[]) columns[i]); } @@ -133,13 +135,13 @@ public IDeviceID getDeviceID(int rowIdx) { deviceIDs = new IDeviceID[1]; } if (deviceIDs[0] == null) { - String[] deviceIdSegments = new String[tagColumnIndices.size() + 1]; + final List presentTagColumnIndices = getPresentTagColumnIndices(); + String[] deviceIdSegments = new String[presentTagColumnIndices.size() + 1]; deviceIdSegments[0] = this.getTableName(); - for (int i = 0; i < tagColumnIndices.size(); i++) { - final Integer columnIndex = tagColumnIndices.get(i); - Object idSeg = ((Object[]) columns[columnIndex])[0]; - boolean isNull = - bitMaps != null && bitMaps[columnIndex] != null && bitMaps[columnIndex].isMarked(0); + for (int i = 0; i < presentTagColumnIndices.size(); i++) { + final Integer columnIndex = presentTagColumnIndices.get(i); + Object idSeg = getColumnValue(columnIndex, 0); + boolean isNull = isNullValue(columnIndex, 0); deviceIdSegments[i + 1] = !isNull && idSeg != null ? idSeg.toString() : null; } deviceIDs[0] = Factory.DEFAULT_FACTORY.create(deviceIdSegments); @@ -150,15 +152,13 @@ public IDeviceID getDeviceID(int rowIdx) { deviceIDs = new IDeviceID[rowCount]; } if (deviceIDs[rowIdx] == null) { - String[] deviceIdSegments = new String[tagColumnIndices.size() + 1]; + final List presentTagColumnIndices = getPresentTagColumnIndices(); + String[] deviceIdSegments = new String[presentTagColumnIndices.size() + 1]; deviceIdSegments[0] = this.getTableName(); - for (int i = 0; i < tagColumnIndices.size(); i++) { - final Integer columnIndex = tagColumnIndices.get(i); - Object idSeg = ((Object[]) columns[columnIndex])[rowIdx]; - boolean isNull = - bitMaps != null - && bitMaps[columnIndex] != null - && bitMaps[columnIndex].isMarked(rowIdx); + for (int i = 0; i < presentTagColumnIndices.size(); i++) { + final Integer columnIndex = presentTagColumnIndices.get(i); + Object idSeg = getColumnValue(columnIndex, rowIdx); + boolean isNull = isNullValue(columnIndex, rowIdx); deviceIdSegments[i + 1] = !isNull && idSeg != null ? idSeg.toString() : null; } IDeviceID currentDeviceId = Factory.DEFAULT_FACTORY.create(deviceIdSegments); @@ -172,16 +172,74 @@ public IDeviceID getDeviceID(int rowIdx) { return deviceIDs[rowIdx]; } + private List getPresentTagColumnIndices() { + final List presentTagColumnIndices = new ArrayList<>(); + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG && isTagColumnPresent(i)) { + presentTagColumnIndices.add(i); + } + } + return presentTagColumnIndices; + } + + private boolean isTagColumnPresent(final int columnIndex) { + return measurements != null + && columnIndex >= 0 + && columnIndex < measurements.length + && measurements[columnIndex] != null + && columns != null + && columnIndex < columns.length + && columns[columnIndex] != null; + } + @Override public R accept(IPlanVisitor visitor, C context) { return ((PlanVisitor) visitor).visitRelationalInsertTablet(this, context); } + @Override + protected boolean shouldSerializeMeasurement(final int index) { + return super.shouldSerializeMeasurement(index) && hasColumnCategory(index); + } + + @Override + protected boolean shouldSerializeMeasurementToWAL(final int index) { + return super.shouldSerializeMeasurementToWAL(index) && hasColumnCategory(index); + } + + private boolean hasColumnCategory(final int index) { + return columnCategories != null + && index >= 0 + && index < columnCategories.length + && columnCategories[index] != null; + } + + private Object getColumnValue(final int columnIndex, final int rowIndex) { + if (columns == null + || columnIndex < 0 + || columnIndex >= columns.length + || columns[columnIndex] == null + || !(columns[columnIndex] instanceof Object[])) { + return null; + } + final Object[] values = (Object[]) columns[columnIndex]; + return rowIndex >= 0 && rowIndex < values.length ? values[rowIndex] : null; + } + + private boolean isNullValue(final int columnIndex, final int rowIndex) { + return bitMaps != null + && columnIndex >= 0 + && columnIndex < bitMaps.length + && bitMaps[columnIndex] != null + && bitMaps[columnIndex].isMarked(rowIndex); + } + @Override protected InsertTabletNode getEmptySplit(int count) { long[] subTimes = new long[count]; - Object[] values = initTabletValues(dataTypes.length, count, dataTypes); - BitMap[] newBitMaps = initBitmapsForSplit(dataTypes.length, count); + final int columnSize = getColumnArrayLength(); + Object[] values = initTabletValuesForSplit(columnSize, count, dataTypes); + BitMap[] newBitMaps = initBitmapsForSplit(columnSize, count); RelationalInsertTabletNode split = new RelationalInsertTabletNode( getPlanNodeId(), @@ -249,8 +307,8 @@ public static RelationalInsertTabletNode deserialize(ByteBuffer byteBuffer) { @Override protected void serializeAttributes(ByteBuffer byteBuffer) { super.serializeAttributes(byteBuffer); - for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { columnCategories[i].serialize(byteBuffer); } } @@ -259,8 +317,8 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { @Override protected void serializeAttributes(DataOutputStream stream) throws IOException { super.serializeAttributes(stream); - for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurement(i)) { columnCategories[i].serialize(stream); } } @@ -284,8 +342,8 @@ void subSerialize(IWALByteBufferView buffer, List rangeList) { @Override void subSerialize(IWALByteBufferView buffer, List rangeList, long encodedSearchIndex) { super.subSerialize(buffer, rangeList, encodedSearchIndex); - for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (shouldSerializeMeasurementToWAL(i)) { buffer.put(columnCategories[i].getCategory()); } } @@ -329,12 +387,12 @@ public static RelationalInsertTabletNode deserializeFromWAL(ByteBuffer buffer) { @Override int subSerializeSize(int start, int end) { - return super.subSerializeSize(start, end) + getValidMeasurementNumber() * Byte.BYTES; + return super.subSerializeSize(start, end) + getValidMeasurementNumberForWAL() * Byte.BYTES; } @Override int subSerializeSize(List rangeList) { - return super.subSerializeSize(rangeList) + getValidMeasurementNumber() * Byte.BYTES; + return super.subSerializeSize(rangeList) + getValidMeasurementNumberForWAL() * Byte.BYTES; } @Override @@ -450,11 +508,12 @@ private List generateOneSplitList( System.arraycopy(times, start, subNode.times, destLoc, length); for (int i = 0; i < subNode.columns.length; i++) { - if (dataTypes[i] != null) { + if (hasColumnForSplit(i)) { System.arraycopy(columns[i], start, subNode.columns[i], destLoc, length); } if (subNode.bitMaps != null && subNode.bitMaps[i] != null + && this.bitMaps != null && i < this.bitMaps.length && this.bitMaps[i] != null) { BitMap.copyOfRange(this.bitMaps[i], start, subNode.bitMaps[i], destLoc, length); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceNormalSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceNormalSchema.java index da37202c47043..a030946034a08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceNormalSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceNormalSchema.java @@ -67,7 +67,9 @@ public int update(final String[] measurements, final IMeasurementSchema[] schema for (int i = 0; i < length; ++i) { // Skip this to avoid instance creation/gc for writing performance - if (measurements[i] == null || measurementMap.containsKey(measurements[i])) { + if (measurements[i] == null + || schemas[i] == null + || measurementMap.containsKey(measurements[i])) { continue; } diff += putEntry(measurements[i], schemas[i], null); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java index 93d75aeff2d8c..c2588606cea69 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java @@ -169,17 +169,18 @@ public ClusterSchemaTree getMatchedNormalSchema(final PartialPath fullPath) { public List computeWithoutTemplate(final ISchemaComputation schemaComputation) { final List indexOfMissingMeasurements = new ArrayList<>(); final String[] measurements = schemaComputation.getMeasurements(); + if (measurements == null) { + return indexOfMissingMeasurements; + } final IDeviceSchema schema = tableDeviceSchemaCache.getDeviceSchema(schemaComputation.getDevicePath().getNodes()); if (!(schema instanceof TreeDeviceNormalSchema)) { - return IntStream.range(0, schemaComputation.getMeasurements().length) - .boxed() - .collect(Collectors.toList()); + return IntStream.range(0, measurements.length).boxed().collect(Collectors.toList()); } final TreeDeviceNormalSchema treeSchema = (TreeDeviceNormalSchema) schema; - for (int i = 0; i < schemaComputation.getMeasurements().length; i++) { + for (int i = 0; i < measurements.length; i++) { final SchemaCacheEntry value = treeSchema.getSchemaCacheEntry(measurements[i]); if (value == null) { indexOfMissingMeasurements.add(i); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java index b5440ba1d8c4c..6acb68adad5aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java @@ -80,10 +80,13 @@ public List getAttributeColumnNameList() { public List getAttributeValueList() { final InsertRowStatement insertRowStatement = getInnerTreeStatement(); final List attrColumnIndices = insertRowStatement.getAttrColumnIndices(); + final Object[] values = insertRowStatement.getValues(); Object[] attrValues = new Object[attrColumnIndices.size()]; for (int j = 0; j < attrColumnIndices.size(); j++) { final int columnIndex = attrColumnIndices.get(j); - attrValues[j] = insertRowStatement.getValues()[columnIndex]; + if (values != null && columnIndex >= 0 && columnIndex < values.length) { + attrValues[j] = values[columnIndex]; + } } return Collections.singletonList(attrValues); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java index 0e53064e95282..4eb49df718360 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java @@ -140,9 +140,17 @@ public List getAttributeColumnNameList() { @Override public List getAttributeValueList() { List attributeValueList = new ArrayList<>(); - for (int i = 0; i < insertRowStatement.getColumnCategories().length; i++) { - if (insertRowStatement.getColumnCategories()[i] == TsTableColumnCategory.ATTRIBUTE) { - attributeValueList.add(insertRowStatement.getValues()[i]); + final TsTableColumnCategory[] columnCategories = insertRowStatement.getColumnCategories(); + final String[] measurements = insertRowStatement.getMeasurements(); + final Object[] values = insertRowStatement.getValues(); + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.ATTRIBUTE + && measurements != null + && i < measurements.length + && measurements[i] != null + && values != null + && i < values.length) { + attributeValueList.add(values[i]); } } return Collections.singletonList(attributeValueList.toArray()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java index 5d25b01ecf348..639e35187a955 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java @@ -103,9 +103,16 @@ public List getAttributeValueList() { deviceID2AttributeValues.get(insertTabletStatement.getTableDeviceID(rowIndex)); for (int attrColNum = 0; attrColNum < attrColumnIndices.size(); attrColNum++) { final int columnIndex = attrColumnIndices.get(attrColNum); - if (!insertTabletStatement.isNull(rowIndex, columnIndex)) { - attrValues[attrColNum] = - ((Object[]) insertTabletStatement.getColumns()[columnIndex])[rowIndex]; + final Object[] columns = insertTabletStatement.getColumns(); + if (!insertTabletStatement.isNull(rowIndex, columnIndex) + && columns != null + && columnIndex >= 0 + && columnIndex < columns.length + && columns[columnIndex] instanceof Object[]) { + final Object[] columnValues = (Object[]) columns[columnIndex]; + if (rowIndex < columnValues.length) { + attrValues[attrColNum] = columnValues[rowIndex]; + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java index f3151fbfe10fe..a476018a745ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java @@ -87,10 +87,11 @@ public InsertNodeMeasurementInfo getInsertNodeMeasurementInfo() { protected TableSchema toTableSchema(InsertBaseStatement insertBaseStatement) { String tableName = insertBaseStatement.getDevicePath().getFullPath(); + final String[] measurements = insertBaseStatement.getMeasurements(); List columnSchemas = - new ArrayList<>(insertBaseStatement.getMeasurements().length); - for (int i = 0; i < insertBaseStatement.getMeasurements().length; i++) { - if (insertBaseStatement.getMeasurements()[i] != null) { + new ArrayList<>(measurements == null ? 0 : measurements.length); + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (insertBaseStatement.isColumnPresent(i)) { TSDataType dataType = insertBaseStatement.getDataType(i); if (dataType == null) { dataType = @@ -99,7 +100,7 @@ protected TableSchema toTableSchema(InsertBaseStatement insertBaseStatement) { } columnSchemas.add( new ColumnSchema( - insertBaseStatement.getMeasurements()[i], + measurements[i], dataType != null ? TypeFactory.getType(dataType) : null, false, insertBaseStatement.getColumnCategory(i))); @@ -113,12 +114,20 @@ protected TableSchema toTableSchema(InsertBaseStatement insertBaseStatement) { protected InsertNodeMeasurementInfo toInsertNodeMeasurementInfo( InsertBaseStatement insertBaseStatement) { String tableName = insertBaseStatement.getDevicePath().getFullPath().toLowerCase(); + final String[] measurements = insertBaseStatement.getMeasurements(); + final String[] validMeasurements = + measurements == null ? null : new String[measurements.length]; + for (int i = 0; measurements != null && i < measurements.length; i++) { + if (insertBaseStatement.isColumnPresent(i)) { + validMeasurements[i] = measurements[i]; + } + } // Use lazy initialization with measurements and dataTypes return new InsertNodeMeasurementInfo( tableName, insertBaseStatement.getColumnCategories(), - insertBaseStatement.getMeasurements(), + validMeasurements, insertBaseStatement.getDataTypes(), index -> TypeInferenceUtils.getPredictedDataType( @@ -243,6 +252,9 @@ void adjustTagColumns( if (baseStatement == null || existingTagColumnIndexMap.isEmpty()) { return; } + if (baseStatement.getMeasurements() == null) { + return; + } // Phase 1: Analyze and determine action in one pass final int oldLength = baseStatement.getMeasurements().length; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java index ddefab4d2d416..cdcbc7b836cbe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java @@ -147,8 +147,11 @@ public void setMeasurementSchemas(MeasurementSchema[] measurementSchemas) { } public void setMeasurementSchema(MeasurementSchema measurementSchema, int i) { - if (measurementSchemas == null) { - measurementSchemas = new MeasurementSchema[measurements.length]; + if (measurementSchemas == null || i >= measurementSchemas.length) { + measurementSchemas = + measurementSchemas == null + ? new MeasurementSchema[getRequiredColumnArrayLength(i)] + : Arrays.copyOf(measurementSchemas, getRequiredColumnArrayLength(i)); } measurementSchemas[i] = measurementSchema; } @@ -166,7 +169,7 @@ public TSDataType[] getDataTypes() { } public TSDataType getDataType(int i) { - if (dataTypes == null) { + if (dataTypes == null || i < 0 || i >= dataTypes.length) { return null; } return dataTypes[i]; @@ -177,8 +180,11 @@ public void setDataTypes(TSDataType[] dataTypes) { } public void setDataType(TSDataType dataType, int i) { - if (dataTypes == null) { - dataTypes = new TSDataType[measurements.length]; + if (dataTypes == null || i >= dataTypes.length) { + dataTypes = + dataTypes == null + ? new TSDataType[getRequiredColumnArrayLength(i)] + : Arrays.copyOf(dataTypes, getRequiredColumnArrayLength(i)); } this.dataTypes[i] = dataType; } @@ -208,35 +214,46 @@ public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryPro /** Check whether data types are matched with measurement schemas */ public void selfCheckDataTypes(int index) throws DataTypeMismatchException, PathNotExistException { + final MeasurementSchema measurementSchema = + measurementSchemas != null && index >= 0 && index < measurementSchemas.length + ? measurementSchemas[index] + : null; + final TSDataType dataType = getDataType(index); + final String measurement = + measurements != null && index >= 0 && index < measurements.length + ? measurements[index] + : null; + final String fullPath = + measurement == null + ? devicePath.getFullPath() + : devicePath.concatNode(measurement).getFullPath(); if (IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) { // if enable partial insert, mark failed measurements with exception - if (measurementSchemas[index] == null) { - markFailedMeasurement( - index, - new PathNotExistException(devicePath.concatNode(measurements[index]).getFullPath())); - } else if ((dataTypes[index] != measurementSchemas[index].getType() - && !checkAndCastDataType(index, measurementSchemas[index].getType()))) { + if (measurementSchema == null) { + markFailedMeasurement(index, new PathNotExistException(fullPath)); + } else if ((dataType != measurementSchema.getType() + && !checkAndCastDataType(index, measurementSchema.getType()))) { markFailedMeasurement( index, new DataTypeMismatchException( devicePath.getFullPath(), - measurements[index], - dataTypes[index], - measurementSchemas[index].getType(), + measurement, + dataType, + measurementSchema.getType(), getMinTime(), getFirstValueOfIndex(index))); } } else { // if not enable partial insert, throw the exception directly - if (measurementSchemas[index] == null) { - throw new PathNotExistException(devicePath.concatNode(measurements[index]).getFullPath()); - } else if ((dataTypes[index] != measurementSchemas[index].getType() - && !checkAndCastDataType(index, measurementSchemas[index].getType()))) { + if (measurementSchema == null) { + throw new PathNotExistException(fullPath); + } else if ((dataType != measurementSchema.getType() + && !checkAndCastDataType(index, measurementSchema.getType()))) { throw new DataTypeMismatchException( devicePath.getFullPath(), - measurements[index], - dataTypes[index], - measurementSchemas[index].getType(), + measurement, + dataType, + measurementSchema.getType(), getMinTime(), getFirstValueOfIndex(index)); } @@ -300,20 +317,33 @@ public void removeAllFailedMeasurementMarks() { } public boolean hasValidMeasurements() { - for (Object o : measurements) { - if (o != null) { + if (measurements == null) { + return false; + } + for (int i = 0; i < measurements.length; i++) { + if (isColumnPresent(i) + && (columnCategories == null + || i < columnCategories.length + && columnCategories[i] == TsTableColumnCategory.FIELD)) { return true; } } return false; } + public boolean isColumnPresent(final int index) { + return measurements != null + && index >= 0 + && index < measurements.length + && measurements[index] != null; + } + public TsTableColumnCategory[] getColumnCategories() { return columnCategories; } public TsTableColumnCategory getColumnCategory(int i) { - if (columnCategories == null) { + if (columnCategories == null || i < 0 || i >= columnCategories.length) { return null; } return columnCategories[i]; @@ -321,21 +351,27 @@ public TsTableColumnCategory getColumnCategory(int i) { public void setColumnCategories(TsTableColumnCategory[] columnCategories) { this.columnCategories = columnCategories; + this.tagColumnIndices = null; + this.attrColumnIndices = null; } public void setColumnCategory(TsTableColumnCategory columnCategory, int i) { - if (columnCategories == null) { - columnCategories = new TsTableColumnCategory[measurements.length]; + if (columnCategories == null || i >= columnCategories.length) { + columnCategories = + columnCategories == null + ? new TsTableColumnCategory[getRequiredColumnArrayLength(i)] + : Arrays.copyOf(columnCategories, getRequiredColumnArrayLength(i)); } this.columnCategories[i] = columnCategory; this.tagColumnIndices = null; + this.attrColumnIndices = null; } public List getTagColumnIndices() { if (tagColumnIndices == null && columnCategories != null) { tagColumnIndices = new ArrayList<>(); for (int i = 0; i < columnCategories.length; i++) { - if (columnCategories[i].equals(TsTableColumnCategory.TAG)) { + if (isColumnPresent(i) && columnCategories[i] == TsTableColumnCategory.TAG) { tagColumnIndices.add(i); } } @@ -347,7 +383,7 @@ public List getAttrColumnIndices() { if (attrColumnIndices == null && columnCategories != null) { attrColumnIndices = new ArrayList<>(); for (int i = 0; i < columnCategories.length; i++) { - if (columnCategories[i].equals(TsTableColumnCategory.ATTRIBUTE)) { + if (isColumnPresent(i) && columnCategories[i] == TsTableColumnCategory.ATTRIBUTE) { attrColumnIndices.add(i); } } @@ -434,7 +470,7 @@ public void removeAttributeColumns() { List columnsToKeep = new ArrayList<>(); for (int i = 0; i < columnCategories.length; i++) { - if (!columnCategories[i].equals(TsTableColumnCategory.ATTRIBUTE)) { + if (columnCategories[i] != TsTableColumnCategory.ATTRIBUTE) { columnsToKeep.add(i); } } @@ -447,18 +483,30 @@ public void removeAttributeColumns() { if (failedMeasurementIndex2Info != null) { failedMeasurementIndex2Info = failedMeasurementIndex2Info.entrySet().stream() + .filter(e -> columnsToKeep.contains(e.getKey())) .collect(Collectors.toMap(e -> columnsToKeep.indexOf(e.getKey()), Entry::getValue)); } if (measurementSchemas != null) { measurementSchemas = - columnsToKeep.stream().map(i -> measurementSchemas[i]).toArray(MeasurementSchema[]::new); + columnsToKeep.stream() + .filter(i -> i < measurementSchemas.length) + .map(i -> measurementSchemas[i]) + .toArray(MeasurementSchema[]::new); } if (measurements != null) { - measurements = columnsToKeep.stream().map(i -> measurements[i]).toArray(String[]::new); + measurements = + columnsToKeep.stream() + .filter(i -> i < measurements.length) + .map(i -> measurements[i]) + .toArray(String[]::new); } if (dataTypes != null) { - dataTypes = columnsToKeep.stream().map(i -> dataTypes[i]).toArray(TSDataType[]::new); + dataTypes = + columnsToKeep.stream() + .filter(i -> i < dataTypes.length) + .map(i -> dataTypes[i]) + .toArray(TSDataType[]::new); } if (columnCategories != null) { columnCategories = @@ -542,6 +590,9 @@ protected Map>> getMapFromDeviceToMeasur Map>> mapFromDeviceToMeasurementAndIndex = new HashMap<>(); for (int i = 0; i < this.measurements.length; i++) { + if (!isColumnPresent(i)) { + continue; + } PartialPath targetDevicePath; String measurementName; if (isLogicalView[i]) { @@ -597,57 +648,53 @@ protected static void validateMapFromDeviceToMeasurement( } public void insertColumn(final int pos, final ColumnSchema columnSchema) { - if (pos < 0 || pos > measurements.length) { + final int oldLength = measurements.length; + if (pos < 0 || pos > oldLength) { throw new ArrayIndexOutOfBoundsException(pos); } + final int newLength = oldLength + 1; if (measurementSchemas != null) { - final MeasurementSchema[] tmp = new MeasurementSchema[measurementSchemas.length + 1]; - System.arraycopy(measurementSchemas, 0, tmp, 0, pos); + final MeasurementSchema[] tmp = new MeasurementSchema[newLength]; + copyWithInsertedSlot(measurementSchemas, tmp, pos); tmp[pos] = new MeasurementSchema( columnSchema.getName(), InternalTypeManager.getTSDataType(columnSchema.getType())); - System.arraycopy(measurementSchemas, pos, tmp, pos + 1, measurementSchemas.length - pos); measurementSchemas = tmp; } - String[] tmpMeasurements = new String[measurements.length + 1]; - System.arraycopy(measurements, 0, tmpMeasurements, 0, pos); + String[] tmpMeasurements = new String[newLength]; + copyWithInsertedSlot(measurements, tmpMeasurements, pos); tmpMeasurements[pos] = columnSchema.getName(); - System.arraycopy(measurements, pos, tmpMeasurements, pos + 1, measurements.length - pos); measurements = tmpMeasurements; if (dataTypes == null) { // sql insertion - dataTypes = new TSDataType[measurements.length + 1]; - dataTypes[pos] = InternalTypeManager.getTSDataType(columnSchema.getType()); + dataTypes = new TSDataType[newLength]; } else { - final TSDataType[] tmpTypes = new TSDataType[dataTypes.length + 1]; - System.arraycopy(dataTypes, 0, tmpTypes, 0, pos); - tmpTypes[pos] = InternalTypeManager.getTSDataType(columnSchema.getType()); - System.arraycopy(dataTypes, pos, tmpTypes, pos + 1, dataTypes.length - pos); + final TSDataType[] tmpTypes = new TSDataType[newLength]; + copyWithInsertedSlot(dataTypes, tmpTypes, pos); dataTypes = tmpTypes; } + dataTypes[pos] = InternalTypeManager.getTSDataType(columnSchema.getType()); if (columnCategories == null) { - columnCategories = new TsTableColumnCategory[measurements.length + 1]; - columnCategories[pos] = columnSchema.getColumnCategory(); + columnCategories = new TsTableColumnCategory[newLength]; } else { - final TsTableColumnCategory[] tmpCategories = - new TsTableColumnCategory[columnCategories.length + 1]; - System.arraycopy(columnCategories, 0, tmpCategories, 0, pos); - tmpCategories[pos] = columnSchema.getColumnCategory(); - System.arraycopy( - columnCategories, pos, tmpCategories, pos + 1, columnCategories.length - pos); + final TsTableColumnCategory[] tmpCategories = new TsTableColumnCategory[newLength]; + copyWithInsertedSlot(columnCategories, tmpCategories, pos); columnCategories = tmpCategories; - tagColumnIndices = null; } + columnCategories[pos] = columnSchema.getColumnCategory(); + tagColumnIndices = null; + attrColumnIndices = null; } public void swapColumn(int src, int target) { if (src < 0 || src >= measurements.length || target < 0 || target >= measurements.length) { throw new ArrayIndexOutOfBoundsException(src + "/" + target); } + ensureBaseArraysLength(measurements.length); if (measurementSchemas != null) { CommonUtils.swapArray(measurementSchemas, src, target); } @@ -666,6 +713,7 @@ public void swapColumn(int src, int target) { CommonUtils.swapArray(inputLocations, src, target); } tagColumnIndices = null; + attrColumnIndices = null; } /** @@ -710,19 +758,19 @@ public void rebuildArraysAfterExpansion( // typeConvertors and inputLocations remain null for missing columns } else { // Copy from old array - if (oldMeasurementSchemas != null) { + if (oldMeasurementSchemas != null && oldIdx < oldMeasurementSchemas.length) { newMeasurementSchemas[newIdx] = oldMeasurementSchemas[oldIdx]; } - if (oldDataTypes != null) { + if (oldDataTypes != null && oldIdx < oldDataTypes.length) { newDataTypes[newIdx] = oldDataTypes[oldIdx]; } - if (oldColumnCategories != null) { + if (oldColumnCategories != null && oldIdx < oldColumnCategories.length) { newColumnCategories[newIdx] = oldColumnCategories[oldIdx]; } - if (oldTypeConvertors != null) { + if (oldTypeConvertors != null && oldIdx < oldTypeConvertors.length) { newTypeConvertors[newIdx] = oldTypeConvertors[oldIdx]; } - if (oldInputLocations != null) { + if (oldInputLocations != null && oldIdx < oldInputLocations.length) { newInputLocations[newIdx] = oldInputLocations[oldIdx]; } } @@ -740,6 +788,45 @@ public void rebuildArraysAfterExpansion( attrColumnIndices = null; } + private int getRequiredColumnArrayLength(final int index) { + return Math.max(measurements == null ? 0 : measurements.length, index + 1); + } + + private void ensureBaseArraysLength(final int length) { + if (measurementSchemas != null && measurementSchemas.length < length) { + measurementSchemas = Arrays.copyOf(measurementSchemas, length); + } + if (dataTypes != null && dataTypes.length < length) { + dataTypes = Arrays.copyOf(dataTypes, length); + } + if (columnCategories != null && columnCategories.length < length) { + columnCategories = Arrays.copyOf(columnCategories, length); + } + if (typeConvertors != null && typeConvertors.length < length) { + typeConvertors = Arrays.copyOf(typeConvertors, length); + } + if (inputLocations != null && inputLocations.length < length) { + inputLocations = Arrays.copyOf(inputLocations, length); + } + } + + protected static void copyWithInsertedSlot( + final T[] source, final T[] target, final int pos) { + if (source == null) { + return; + } + final int prefixLength = Math.min(pos, source.length); + if (prefixLength > 0) { + System.arraycopy(source, 0, target, 0, prefixLength); + } + if (pos < source.length) { + final int suffixLength = Math.min(source.length - pos, target.length - pos - 1); + if (suffixLength > 0) { + System.arraycopy(source, pos, target, pos + 1, suffixLength); + } + } + } + public boolean isWriteToTable() { return writeToTable; } @@ -796,8 +883,9 @@ public void toLowerCaseForDevicePath() { @TableModel public List getAttributeColumnNameList() { final List attributeColumnNameList = new ArrayList<>(); - for (int i = 0; i < getColumnCategories().length; i++) { - if (getColumnCategories()[i] == TsTableColumnCategory.ATTRIBUTE) { + final TsTableColumnCategory[] columnCategories = getColumnCategories(); + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (isColumnPresent(i) && columnCategories[i] == TsTableColumnCategory.ATTRIBUTE) { attributeColumnNameList.add(getMeasurements()[i]); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index d0ec0b19ee081..fc12391eba1b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -99,7 +100,13 @@ public InsertRowStatement() { @Override public List getPaths() { List ret = new ArrayList<>(); + if (measurements == null) { + return ret; + } for (String m : measurements) { + if (m == null) { + continue; + } PartialPath fullPath = devicePath.concatAsMeasurementPath(m); ret.add(fullPath); } @@ -120,6 +127,13 @@ public Object[] getValues() { public void setValues(Object[] values) { this.values = values; + deviceID = null; + } + + @Override + public void setColumnCategories(TsTableColumnCategory[] columnCategories) { + super.setColumnCategories(columnCategories); + deviceID = null; } public boolean isNeedInferType() { @@ -193,11 +207,27 @@ public long getMinTime() { @Override public Object getFirstValueOfIndex(int index) { + if (values == null || index < 0 || index >= values.length) { + return null; + } return values[index]; } + @Override + public boolean isColumnPresent(final int index) { + return super.isColumnPresent(index) && values != null && index < values.length; + } + @Override protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { + if (dataTypes == null + || values == null + || columnIndex < 0 + || columnIndex >= dataTypes.length + || columnIndex >= values.length + || dataTypes[columnIndex] == null) { + return false; + } if (dataType.isCompatible(dataTypes[columnIndex])) { values[columnIndex] = dataType.castFromSingleValue(dataTypes[columnIndex], values[columnIndex]); @@ -213,8 +243,18 @@ protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { */ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning public void transferType(ZoneId zoneId) throws QueryProcessException { + if (measurementSchemas == null) { + return; + } for (int i = 0; i < measurementSchemas.length; i++) { + if (!isColumnPresent(i) + || values == null + || i >= values.length + || dataTypes == null + || i >= dataTypes.length) { + continue; + } // null when time series doesn't exist if (measurementSchemas[i] == null) { if (!IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) { @@ -266,7 +306,10 @@ public void transferType(ZoneId zoneId) throws QueryProcessException { @Override public void markFailedMeasurement(int index, Exception cause) { - if (measurements[index] == null) { + if (measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null) { return; } @@ -276,12 +319,19 @@ public void markFailedMeasurement(int index, Exception cause) { InsertBaseStatement.FailedMeasurementInfo failedMeasurementInfo = new InsertBaseStatement.FailedMeasurementInfo( - measurements[index], dataTypes[index], values[index], cause); + measurements[index], + dataTypes != null && index < dataTypes.length ? dataTypes[index] : null, + values != null && index < values.length ? values[index] : null, + cause); failedMeasurementIndex2Info.putIfAbsent(index, failedMeasurementInfo); measurements[index] = null; - dataTypes[index] = null; - values[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + if (values != null && index < values.length) { + values[index] = null; + } } @Override @@ -291,15 +341,15 @@ public void removeAllFailedMeasurementMarks() { } failedMeasurementIndex2Info.forEach( (index, info) -> { - if (measurements != null) { + if (measurements != null && index < measurements.length) { measurements[index] = info.getMeasurement(); } - if (dataTypes != null) { + if (dataTypes != null && index < dataTypes.length) { dataTypes[index] = info.getDataType(); } - if (values != null) { + if (values != null && index < values.length) { values[index] = info.getValue(); } }); @@ -347,12 +397,19 @@ public List getSplitList() { TSDataType[] dataTypes = new TSDataType[pairList.size()]; for (int i = 0; i < pairList.size(); i++) { int realIndex = pairList.get(i).right; - copiedValues[i] = this.values[realIndex]; + copiedValues[i] = + this.values != null && realIndex < this.values.length ? this.values[realIndex] : null; measurements[i] = Objects.nonNull(this.measurements[realIndex]) ? pairList.get(i).left : null; - measurementSchemas[i] = this.measurementSchemas[realIndex]; - dataTypes[i] = this.dataTypes[realIndex]; - if (this.measurementIsAligned != null) { + measurementSchemas[i] = + this.measurementSchemas != null && realIndex < this.measurementSchemas.length + ? this.measurementSchemas[realIndex] + : null; + dataTypes[i] = + this.dataTypes != null && realIndex < this.dataTypes.length + ? this.dataTypes[realIndex] + : null; + if (this.measurementIsAligned != null && realIndex < this.measurementIsAligned.length) { statement.setAligned(this.measurementIsAligned[realIndex]); } } @@ -399,14 +456,22 @@ public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryPro @Override public TSDataType getDataType(int index) { - if (isNeedInferType && (dataTypes == null || dataTypes[index] == null)) { + if (index < 0 || measurements == null || index >= measurements.length) { + return null; + } + if (isNeedInferType + && (dataTypes == null || index >= dataTypes.length || dataTypes[index] == null)) { if (dataTypes == null) { dataTypes = new TSDataType[measurements.length]; + } else if (index >= dataTypes.length) { + dataTypes = Arrays.copyOf(dataTypes, measurements.length); } - dataTypes[index] = TypeInferenceUtils.getPredictedDataType(values[index], true); + dataTypes[index] = + TypeInferenceUtils.getPredictedDataType( + values != null && index < values.length ? values[index] : null, true); return dataTypes[index]; } else { - return dataTypes != null ? dataTypes[index] : null; + return dataTypes != null && index < dataTypes.length ? dataTypes[index] : null; } } @@ -429,6 +494,8 @@ public void validateDeviceSchema(boolean isAligned) { public void validateMeasurementSchema(int index, IMeasurementSchemaInfo measurementSchemaInfo) { if (measurementSchemas == null) { measurementSchemas = new MeasurementSchema[measurements.length]; + } else if (index >= measurementSchemas.length) { + measurementSchemas = Arrays.copyOf(measurementSchemas, measurements.length); } if (measurementSchemaInfo == null) { measurementSchemas[index] = null; @@ -463,6 +530,9 @@ public void validateMeasurementSchema( if (this.measurementIsAligned == null) { this.measurementIsAligned = new boolean[this.measurements.length]; Arrays.fill(this.measurementIsAligned, this.isAligned); + } else if (index >= this.measurementIsAligned.length) { + this.measurementIsAligned = + Arrays.copyOf(this.measurementIsAligned, this.measurements.length); } this.measurementIsAligned[index] = isAligned; } @@ -506,8 +576,11 @@ public IDeviceID getTableDeviceID() { deviceIdSegments[0] = this.getTableName(); for (int i = 0; i < getTagColumnIndices().size(); i++) { final Integer columnIndex = getTagColumnIndices().get(i); - deviceIdSegments[i + 1] = - values[columnIndex] != null ? values[columnIndex].toString() : null; + final Object idSegment = + values != null && columnIndex >= 0 && columnIndex < values.length + ? values[columnIndex] + : null; + deviceIdSegments[i + 1] = idSegment != null ? idSegment.toString() : null; } deviceID = Factory.DEFAULT_FACTORY.create(deviceIdSegments); } @@ -530,17 +603,23 @@ public Statement toRelationalStatement(MPPQueryContext context) { @Override public void insertColumn(int pos, ColumnSchema columnSchema) { super.insertColumn(pos, columnSchema); - Object[] tmpValues = new Object[values.length + 1]; - System.arraycopy(values, 0, tmpValues, 0, pos); - System.arraycopy(values, pos, tmpValues, pos + 1, values.length - pos); + Object[] tmpValues = new Object[measurements.length]; + copyWithInsertedSlot(values, tmpValues, pos); values = tmpValues; + deviceID = null; } @TableModel @Override public void swapColumn(int src, int target) { super.swapColumn(src, target); + if (values == null) { + values = new Object[measurements.length]; + } else if (values.length < measurements.length) { + values = Arrays.copyOf(values, measurements.length); + } CommonUtils.swapArray(values, src, target); + deviceID = null; } @TableModel @@ -574,15 +653,19 @@ public void rebuildArraysAfterExpansion( } } else { // Copy from old array - values[newIdx] = oldValues[oldIdx]; + if (oldValues != null && oldIdx < oldValues.length) { + values[newIdx] = oldValues[oldIdx]; + } if (newMeasurementIsAligned != null && oldMeasurementIsAligned != null) { - newMeasurementIsAligned[newIdx] = oldMeasurementIsAligned[oldIdx]; + newMeasurementIsAligned[newIdx] = + oldIdx < oldMeasurementIsAligned.length && oldMeasurementIsAligned[oldIdx]; } } } // Replace old array with new array measurementIsAligned = newMeasurementIsAligned; + deviceID = null; } @Override @@ -596,8 +679,9 @@ protected long calculateBytesUsed() { @Override protected void subRemoveAttributeColumns(List columnsToKeep) { if (values != null) { - values = columnsToKeep.stream().map(i -> values[i]).toArray(); + values = columnsToKeep.stream().filter(i -> i < values.length).map(i -> values[i]).toArray(); } + deviceID = null; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java index e4052e505607c..bfae0797a4aa5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowsOfOneDeviceStatement.java @@ -78,7 +78,13 @@ public void setInsertRowStatementList(List insertRowStatemen List measurementList = new ArrayList<>(); for (InsertRowStatement insertRowStatement : insertRowStatementList) { String[] measurements = insertRowStatement.getMeasurements(); + if (measurements == null) { + continue; + } for (String measurement : measurements) { + if (measurement == null) { + continue; + } if (!measurementSet.contains(measurement)) { measurementList.add(measurement); measurementSet.add(measurement); @@ -105,7 +111,13 @@ public R accept(StatementVisitor visitor, C context) { @Override public List getPaths() { List ret = new ArrayList<>(); + if (measurements == null) { + return ret; + } for (String m : measurements) { + if (m == null) { + continue; + } PartialPath fullPath = devicePath.concatAsMeasurementPath(m); ret.add(fullPath); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index 8ce0d534a4980..28f1043e86d15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -220,6 +220,7 @@ public Object[] getColumns() { public void setColumns(Object[] columns) { this.columns = columns; + deviceIDs = null; } public BitMap[] getBitMaps() { @@ -228,6 +229,13 @@ public BitMap[] getBitMaps() { public void setBitMaps(BitMap[] bitMaps) { this.nullBitMaps = bitMaps; + deviceIDs = null; + } + + @Override + public void setColumnCategories(TsTableColumnCategory[] columnCategories) { + super.setColumnCategories(columnCategories); + deviceIDs = null; } public long[] getTimes() { @@ -275,7 +283,13 @@ public R accept(StatementVisitor visitor, C context) { @Override public List getPaths() { List ret = new ArrayList<>(); + if (measurements == null) { + return ret; + } for (String m : measurements) { + if (m == null) { + continue; + } PartialPath fullPath = devicePath.concatAsMeasurementPath(m); ret.add(fullPath); } @@ -294,6 +308,15 @@ public List getSchemaValidationList() { @Override protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { + if (dataTypes == null + || columns == null + || columnIndex < 0 + || columnIndex >= dataTypes.length + || columnIndex >= columns.length + || dataTypes[columnIndex] == null + || columns[columnIndex] == null) { + return false; + } if (dataType.isCompatible(dataTypes[columnIndex])) { columns[columnIndex] = dataType.castFromArray(dataTypes[columnIndex], columns[columnIndex]); dataTypes[columnIndex] = dataType; @@ -304,7 +327,10 @@ protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { @Override public void markFailedMeasurement(int index, Exception cause) { - if (measurements[index] == null) { + if (measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null) { return; } @@ -314,12 +340,19 @@ public void markFailedMeasurement(int index, Exception cause) { InsertBaseStatement.FailedMeasurementInfo failedMeasurementInfo = new InsertBaseStatement.FailedMeasurementInfo( - measurements[index], dataTypes[index], columns[index], cause); + measurements[index], + dataTypes != null && index < dataTypes.length ? dataTypes[index] : null, + columns != null && index < columns.length ? columns[index] : null, + cause); failedMeasurementIndex2Info.putIfAbsent(index, failedMeasurementInfo); measurements[index] = null; - dataTypes[index] = null; - columns[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + if (columns != null && index < columns.length) { + columns[index] = null; + } } @Override @@ -329,9 +362,15 @@ public void removeAllFailedMeasurementMarks() { } failedMeasurementIndex2Info.forEach( (index, info) -> { - measurements[index] = info.getMeasurement(); - dataTypes[index] = info.getDataType(); - columns[index] = info.getValue(); + if (measurements != null && index < measurements.length) { + measurements[index] = info.getMeasurement(); + } + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = info.getDataType(); + } + if (columns != null && index < columns.length) { + columns[index] = info.getValue(); + } }); failedMeasurementIndex2Info.clear(); } @@ -374,15 +413,24 @@ public List getSplitList() { TSDataType[] dataTypes = new TSDataType[pairList.size()]; for (int i = 0; i < pairList.size(); i++) { int realIndex = pairList.get(i).right; - copiedColumns[i] = this.columns[realIndex]; + copiedColumns[i] = + this.columns != null && realIndex < this.columns.length + ? this.columns[realIndex] + : null; measurements[i] = Objects.nonNull(this.measurements[realIndex]) ? pairList.get(i).left : null; - measurementSchemas[i] = this.measurementSchemas[realIndex]; - dataTypes[i] = this.dataTypes[realIndex]; - if (this.nullBitMaps != null) { + measurementSchemas[i] = + this.measurementSchemas != null && realIndex < this.measurementSchemas.length + ? this.measurementSchemas[realIndex] + : null; + dataTypes[i] = + this.dataTypes != null && realIndex < this.dataTypes.length + ? this.dataTypes[realIndex] + : null; + if (this.nullBitMaps != null && realIndex < this.nullBitMaps.length) { copiedBitMaps[i] = this.nullBitMaps[realIndex]; } - if (this.measurementIsAligned != null) { + if (this.measurementIsAligned != null && realIndex < this.measurementIsAligned.length) { statement.setAligned(this.measurementIsAligned[realIndex]); } } @@ -420,6 +468,15 @@ public long getMinTime() { @Override public Object getFirstValueOfIndex(int index) { + if (dataTypes == null + || columns == null + || index < 0 + || index >= dataTypes.length + || index >= columns.length + || dataTypes[index] == null + || columns[index] == null) { + return null; + } Object value; switch (dataTypes[index]) { case INT32: @@ -457,9 +514,17 @@ public Object getFirstValueOfIndex(int index) { return value; } + @Override + public boolean isColumnPresent(final int index) { + return super.isColumnPresent(index) + && columns != null + && index < columns.length + && columns[index] != null; + } + @Override public TSDataType getDataType(int index) { - return dataTypes != null ? dataTypes[index] : null; + return dataTypes != null && index >= 0 && index < dataTypes.length ? dataTypes[index] : null; } @Override @@ -481,6 +546,8 @@ public void validateDeviceSchema(boolean isAligned) { public void validateMeasurementSchema(int index, IMeasurementSchemaInfo measurementSchemaInfo) { if (measurementSchemas == null) { measurementSchemas = new MeasurementSchema[measurements.length]; + } else if (index >= measurementSchemas.length) { + measurementSchemas = Arrays.copyOf(measurementSchemas, measurements.length); } if (measurementSchemaInfo == null) { measurementSchemas[index] = null; @@ -512,6 +579,9 @@ public void validateMeasurementSchema( if (this.measurementIsAligned == null) { this.measurementIsAligned = new boolean[this.measurements.length]; Arrays.fill(this.measurementIsAligned, this.isAligned); + } else if (index >= this.measurementIsAligned.length) { + this.measurementIsAligned = + Arrays.copyOf(this.measurementIsAligned, this.measurements.length); } this.measurementIsAligned[index] = isAligned; } @@ -562,9 +632,9 @@ public IDeviceID getTableDeviceID(int rowIdx) { deviceIdSegments[0] = this.getTableName(); for (int i = 0; i < getTagColumnIndices().size(); i++) { final Integer columnIndex = getTagColumnIndices().get(i); - boolean isNull = isNull(rowIdx, i); - deviceIdSegments[i + 1] = - isNull ? null : ((Object[]) columns[columnIndex])[rowIdx].toString(); + final Object idSegment = + isNull(rowIdx, columnIndex) ? null : getColumnValue(rowIdx, columnIndex); + deviceIdSegments[i + 1] = idSegment != null ? idSegment.toString() : null; } deviceIDs[rowIdx] = Factory.DEFAULT_FACTORY.create(deviceIdSegments); } @@ -605,14 +675,13 @@ public void insertColumn(int pos, ColumnSchema columnSchema) { nullBitMaps = tmpBitmaps; } - Object[] tmpColumns = new Object[columns.length + 1]; - System.arraycopy(columns, 0, tmpColumns, 0, pos); + Object[] tmpColumns = new Object[measurements.length]; + copyWithInsertedSlot(columns, tmpColumns, pos); tmpColumns[pos] = CommonUtils.createValueColumnOfDataType( InternalTypeManager.getTSDataType(columnSchema.getType()), columnSchema.getColumnCategory(), rowCount); - System.arraycopy(columns, pos, tmpColumns, pos + 1, columns.length - pos); columns = tmpColumns; deviceIDs = null; @@ -622,8 +691,16 @@ public void insertColumn(int pos, ColumnSchema columnSchema) { public void swapColumn(int src, int target) { super.swapColumn(src, target); if (nullBitMaps != null) { + if (nullBitMaps.length < measurements.length) { + nullBitMaps = Arrays.copyOf(nullBitMaps, measurements.length); + } CommonUtils.swapArray(nullBitMaps, src, target); } + if (columns == null) { + columns = new Object[measurements.length]; + } else if (columns.length < measurements.length) { + columns = Arrays.copyOf(columns, measurements.length); + } CommonUtils.swapArray(columns, src, target); deviceIDs = null; } @@ -667,14 +744,15 @@ public void rebuildArraysAfterExpansion( } } else { // Copy from old array - if (oldNullBitMaps != null) { + if (oldNullBitMaps != null && oldIdx < oldNullBitMaps.length) { newNullBitMaps[newIdx] = oldNullBitMaps[oldIdx]; } - if (newColumns != null && oldColumns != null) { + if (newColumns != null && oldColumns != null && oldIdx < oldColumns.length) { newColumns[newIdx] = oldColumns[oldIdx]; } if (newMeasurementIsAligned != null && oldMeasurementIsAligned != null) { - newMeasurementIsAligned[newIdx] = oldMeasurementIsAligned[oldIdx]; + newMeasurementIsAligned[newIdx] = + oldIdx < oldMeasurementIsAligned.length && oldMeasurementIsAligned[oldIdx]; } } } @@ -701,20 +779,40 @@ protected long calculateBytesUsed() { } public boolean isNull(int row, int col) { - if (nullBitMaps == null || nullBitMaps[col] == null) { - return false; + return nullBitMaps != null + && row >= 0 + && col >= 0 + && col < nullBitMaps.length + && nullBitMaps[col] != null + && nullBitMaps[col].isMarked(row); + } + + private Object getColumnValue(final int rowIdx, final int columnIndex) { + if (columns == null + || columnIndex < 0 + || columnIndex >= columns.length + || columns[columnIndex] == null + || !(columns[columnIndex] instanceof Object[])) { + return null; } - return nullBitMaps[col].isMarked(row); + final Object[] values = (Object[]) columns[columnIndex]; + return rowIdx >= 0 && rowIdx < values.length ? values[rowIdx] : null; } @Override protected void subRemoveAttributeColumns(List columnsToKeep) { if (columns != null) { - columns = columnsToKeep.stream().map(i -> columns[i]).toArray(); + columns = + columnsToKeep.stream().filter(i -> i < columns.length).map(i -> columns[i]).toArray(); } if (nullBitMaps != null) { - nullBitMaps = columnsToKeep.stream().map(i -> nullBitMaps[i]).toArray(BitMap[]::new); + nullBitMaps = + columnsToKeep.stream() + .filter(i -> i < nullBitMaps.length) + .map(i -> nullBitMaps[i]) + .toArray(BitMap[]::new); } + deviceIDs = null; } /** @@ -743,10 +841,14 @@ public Tablet convertToTablet() throws MetadataException { final List schemas = new ArrayList<>(originalSchemaSize); final int[] validColumnIndices = new int[originalSchemaSize]; int validColumnCount = 0; - if (dataTypes != null) { + final Object[] statementColumns = this.getColumns(); + if (dataTypes != null && statementColumns != null) { final int dataTypeSize = Math.min(originalSchemaSize, dataTypes.length); for (int i = 0; i < dataTypeSize; i++) { - if (measurements[i] != null && dataTypes[i] != null) { + if (measurements[i] != null + && dataTypes[i] != null + && i < statementColumns.length + && statementColumns[i] != null) { final MeasurementSchema measurementSchema = measurementSchemas != null && i < measurementSchemas.length ? measurementSchemas[i] @@ -803,7 +905,6 @@ public Tablet convertToTablet() throws MetadataException { // Get values - convert Statement columns to Tablet format, only for valid columns // All arrays are copied to rowSize length - final Object[] statementColumns = this.getColumns(); final Object[] tabletValues = new Object[schemaSize]; if (statementColumns != null && statementColumns.length > 0) { for (int i = 0; i < schemaSize; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index bbafc1c4cd915..12113e030b07d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -4787,7 +4787,8 @@ public void insert(InsertRowsNode insertRowsNode) DateTimeUtils.convertLongToDate(insertRowNode.getTime()), DateTimeUtils.convertLongToDate( CommonDateTimeUtils.currentTime() - ttl)))); - insertRowNode.setFailedMeasurementNumber(insertRowNode.getMeasurements().length); + insertRowNode.setFailedMeasurementNumber( + insertRowNode.getMeasurements() == null ? 0 : insertRowNode.getMeasurements().length); insertRowNode.setMeasurements(null); continue; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 6ba5f6307cb82..eea69f98a3f08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -26,7 +26,6 @@ import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.WriteProcessException; @@ -176,14 +175,14 @@ private IWritableMemChunkGroup createMemChunkGroupIfNotExistAndGet( private IWritableMemChunkGroup createAlignedMemChunkGroupIfNotExistAndGet( IDeviceID deviceId, List schemaList) { + List filteredSchemaList = + schemaList.stream().filter(Objects::nonNull).collect(Collectors.toList()); IWritableMemChunkGroup memChunkGroup = memTableMap.computeIfAbsent( deviceId, k -> { - seriesNumber += schemaList.size(); - return new AlignedWritableMemChunkGroup( - schemaList.stream().filter(Objects::nonNull).collect(Collectors.toList()), - k.isTableModel()); + seriesNumber += filteredSchemaList.size(); + return new AlignedWritableMemChunkGroup(filteredSchemaList, k.isTableModel()); }); for (IMeasurementSchema schema : schemaList) { if (schema != null && !memChunkGroup.contains(schema.getMeasurementName())) { @@ -201,11 +200,14 @@ public int insert(InsertRowNode insertRowNode) { List schemaList = new ArrayList<>(); List dataTypes = new ArrayList<>(); + Object[] writableValues = new Object[measurements == null ? 0 : measurements.length]; int nullPointsNumber = 0; - for (int i = 0; i < insertRowNode.getMeasurements().length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + final boolean hasValue = values != null && i < values.length; + final Object value = hasValue ? values[i] : null; // Use measurements[i] to ignore failed partial insert - if (measurements[i] == null || values[i] == null) { - if (values[i] == null) { + if (!insertRowNode.isValidMeasurement(i, true) || !hasValue || value == null) { + if (hasValue && insertRowNode.isValidMeasurement(i, true) && value == null) { nullPointsNumber++; } schemaList.add(null); @@ -213,14 +215,15 @@ public int insert(InsertRowNode insertRowNode) { IMeasurementSchema schema = insertRowNode.getMeasurementSchemas()[i]; schemaList.add(schema); dataTypes.add(schema.getType()); + writableValues[i] = value; } } - memSize += MemUtils.getRowRecordSize(dataTypes, values); - write(insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), values); + memSize += + MemUtils.getRowRecordSize(dataTypes, writableValues, insertRowNode.getColumnCategories()); + write(insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), writableValues); int pointsInserted = - insertRowNode.getMeasurements().length - - insertRowNode.getFailedMeasurementNumber() + insertRowNode.getValidMeasurementNumber(true) - (IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric() ? 0 : nullPointsNumber); @@ -236,14 +239,14 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { Object[] values = insertRowNode.getValues(); List schemaList = new ArrayList<>(); List dataTypes = new ArrayList<>(); + Object[] writableValues = new Object[measurements == null ? 0 : measurements.length]; int nullPointsNumber = 0; - for (int i = 0; i < insertRowNode.getMeasurements().length; i++) { + for (int i = 0; measurements != null && i < measurements.length; i++) { + final boolean hasValue = values != null && i < values.length; + final Object value = hasValue ? values[i] : null; // Use measurements[i] to ignore failed partial insert - if (measurements[i] == null - || values[i] == null - || insertRowNode.getColumnCategories() != null - && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD) { - if (measurements[i] != null && values[i] == null) { + if (!insertRowNode.isValidMeasurement(i, true) || !hasValue || value == null) { + if (hasValue && insertRowNode.isValidMeasurement(i, true) && value == null) { // do not include failed measurement to avoid a negative pointsInserted nullPointsNumber++; } @@ -253,16 +256,18 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { IMeasurementSchema schema = insertRowNode.getMeasurementSchemas()[i]; schemaList.add(schema); dataTypes.add(schema.getType()); + writableValues[i] = value; } if (schemaList.isEmpty()) { return 0; } memSize += - MemUtils.getAlignedRowRecordSize(dataTypes, values, insertRowNode.getColumnCategories()); - writeAlignedRow(insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), values); + MemUtils.getAlignedRowRecordSize( + dataTypes, writableValues, insertRowNode.getColumnCategories()); + writeAlignedRow( + insertRowNode.getDeviceID(), schemaList, insertRowNode.getTime(), writableValues); int pointsInserted = - insertRowNode.getMeasurementColumnCnt() - - insertRowNode.getFailedMeasurementNumber() + insertRowNode.getValidMeasurementNumber(true) - (IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric() ? 0 : nullPointsNumber); @@ -274,12 +279,11 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { public int insertTablet(InsertTabletNode insertTabletNode, int start, int end) throws WriteProcessException { try { - int nullPointsNumber = computeTabletNullPointsNumber(insertTabletNode, start, end); + int nullPointsNumber = computeTabletNullPointsNumber(insertTabletNode, start, end, true); writeTabletNode(insertTabletNode, start, end); memSize += MemUtils.getTabletSize(insertTabletNode, start, end); int pointsInserted = - ((insertTabletNode.getDataTypes().length - insertTabletNode.getFailedMeasurementNumber()) - * (end - start)) + (insertTabletNode.getValidMeasurementNumber(true) * (end - start)) - (IoTDBDescriptor.getInstance() .getConfig() .isIncludeNullValueInWriteThroughputMetric() @@ -297,14 +301,12 @@ public int insertAlignedTablet( InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results) throws WriteProcessException { try { - int nullPointsNumber = computeTabletNullPointsNumber(insertTabletNode, start, end); + int nullPointsNumber = computeTabletNullPointsNumber(insertTabletNode, start, end, true); writeAlignedTablet(insertTabletNode, start, end, results); // TODO-Table: what is the relation between this and TsFileProcessor.checkMemCost memSize += MemUtils.getAlignedTabletSize(insertTabletNode, start, end, results); int pointsInserted = - ((insertTabletNode.getMeasurementColumnCnt() - - insertTabletNode.getFailedMeasurementNumber()) - * (end - start)) + (insertTabletNode.getValidMeasurementNumber(true) * (end - start)) - (IoTDBDescriptor.getInstance() .getConfig() .isIncludeNullValueInWriteThroughputMetric() @@ -318,17 +320,23 @@ public int insertAlignedTablet( } private static int computeTabletNullPointsNumber( - InsertTabletNode insertTabletNode, int start, int end) { + InsertTabletNode insertTabletNode, int start, int end, boolean countFieldOnly) { Object[] values = insertTabletNode.getBitMaps(); int nullPointsNumber = 0; if (values != null) { - for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) { - if (insertTabletNode.isMeasurementFailed(i)) { + for (int i = 0; + insertTabletNode.getMeasurements() != null + && i < insertTabletNode.getMeasurements().length; + i++) { + if (!insertTabletNode.isValidMeasurement(i, countFieldOnly) + || insertTabletNode.getColumns() == null + || i >= insertTabletNode.getColumns().length + || insertTabletNode.getColumns()[i] == null) { // do not include failed measurement to avoid a negative pointsInserted continue; } - BitMap bitMap = (BitMap) values[i]; + BitMap bitMap = i < values.length ? (BitMap) values[i] : null; if (bitMap != null && !bitMap.isAllUnmarked()) { for (int j = start; j < end; j++) { if (bitMap.isMarked(j)) { @@ -365,19 +373,33 @@ public void writeAlignedRow( public void writeTabletNode(InsertTabletNode insertTabletNode, int start, int end) { List schemaList = new ArrayList<>(); - for (int i = 0; i < insertTabletNode.getMeasurementSchemas().length; i++) { - if (insertTabletNode.getColumns()[i] == null) { + final int measurementSize = + insertTabletNode.getMeasurements() == null ? 0 : insertTabletNode.getMeasurements().length; + final Object[] writableColumns = new Object[measurementSize]; + final BitMap[] writableBitMaps = + insertTabletNode.getBitMaps() == null ? null : new BitMap[measurementSize]; + for (int i = 0; i < measurementSize; i++) { + if (!insertTabletNode.isValidMeasurement(i, true) + || insertTabletNode.getColumns() == null + || i >= insertTabletNode.getColumns().length + || insertTabletNode.getColumns()[i] == null) { schemaList.add(null); } else { schemaList.add(insertTabletNode.getMeasurementSchemas()[i]); + writableColumns[i] = insertTabletNode.getColumns()[i]; + if (writableBitMaps != null + && insertTabletNode.getBitMaps() != null + && i < insertTabletNode.getBitMaps().length) { + writableBitMaps[i] = insertTabletNode.getBitMaps()[i]; + } } } IWritableMemChunkGroup memChunkGroup = createMemChunkGroupIfNotExistAndGet(insertTabletNode.getDeviceID(), schemaList); memChunkGroup.writeTablet( insertTabletNode.getTimes(), - insertTabletNode.getColumns(), - insertTabletNode.getBitMaps(), + writableColumns, + writableBitMaps, schemaList, start, end, @@ -387,13 +409,25 @@ public void writeTabletNode(InsertTabletNode insertTabletNode, int start, int en public void writeAlignedTablet( InsertTabletNode insertTabletNode, int start, int end, TSStatus[] results) { List schemaList = new ArrayList<>(); - for (int i = 0; i < insertTabletNode.getMeasurementSchemas().length; i++) { - if (insertTabletNode.getColumns()[i] == null - || (insertTabletNode.getColumnCategories() != null - && insertTabletNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD)) { + final int measurementSize = + insertTabletNode.getMeasurements() == null ? 0 : insertTabletNode.getMeasurements().length; + final Object[] writableColumns = new Object[measurementSize]; + final BitMap[] writableBitMaps = + insertTabletNode.getBitMaps() == null ? null : new BitMap[measurementSize]; + for (int i = 0; i < measurementSize; i++) { + if (!insertTabletNode.isValidMeasurement(i, true) + || insertTabletNode.getColumns() == null + || i >= insertTabletNode.getColumns().length + || insertTabletNode.getColumns()[i] == null) { schemaList.add(null); } else { schemaList.add(insertTabletNode.getMeasurementSchemas()[i]); + writableColumns[i] = insertTabletNode.getColumns()[i]; + if (writableBitMaps != null + && insertTabletNode.getBitMaps() != null + && i < insertTabletNode.getBitMaps().length) { + writableBitMaps[i] = insertTabletNode.getBitMaps()[i]; + } } } if (schemaList.isEmpty()) { @@ -409,8 +443,8 @@ public void writeAlignedTablet( createAlignedMemChunkGroupIfNotExistAndGet(deviceID, schemaList); memChunkGroup.writeTablet( insertTabletNode.getTimes(), - insertTabletNode.getColumns(), - insertTabletNode.getBitMaps(), + writableColumns, + writableBitMaps, schemaList, splitStart, splitEnd, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java index 873f49c54e217..2a574f0cb7b6e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java @@ -37,6 +37,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Iterator; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -279,6 +282,20 @@ public abstract void encode( @Override public abstract void setEncryptParameter(EncryptParameter encryptParameter); + protected static byte[] serializeSchemaToWALBytes(IMeasurementSchema schema) { + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(schema.serializedSize()); + schema.serializeTo(outputStream); + return outputStream.toByteArray(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + protected static int getSerializedSchemaSize(IMeasurementSchema schema) { + return serializeSchemaToWALBytes(schema).length; + } + public synchronized TVList initWorkingListForFlushIfNecessary( TVList workingList, boolean needCloneTimesAndIndicesInWorkingTVList) { if (workingListForFlush == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index b82786a32ed3d..949466a9f979f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -49,7 +49,6 @@ import java.io.DataInputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; @@ -898,7 +897,7 @@ public int serializedSize() { int size = 0; size += Integer.BYTES; for (IMeasurementSchema schema : schemaList) { - size += schema.serializedSize(); + size += getSerializedSchemaSize(schema); } size += Integer.BYTES; for (AlignedTVList alignedTvList : sortedList) { @@ -912,9 +911,7 @@ public int serializedSize() { public void serializeToWAL(IWALByteBufferView buffer) { WALWriteUtils.write(schemaList.size(), buffer); for (IMeasurementSchema schema : schemaList) { - byte[] bytes = new byte[schema.serializedSize()]; - schema.serializeTo(ByteBuffer.wrap(bytes)); - buffer.put(bytes); + buffer.put(serializeSchemaToWALBytes(schema)); } buffer.putInt(sortedList.size()); for (AlignedTVList alignedTvList : sortedList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index c4e488f84863f..2572cbec5020d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -306,8 +306,11 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) } else { memIncrements = checkMemCostAndAddToTspInfoForRow( - insertRowNode.getDeviceID(), insertRowNode.getMeasurements(), - insertRowNode.getDataTypes(), insertRowNode.getValues()); + insertRowNode.getDeviceID(), + insertRowNode.getMeasurements(), + insertRowNode.getDataTypes(), + insertRowNode.getValues(), + insertRowNode.getColumnCategories()); } // recordScheduleMemoryBlockCost infoForMetrics[1] += System.nanoTime() - memControlStartTime; @@ -523,6 +526,7 @@ private long[] checkMemCost( insertTabletNode.getMeasurements(), insertTabletNode.getDataTypes(), insertTabletNode.getColumns(), + insertTabletNode.getColumnCategories(), start, end); } @@ -676,16 +680,20 @@ public void insertTablet( @SuppressWarnings("squid:S3776") // High Cognitive Complexity private long[] checkMemCostAndAddToTspInfoForRow( - IDeviceID deviceId, String[] measurements, TSDataType[] dataTypes, Object[] values) + IDeviceID deviceId, + String[] measurements, + TSDataType[] dataTypes, + Object[] values, + TsTableColumnCategory[] columnCategories) throws WriteProcessException { // Memory of increased PrimitiveArray and TEXT values, e.g., add a long[128], add 128*8 long memTableIncrement = 0L; long textDataIncrement = 0L; long chunkMetadataIncrement = 0L; - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null || measurements[i] == null) { + if (!isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i)) { continue; } IWritableMemChunk memChunk = workMemTable.getWritableMemChunk(deviceId, measurements[i]); @@ -723,9 +731,10 @@ private long[] checkMemCostAndAddToTspInfoForRows(List insertRowN TSDataType[] dataTypes = insertRowNode.getDataTypes(); Object[] values = insertRowNode.getValues(); String[] measurements = insertRowNode.getMeasurements(); - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null || measurements[i] == null) { + if (!isWritableFieldMeasurement( + measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { continue; } IWritableMemChunk memChunk = workMemTable.getWritableMemChunk(deviceId, measurements[i]); @@ -779,21 +788,21 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( IWritableMemChunk memChunk = workMemTable.getWritableMemChunk(deviceId, AlignedPath.VECTOR_PLACEHOLDER); if (memChunk == null) { + TSDataType[] writableFieldDataTypes = + getWritableFieldDataTypes(measurements, dataTypes, values, columnCategories); // For new device of this mem table // ChunkMetadataIncrement chunkMetadataIncrement += ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR) - * dataTypes.length; - memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes, columnCategories); + * writableFieldDataTypes.length; + memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(writableFieldDataTypes, null); } else { // For existed device of this mem table AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk; List dataTypesInTVList = new ArrayList<>(); - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null - || measurements[i] == null - || (columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i)) { continue; } @@ -815,9 +824,10 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( } } - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // TEXT data mem size - if (dataTypes[i] != null && dataTypes[i].isBinary() && values[i] != null) { + if (isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i) + && dataTypes[i].isBinary()) { textDataIncrement += MemUtils.getBinarySize((Binary) values[i]); } } @@ -843,25 +853,26 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins IWritableMemChunk memChunk = workMemTable.getWritableMemChunk(deviceId, AlignedPath.VECTOR_PLACEHOLDER); if (memChunk == null && !increasingMemTableInfo.containsKey(deviceId)) { + TSDataType[] writableFieldDataTypes = + getWritableFieldDataTypes( + measurements, dataTypes, values, insertRowNode.getColumnCategories()); + Pair, Integer> addingPointNumInfo = + increasingMemTableInfo.computeIfAbsent(deviceId, k -> new Pair<>(new HashMap<>(), 0)); // For new device of this mem table // ChunkMetadataIncrement chunkMetadataIncrement += ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR) - * dataTypes.length; - memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes, null); - for (int i = 0; i < dataTypes.length; i++) { + * writableFieldDataTypes.length; + memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(writableFieldDataTypes, null); + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null - || measurements[i] == null - || (insertRowNode.getColumnCategories() != null - && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement( + measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { continue; } - increasingMemTableInfo - .computeIfAbsent(deviceId, k -> new Pair<>(new HashMap<>(), 1)) - .left - .put(measurements[i], dataTypes[i]); + addingPointNumInfo.left.put(measurements[i], dataTypes[i]); } + addingPointNumInfo.setRight(1); } else { // For existed device of this mem table @@ -870,12 +881,10 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins List dataTypesInTVList = new ArrayList<>(); Pair, Integer> addingPointNumInfo = increasingMemTableInfo.computeIfAbsent(deviceId, k -> new Pair<>(new HashMap<>(), 0)); - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null - || measurements[i] == null - || (insertRowNode.getColumnCategories() != null - && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement( + measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { continue; } @@ -912,12 +921,10 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins addingPointNumInfo.setRight(addingPointNum + 1); } - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null - || measurements[i] == null - || (insertRowNode.getColumnCategories() != null - && insertRowNode.getColumnCategories()[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement( + measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { continue; } // TEXT data mem size @@ -935,6 +942,7 @@ private long[] checkMemCostAndAddToTspInfoForTablet( String[] measurements, TSDataType[] dataTypes, Object[] columns, + TsTableColumnCategory[] columnCategories, int start, int end) throws WriteProcessException { @@ -943,9 +951,9 @@ private long[] checkMemCostAndAddToTspInfoForTablet( } long[] memIncrements = new long[3]; // memTable, text, chunk metadata - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements - if (dataTypes[i] == null || columns[i] == null || measurements[i] == null) { + if (!isWritableFieldMeasurement(measurements, dataTypes, columns, columnCategories, i)) { continue; } updateMemCost(dataTypes[i], measurements[i], deviceId, start, end, memIncrements, columns[i]); @@ -1053,16 +1061,8 @@ private void updateAlignedMemCost( } } - int measurementColumnNum = 0; - if (columnCategories == null) { - measurementColumnNum = dataTypes.length; - } else { - for (TsTableColumnCategory columnCategory : columnCategories) { - if (columnCategory == TsTableColumnCategory.FIELD) { - measurementColumnNum++; - } - } - } + TSDataType[] writableFieldDataTypes = + getWritableFieldDataTypes(measurementIds, dataTypes, columns, columnCategories); // memIncrements = [memTable, text, chunk metadata] respectively IWritableMemChunk memChunk = @@ -1071,7 +1071,7 @@ private void updateAlignedMemCost( // new devices introduce new ChunkMetadata // ChunkMetadata memory Increment memIncrements[2] += - measurementColumnNum + writableFieldDataTypes.length * ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR); // TVList memory @@ -1079,20 +1079,15 @@ private void updateAlignedMemCost( incomingPointNum / PrimitiveArrayManager.ARRAY_SIZE + (incomingPointNum % PrimitiveArrayManager.ARRAY_SIZE > 0 ? 1 : 0); memIncrements[0] += - numArraysToAdd * AlignedTVList.alignedTvListArrayMemCost(dataTypes, columnCategories); + numArraysToAdd * AlignedTVList.alignedTvListArrayMemCost(writableFieldDataTypes, null); } else { AlignedWritableMemChunk alignedMemChunk = (AlignedWritableMemChunk) memChunk; List dataTypesInTVList = new ArrayList<>(); int currentPointNum = alignedMemChunk.alignedListSize(); int newPointNum = currentPointNum + incomingPointNum; - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { TSDataType dataType = dataTypes[i]; - String measurement = measurementIds[i]; - Object column = columns[i]; - if (dataType == null - || column == null - || measurement == null - || (columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement(measurementIds, dataTypes, columns, columnCategories, i)) { continue; } @@ -1123,14 +1118,9 @@ private void updateAlignedMemCost( } // flexible-length data size - for (int i = 0; i < dataTypes.length; i++) { + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { TSDataType dataType = dataTypes[i]; - String measurement = measurementIds[i]; - Object column = columns[i]; - if (dataType == null - || column == null - || measurement == null - || (columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD)) { + if (!isWritableFieldMeasurement(measurementIds, dataTypes, columns, columnCategories, i)) { continue; } @@ -1141,6 +1131,50 @@ private void updateAlignedMemCost( } } + private static TSDataType[] getWritableFieldDataTypes( + String[] measurementIds, + TSDataType[] dataTypes, + Object[] valuesOrColumns, + TsTableColumnCategory[] columnCategories) { + List writableFieldDataTypes = new ArrayList<>(); + for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { + if (isWritableFieldMeasurement( + measurementIds, dataTypes, valuesOrColumns, columnCategories, i)) { + writableFieldDataTypes.add(dataTypes[i]); + } + } + return writableFieldDataTypes.toArray(new TSDataType[0]); + } + + private static boolean isWritableFieldMeasurement( + String[] measurementIds, + TSDataType[] dataTypes, + Object[] valuesOrColumns, + TsTableColumnCategory[] columnCategories, + int index) { + return isFieldMeasurement(measurementIds, dataTypes, columnCategories, index) + && valuesOrColumns != null + && index < valuesOrColumns.length + && valuesOrColumns[index] != null; + } + + private static boolean isFieldMeasurement( + String[] measurementIds, + TSDataType[] dataTypes, + TsTableColumnCategory[] columnCategories, + int index) { + return measurementIds != null + && dataTypes != null + && index >= 0 + && index < measurementIds.length + && index < dataTypes.length + && dataTypes[index] != null + && measurementIds[index] != null + && (columnCategories == null + || index < columnCategories.length + && columnCategories[index] == TsTableColumnCategory.FIELD); + } + private void updateMemoryInfo( long memTableIncrement, long chunkMetadataIncrement, long textDataIncrement) throws WriteProcessRejectException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java index 5abd87d07292d..a5a4ce25848c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java @@ -50,7 +50,6 @@ import java.io.DataInputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -528,7 +527,7 @@ public void release() { @Override public int serializedSize() { - int serializedSize = schema.serializedSize() + list.serializedSize(); + int serializedSize = getSerializedSchemaSize(schema) + list.serializedSize(); serializedSize += Integer.BYTES; for (TVList tvList : sortedList) { serializedSize += tvList.serializedSize(); @@ -538,9 +537,7 @@ public int serializedSize() { @Override public void serializeToWAL(IWALByteBufferView buffer) { - byte[] bytes = new byte[schema.serializedSize()]; - schema.serializeTo(ByteBuffer.wrap(bytes)); - buffer.put(bytes); + buffer.put(serializeSchemaToWALBytes(schema)); buffer.putInt(sortedList.size()); for (TVList tvList : sortedList) { tvList.serializeToWAL(buffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java index 89f28726c98d7..6f5c63824c491 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java @@ -179,7 +179,7 @@ public int serializedSize() { int size = 0; size += Integer.BYTES; for (Map.Entry entry : memChunkMap.entrySet()) { - size += ReadWriteIOUtils.sizeToWrite(entry.getKey()); + size += WALWriteUtils.sizeToWrite(entry.getKey()); size += entry.getValue().serializedSize(); } return size; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java index ad247680cb818..300c3c792e940 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java @@ -230,7 +230,8 @@ private List convertInsertRowNode(final InsertRowNode node) { final String[] measurements = node.getMeasurements(); final TSDataType[] dataTypes = node.getDataTypes(); final Object[] values = node.getValues(); - final List matchedColumnIndices = getMatchedTreeColumnIndices(deviceId, measurements); + final List matchedColumnIndices = + getMatchedTreeColumnIndices(deviceId, measurements, dataTypes, values, false); if (matchedColumnIndices.isEmpty()) { return Collections.emptyList(); @@ -283,7 +284,8 @@ private List convertInsertTabletNode(final InsertTabletNode node) { final int rowCount = node.getRowCount(); // Column filtering - final List matchedColumnIndices = getMatchedTreeColumnIndices(deviceId, measurements); + final List matchedColumnIndices = + getMatchedTreeColumnIndices(deviceId, measurements, dataTypes, columns, true); if (matchedColumnIndices.isEmpty()) { return Collections.emptyList(); } @@ -378,7 +380,9 @@ private MatchedTreeRow matchTreeInsertRowNode(final InsertRowNode node) { final String[] measurements = node.getMeasurements(); final TSDataType[] dataTypes = node.getDataTypes(); - final List matchedColumnIndices = getMatchedTreeColumnIndices(deviceId, measurements); + final Object[] values = node.getValues(); + final List matchedColumnIndices = + getMatchedTreeColumnIndices(deviceId, measurements, dataTypes, values, false); return matchedColumnIndices.isEmpty() ? null : new MatchedTreeRow( @@ -455,7 +459,8 @@ private List convertRelationalInsertRowNode(final RelationalInsertRowNod final TSDataType[] dataTypes = node.getDataTypes(); final Object[] values = node.getValues(); final List matchedColumnIndices = - getMatchedTableColumnIndices(measurements, node.getColumnCategories()); + getMatchedTableColumnIndices( + measurements, dataTypes, values, node.getColumnCategories(), false); if (matchedColumnIndices.isEmpty()) { return Collections.emptyList(); } @@ -512,7 +517,8 @@ private List convertRelationalInsertTabletNode(final RelationalInsertTab final BitMap[] bitMaps = node.getBitMaps(); final int rowCount = node.getRowCount(); final List matchedColumnIndices = - getMatchedTableColumnIndices(measurements, node.getColumnCategories()); + getMatchedTableColumnIndices( + measurements, dataTypes, columns, node.getColumnCategories(), true); if (matchedColumnIndices.isEmpty()) { return Collections.emptyList(); } @@ -568,12 +574,19 @@ private List convertRelationalInsertRowsNode(final RelationalInsertRowsN * column indices are returned. */ private List getMatchedTreeColumnIndices( - final IDeviceID deviceId, final String[] measurements) { + final IDeviceID deviceId, + final String[] measurements, + final TSDataType[] dataTypes, + final Object[] valuesOrColumns, + final boolean requireNonNullValue) { + if (measurements == null) { + return Collections.emptyList(); + } if (treePattern == null || treePattern.isRoot() || treePattern.coversDevice(deviceId)) { // All columns match final List allIndices = new ArrayList<>(measurements.length); for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null) { + if (isValidColumn(measurements, dataTypes, valuesOrColumns, i, requireNonNullValue)) { allIndices.add(i); } } @@ -582,7 +595,8 @@ private List getMatchedTreeColumnIndices( final List matchedIndices = new ArrayList<>(); for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null && treePattern.matchesMeasurement(deviceId, measurements[i])) { + if (isValidColumn(measurements, dataTypes, valuesOrColumns, i, requireNonNullValue) + && treePattern.matchesMeasurement(deviceId, measurements[i])) { matchedIndices.add(i); } } @@ -595,11 +609,18 @@ private List getMatchedTreeColumnIndices( * If no table column pattern is specified, all non-null columns are returned. */ private List getMatchedTableColumnIndices( - final String[] measurements, final TsTableColumnCategory[] columnCategories) { + final String[] measurements, + final TSDataType[] dataTypes, + final Object[] valuesOrColumns, + final TsTableColumnCategory[] columnCategories, + final boolean requireNonNullValue) { + if (measurements == null) { + return Collections.emptyList(); + } final boolean[] selectedColumns = new boolean[measurements.length]; boolean hasMatchedColumn = false; for (int i = 0; i < measurements.length; i++) { - if (measurements[i] == null) { + if (!isValidColumn(measurements, dataTypes, valuesOrColumns, i, requireNonNullValue)) { continue; } if (tableColumnPattern == null || tableColumnPattern.matcher(measurements[i]).matches()) { @@ -613,7 +634,8 @@ private List getMatchedTableColumnIndices( } for (int i = 0; i < measurements.length; i++) { - if (measurements[i] != null && isTagColumn(columnCategories, i)) { + if (isValidColumn(measurements, dataTypes, valuesOrColumns, i, requireNonNullValue) + && isTagColumn(columnCategories, i)) { selectedColumns[i] = true; } } @@ -629,16 +651,38 @@ private List getMatchedTableColumnIndices( private boolean isTagColumn( final TsTableColumnCategory[] columnCategories, final int columnIndex) { - return columnCategories != null && columnCategories[columnIndex] == TsTableColumnCategory.TAG; + return columnCategories != null + && columnIndex < columnCategories.length + && columnCategories[columnIndex] == TsTableColumnCategory.TAG; } private ColumnCategory toTsFileColumnCategory( final TsTableColumnCategory[] columnCategories, final int columnIndex) { - return columnCategories != null && columnCategories[columnIndex] != null + return columnCategories != null + && columnIndex < columnCategories.length + && columnCategories[columnIndex] != null ? columnCategories[columnIndex].toTsFileColumnType() : ColumnCategory.FIELD; } + private boolean isValidColumn( + final String[] measurements, + final TSDataType[] dataTypes, + final Object[] valuesOrColumns, + final int index, + final boolean requireNonNullValue) { + return measurements != null + && index >= 0 + && index < measurements.length + && measurements[index] != null + && dataTypes != null + && index < dataTypes.length + && dataTypes[index] != null + && valuesOrColumns != null + && index < valuesOrColumns.length + && (!requireNonNullValue || valuesOrColumns[index] != null); + } + /** * Adds a single value to the tablet at the specified position. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java index 6fc369d89e34c..fab71cf8014b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/trigger/executor/TriggerFireVisitor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.consensus.ConfigRegionId; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.trigger.TriggerInformation; import org.apache.iotdb.commons.trigger.TriggerTable; import org.apache.iotdb.commons.trigger.exception.TriggerExecutionException; @@ -272,17 +273,20 @@ private Map constructMeasurementToSchemaIndexMap( // The index of measurement and schema is the same now. // However, in case one day the order changes, we need to construct an index map. Map indexMap = new HashMap<>(); + if (measurements == null || schemas == null) { + return indexMap; + } for (int i = 0, n = measurements.length; i < n; i++) { - if (measurements[i] == null) { + if (measurements[i] == null || i >= schemas.length) { continue; } // It is the same now - if (schemas[i] != null && schemas[i].getMeasurementName().equals(measurements[i])) { + if (schemas[i] != null && measurements[i].equals(schemas[i].getMeasurementName())) { indexMap.put(measurements[i], i); continue; } for (int j = 0, m = schemas.length; j < m; j++) { - if (schemas[j] != null && schemas[j].getMeasurementName().equals(measurements[i])) { + if (schemas[j] != null && measurements[i].equals(schemas[j].getMeasurementName())) { indexMap.put(measurements[i], j); break; } @@ -299,9 +303,13 @@ private Map> constructTriggerNameToMeasurementListMap( return Collections.emptyMap(); } List measurements = new ArrayList<>(); - for (String measurement : node.getMeasurements()) { - if (measurement != null) { - measurements.add(measurement); + final String[] nodeMeasurements = node.getMeasurements(); + if (nodeMeasurements == null) { + return Collections.emptyMap(); + } + for (int i = 0; i < nodeMeasurements.length; i++) { + if (isTriggerableMeasurement(node, i)) { + measurements.add(nodeMeasurements[i]); } } @@ -334,6 +342,39 @@ private Map> constructTriggerNameToMeasurementListMap( return triggerNameToPaths; } + private boolean isTriggerableMeasurement(final InsertNode node, final int index) { + final String[] measurements = node.getMeasurements(); + final MeasurementSchema[] schemas = node.getMeasurementSchemas(); + if (measurements == null + || index < 0 + || index >= measurements.length + || measurements[index] == null + || schemas == null + || index >= schemas.length + || schemas[index] == null + || schemas[index].getMeasurementName() == null + || schemas[index].getType() == null + || !isFieldMeasurement(node, index)) { + return false; + } + if (node instanceof InsertTabletNode) { + final Object[] columns = ((InsertTabletNode) node).getColumns(); + return columns != null && index < columns.length && columns[index] != null; + } + if (node instanceof InsertRowNode) { + final Object[] values = ((InsertRowNode) node).getValues(); + return values != null && index < values.length; + } + return true; + } + + private boolean isFieldMeasurement(final InsertNode node, final int index) { + final TsTableColumnCategory[] columnCategories = node.getColumnCategories(); + return columnCategories == null + || index < columnCategories.length + && columnCategories[index] == TsTableColumnCategory.FIELD; + } + private TriggerFireResult fire(String triggerName, Tablet tablet, TriggerEvent event) { TriggerFireResult result = TriggerFireResult.SUCCESS; for (int i = 0; i < FIRE_RETRY_NUM; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java index 3c4d24b5796d7..b724c4d3c6f98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java @@ -67,14 +67,22 @@ public static long getRecordSize(TSDataType dataType, Object value, boolean addi * memory before insertion */ public static long getRowRecordSize(List dataTypes, Object[] value) { - int emptyRecordCount = 0; + return getRowRecordSize(dataTypes, value, null); + } + + public static long getRowRecordSize( + List dataTypes, Object[] value, TsTableColumnCategory[] columnCategories) { + // dataTypes contains only non-null FIELD types, in the same order as non-null FIELD values. + if (dataTypes == null) { + return 0L; + } + int dataTypeIndex = 0; long memSize = 0L; - for (int i = 0; i < value.length; i++) { - if (value[i] == null) { - emptyRecordCount++; + for (int i = 0; value != null && i < value.length && dataTypeIndex < dataTypes.size(); i++) { + if (value[i] == null || !isFieldColumn(columnCategories, i)) { continue; } - memSize += getRecordSize(dataTypes.get(i - emptyRecordCount), value[i], false); + memSize += getRecordSize(dataTypes.get(dataTypeIndex++), value[i], false); } return memSize; } @@ -85,15 +93,21 @@ public static long getRowRecordSize(List dataTypes, Object[] value) */ public static long getAlignedRowRecordSize( List dataTypes, Object[] value, TsTableColumnCategory[] columnCategories) { + // dataTypes contains only non-null FIELD types, in the same order as non-null FIELD values. + if (dataTypes == null) { + return 8L + 4L; + } // time and index size long memSize = 8L + 4L; - for (int i = 0; i < dataTypes.size(); i++) { - if (value[i] == null - || dataTypes.get(i).isBinary() - || columnCategories != null && columnCategories[i] != TsTableColumnCategory.FIELD) { + int dataTypeIndex = 0; + for (int i = 0; value != null && i < value.length && dataTypeIndex < dataTypes.size(); i++) { + if (value[i] == null || !isFieldColumn(columnCategories, i)) { continue; } - memSize += dataTypes.get(i).getDataTypeSize(); + TSDataType dataType = dataTypes.get(dataTypeIndex++); + if (!dataType.isBinary()) { + memSize += dataType.getDataTypeSize(); + } } return memSize; } @@ -126,8 +140,10 @@ public static long getTabletSize(InsertTabletNode insertTabletNode, int start, i return 0L; } long memSize = 0; - for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) { - if (insertTabletNode.getMeasurements()[i] == null) { + for (int i = 0; + insertTabletNode.getMeasurements() != null && i < insertTabletNode.getMeasurements().length; + i++) { + if (!isWritableTabletMeasurement(insertTabletNode, i, true)) { continue; } // Time column memSize @@ -143,8 +159,10 @@ public static long getAlignedTabletSize( return 0L; } long memSize = 0; - for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) { - if (!insertTabletNode.isValidMeasurement(i)) { + for (int i = 0; + insertTabletNode.getMeasurements() != null && i < insertTabletNode.getMeasurements().length; + i++) { + if (!isWritableTabletMeasurement(insertTabletNode, i, true)) { continue; } if (results == null) { @@ -163,6 +181,31 @@ public static long getAlignedTabletSize( return memSize; } + private static boolean isWritableTabletMeasurement( + InsertTabletNode insertTabletNode, int index, boolean fieldOnly) { + return insertTabletNode.getMeasurements() != null + && index >= 0 + && index < insertTabletNode.getMeasurements().length + && insertTabletNode.getMeasurements()[index] != null + && insertTabletNode.getColumns() != null + && index < insertTabletNode.getColumns().length + && insertTabletNode.getColumns()[index] != null + && insertTabletNode.getDataTypes() != null + && index < insertTabletNode.getDataTypes().length + && insertTabletNode.getDataTypes()[index] != null + && insertTabletNode.getMeasurementSchemas() != null + && index < insertTabletNode.getMeasurementSchemas().length + && insertTabletNode.getMeasurementSchemas()[index] != null + && (!fieldOnly || isFieldColumn(insertTabletNode.getColumnCategories(), index)); + } + + private static boolean isFieldColumn(TsTableColumnCategory[] columnCategories, int columnIndex) { + return columnCategories == null + || columnIndex >= 0 + && columnIndex < columnCategories.length + && columnCategories[columnIndex] == TsTableColumnCategory.FIELD; + } + /** Calculate how much memory will be used if the given record is written to sequence file. */ public static long getTsRecordMem(TSRecord tsRecord) { long memUsed = 8; // time diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java index da3dee91caad9..6b1f9e3d3d3b1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java @@ -33,10 +33,12 @@ import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTablePatternParser; import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTreePatternParser; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.tsfile.enums.TSDataType; @@ -350,6 +352,75 @@ nodeWithSparseColumn, new PrefixTreePattern(pattern)) Assert.assertTrue(tablet.isNull(1, 1)); } + @Test + public void convertToTabletSkipsFailedMeasurementsForCoveredTreePattern() throws Exception { + final InsertRowNode rowNode = + new InsertRowNode( + new PlanNodeId("plan node failed row"), + new PartialPath(deviceId), + false, + Arrays.copyOf(measurementIds, measurementIds.length), + Arrays.copyOf(dataTypes, dataTypes.length), + Arrays.copyOf(schemas, schemas.length), + times[0], + Arrays.copyOf(insertRowNode.getValues(), schemas.length), + false); + rowNode.markFailedMeasurement(1); + + final Tablet rowTablet = + new TabletInsertionEventTreePatternParser(rowNode, new PrefixTreePattern(pattern)) + .convertToTablet(); + assertTabletDoesNotContainMeasurement(rowTablet, "s2", schemas.length - 1); + + final InsertTabletNode tabletNode = + new InsertTabletNode( + new PlanNodeId("plan node failed tablet"), + new PartialPath(deviceId), + false, + Arrays.copyOf(measurementIds, measurementIds.length), + Arrays.copyOf(dataTypes, dataTypes.length), + Arrays.copyOf(schemas, schemas.length), + times, + null, + Arrays.copyOf(insertTabletNode.getColumns(), schemas.length), + times.length); + tabletNode.markFailedMeasurement(1); + + final Tablet tablet = + new TabletInsertionEventTreePatternParser(tabletNode, new PrefixTreePattern(pattern)) + .convertToTablet(); + assertTabletDoesNotContainMeasurement(tablet, "s2", schemas.length - 1); + } + + @Test + public void convertToTabletSkipsFailedMeasurementsForTablePattern() throws Exception { + final TsTableColumnCategory[] columnCategories = new TsTableColumnCategory[schemas.length]; + Arrays.fill(columnCategories, TsTableColumnCategory.FIELD); + columnCategories[0] = TsTableColumnCategory.TAG; + columnCategories[2] = TsTableColumnCategory.ATTRIBUTE; + + final RelationalInsertTabletNode tabletNode = + new RelationalInsertTabletNode( + new PlanNodeId("plan node failed relational tablet"), + new PartialPath("table1", false), + false, + Arrays.copyOf(measurementIds, measurementIds.length), + Arrays.copyOf(dataTypes, dataTypes.length), + times, + null, + Arrays.copyOf(insertTabletNode.getColumns(), schemas.length), + times.length, + columnCategories); + tabletNode.setMeasurementSchemas(Arrays.copyOf(schemas, schemas.length)); + tabletNode.markFailedMeasurement(1); + + final Tablet tablet = + new TabletInsertionEventTablePatternParser( + null, null, tabletNode, new TablePattern(true, null, null)) + .convertToTablet(); + assertTabletDoesNotContainMeasurement(tablet, "s2", schemas.length - 1); + } + @Test public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container1 = @@ -538,4 +609,14 @@ public TSStatus checkSeriesPrivilege4Pipe( : new TSStatus(org.apache.iotdb.rpc.TSStatusCode.SUCCESS_STATUS.getStatusCode()); } } + + private static void assertTabletDoesNotContainMeasurement( + final Tablet tablet, final String measurement, final int expectedSchemaSize) { + Assert.assertEquals(expectedSchemaSize, tablet.getSchemas().size()); + for (int i = 0; i < tablet.getSchemas().size(); i++) { + Assert.assertNotNull(tablet.getSchemas().get(i)); + Assert.assertNotEquals(measurement, tablet.getSchemas().get(i).getMeasurementName()); + Assert.assertNotNull(tablet.getValues()[i]); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java new file mode 100644 index 0000000000000..8c563a84b1db4 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.receiver.transform.statement; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.time.ZoneId; + +public class PipeConvertedInsertRowStatementTest { + + @Test + public void testTransferTypeKeepsNullValue() throws Exception { + final InsertRowStatement statement = new InsertRowStatement(); + statement.setDevicePath(new PartialPath("root.sg.d1")); + statement.setMeasurements(new String[] {"s0"}); + statement.setMeasurementSchemas( + new MeasurementSchema[] {new MeasurementSchema("s0", TSDataType.INT32)}); + statement.setDataTypes(new TSDataType[] {null}); + statement.setTime(1L); + statement.setValues(new Object[] {null}); + statement.setNeedInferType(true); + + final PipeConvertedInsertRowStatement convertedStatement = + new PipeConvertedInsertRowStatement(statement); + convertedStatement.transferType(ZoneId.systemDefault()); + + Assert.assertEquals(TSDataType.INT32, convertedStatement.getDataTypes()[0]); + Assert.assertEquals("s0", convertedStatement.getMeasurements()[0]); + Assert.assertNull(convertedStatement.getValues()[0]); + Assert.assertFalse(convertedStatement.isNeedInferType()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java index cefc2377dc79e..06db08aaf57b0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java @@ -134,6 +134,18 @@ public void testInsertMultiTabletsNodeLaterTabletSizeIsEstimated() throws Illega Assert.assertTrue(largerNodeSize > baselineSize); } + @Test + public void testInsertTabletNodeWithNullColumnIsEstimated() throws IllegalPathException { + InsertTabletNode tablet = createTextInsertTabletNode("tablet", "root.sg.d1", 2, 4, 8); + + long fullSize = InsertNodeMemoryEstimator.sizeOf(tablet); + tablet.getColumns()[1] = null; + long sizeWithNullColumn = InsertNodeMemoryEstimator.sizeOf(tablet); + + Assert.assertTrue(sizeWithNullColumn > 0); + Assert.assertTrue(sizeWithNullColumn < fullSize); + } + @Test public void testPlanNodeIdIsEstimated() throws IllegalPathException { InsertRowNode shortPlanNodeIdRow = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java index 410afc7613030..0a954ef4b5e97 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/util/TabletStatementConverterTest.java @@ -20,11 +20,13 @@ package org.apache.iotdb.db.pipe.sink.util; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -172,6 +174,68 @@ public void testRoundTripConversionTableModel() throws MetadataException { assertTabletsEqual(originalTablet, convertedTablet); } + @Test + public void testDeserializeStatementFromTabletFormatWithNullSchemaAndNullColumn() + throws IOException, MetadataException { + final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + ReadWriteIOUtils.write("root.sg.device", outputStream); + ReadWriteIOUtils.write(1, outputStream); + + ReadWriteIOUtils.write(BytesUtils.boolToByte(true), outputStream); + ReadWriteIOUtils.write(1, outputStream); + ReadWriteIOUtils.write(BytesUtils.boolToByte(false), outputStream); + + ReadWriteIOUtils.write(BytesUtils.boolToByte(true), outputStream); + ReadWriteIOUtils.write(1L, outputStream); + + ReadWriteIOUtils.write(BytesUtils.boolToByte(false), outputStream); + + ReadWriteIOUtils.write(BytesUtils.boolToByte(true), outputStream); + ReadWriteIOUtils.write(BytesUtils.boolToByte(false), outputStream); + + ReadWriteIOUtils.write(true, outputStream); + + final ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + + final InsertTabletStatement statement = + TabletStatementConverter.deserializeStatementFromTabletFormat(buffer); + + Assert.assertArrayEquals(new String[] {null}, statement.getMeasurements()); + Assert.assertArrayEquals(new TSDataType[] {null}, statement.getDataTypes()); + Assert.assertNull(statement.getColumns()[0]); + Assert.assertTrue(statement.isAligned()); + } + + @Test + public void testConvertStatementToTabletSkipsNullColumn() throws Exception { + final InsertTabletStatement statement = new InsertTabletStatement(); + statement.setDevicePath(new PartialPath("root.sg.device")); + statement.setTimes(new long[] {1L, 2L}); + statement.setRowCount(2); + statement.setMeasurements(new String[] {"s1", "s2", "s3"}); + statement.setDataTypes( + new TSDataType[] {TSDataType.INT32, TSDataType.INT64, TSDataType.DOUBLE}); + statement.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64), + new MeasurementSchema("s3", TSDataType.DOUBLE) + }); + statement.setColumns(new Object[] {new int[] {1, 2}, null, new double[] {1.0, 2.0}}); + + final Tablet convertedTablet = statement.convertToTablet(); + + Assert.assertEquals(2, convertedTablet.getSchemas().size()); + Assert.assertEquals("s1", convertedTablet.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals("s3", convertedTablet.getSchemas().get(1).getMeasurementName()); + Assert.assertArrayEquals(new int[] {1, 2}, (int[]) convertedTablet.getValues()[0]); + Assert.assertArrayEquals( + new double[] {1.0, 2.0}, (double[]) convertedTablet.getValues()[1], 0.0); + } + /** * Generate a Tablet for tree model with all data types and specified number of columns and rows. * diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java index f068173632981..5a5f2d9d35d8c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java @@ -176,6 +176,133 @@ public void testRelationalSerializedSizeWithFailedMeasurement() { Assert.assertEquals(insertRowNode.serializedSize(), byteBuffer.position()); } + @Test + public void testSerializeSkipsRetainedMeasurementWithMissingValue() throws IllegalPathException { + InsertRowNode insertRowNode = + new InsertRowNode( + new PlanNodeId("plannode missing value"), + new PartialPath("root.sg.d1"), + false, + new String[] {"s1", "s2", "s3"}, + new TSDataType[] {TSDataType.INT32, TSDataType.INT32, TSDataType.INT32}, + 1L, + new Object[] {1, 2}, + false); + + ByteBuffer byteBuffer = ByteBuffer.allocate(10000); + insertRowNode.serialize(byteBuffer); + byteBuffer.flip(); + + Assert.assertEquals(PlanNodeType.INSERT_ROW.getNodeType(), byteBuffer.getShort()); + + InsertRowNode tmpNode = InsertRowNode.deserialize(byteBuffer); + Assert.assertArrayEquals(new String[] {"s1", "s2"}, tmpNode.getMeasurements()); + } + + @Test + public void testSerializeKeepsNullRowValueWithoutType() throws IllegalPathException { + InsertRowNode insertRowNode = + new InsertRowNode( + new PlanNodeId("plannode null value"), + new PartialPath("root.sg.d1"), + false, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32, null}, + 1L, + new Object[] {1, null}, + false); + + ByteBuffer byteBuffer = ByteBuffer.allocate(10000); + insertRowNode.serialize(byteBuffer); + byteBuffer.flip(); + + Assert.assertEquals(PlanNodeType.INSERT_ROW.getNodeType(), byteBuffer.getShort()); + + InsertRowNode tmpNode = InsertRowNode.deserialize(byteBuffer); + Assert.assertArrayEquals(new String[] {"s1", "s2"}, tmpNode.getMeasurements()); + Assert.assertArrayEquals(new TSDataType[] {TSDataType.INT32, null}, tmpNode.getDataTypes()); + Assert.assertArrayEquals(new Object[] {1, null}, tmpNode.getValues()); + } + + @Test + public void testDeserializeFromWALSkipsRetainedMeasurementWithNullSchema() + throws IllegalPathException, IOException { + InsertRowNode insertRowNode = getInsertRowNodeWithMeasurementSchemas(); + insertRowNode.getMeasurementSchemas()[1] = null; + + byte[] bytes = new byte[insertRowNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertRowNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_ROW.getNodeType(), dataInputStream.readShort()); + + InsertRowNode tmpNode = InsertRowNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + } + + @Test + public void testDeserializeFromWALSkipsRetainedMeasurementWithMissingValue() + throws IllegalPathException, IOException { + InsertRowNode insertRowNode = getInsertRowNodeWithMeasurementSchemas(); + insertRowNode.setValues(new Object[] {5.0, 6.0f}); + + byte[] bytes = new byte[insertRowNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertRowNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_ROW.getNodeType(), dataInputStream.readShort()); + + InsertRowNode tmpNode = InsertRowNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "\u6e7f\u5ea6"}, tmpNode.getMeasurements()); + } + + @Test + public void testRelationalDeserializeFromWALSkipsRetainedMeasurementWithNullCategory() + throws IOException { + RelationalInsertRowNode insertRowNode = getRelationalInsertRowNodeWithMeasurementSchemas(); + insertRowNode.getColumnCategories()[1] = null; + + byte[] bytes = new byte[insertRowNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertRowNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals( + PlanNodeType.RELATIONAL_INSERT_ROW.getNodeType(), dataInputStream.readShort()); + + RelationalInsertRowNode tmpNode = RelationalInsertRowNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals(new String[] {"id", "value"}, tmpNode.getMeasurements()); + Assert.assertArrayEquals( + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD}, + tmpNode.getColumnCategories()); + } + + @Test + public void testDeserializeFromWALWithMarkedFailedMeasurementOnly() + throws IllegalPathException, IOException { + InsertRowNode insertRowNode = getInsertRowNodeWithMeasurementSchemas(); + insertRowNode.markFailedMeasurement(1); + + byte[] bytes = new byte[insertRowNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertRowNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_ROW.getNodeType(), dataInputStream.readShort()); + + InsertRowNode tmpNode = InsertRowNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + } + private InsertRowNode getInsertRowNode() throws IllegalPathException { long time = 110L; TSDataType[] dataTypes = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java index 034d4cb57de40..907af18ef9adb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsNodeSerdeTest.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; @@ -270,6 +271,40 @@ public void TestSerializeAndDeserializeRelational() throws IllegalPathException } } + @Test + public void testRelationalRowsGetDeviceIDSkipsMissingTagValue() throws IllegalPathException { + RelationalInsertRowsNode node = new RelationalInsertRowsNode(new PlanNodeId("plan node 1")); + node.addOneInsertRowNode( + new RelationalInsertRowNode( + new PlanNodeId("plan node 1"), + new PartialPath("table1", false), + false, + new String[] {"id", "value"}, + new TSDataType[] {TSDataType.STRING, TSDataType.DOUBLE}, + 1000L, + new Object[] {"id1", 1.0}, + false, + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD}), + 0); + node.addOneInsertRowNode( + new RelationalInsertRowNode( + new PlanNodeId("plan node 1"), + new PartialPath("table1", false), + false, + new String[] {"id", "value"}, + new TSDataType[] {TSDataType.STRING, TSDataType.DOUBLE}, + 2000L, + new Object[] {2.0}, + false, + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD, TsTableColumnCategory.TAG}), + 1); + + Assert.assertEquals( + Factory.DEFAULT_FACTORY.create(new String[] {"table1", "id1"}), node.getDeviceID(0)); + Assert.assertEquals( + Factory.DEFAULT_FACTORY.create(new String[] {"table1", null}), node.getDeviceID(1)); + } + @Test public void testSerializeAndDeserializeForWALRelational() throws IOException { for (String tableName : new String[] {"table1", "ta`ble1", "root.table1"}) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsOfOneDeviceNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsOfOneDeviceNodeSerdeTest.java index 2cffd65e41e21..35204133bb8ef 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsOfOneDeviceNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowsOfOneDeviceNodeSerdeTest.java @@ -79,4 +79,38 @@ public void TestSerializeAndDeserialize() throws IllegalPathException { Assert.assertEquals(node, InsertRowsOfOneDeviceNode.deserialize(byteBuffer)); } + + @Test + public void testStoreMeasurementsSkipsFailedMeasurements() throws IllegalPathException { + PartialPath device = new PartialPath("root.sg.d"); + InsertRowsOfOneDeviceNode node = new InsertRowsOfOneDeviceNode(new PlanNodeId("plan node 1")); + + List insertRowNodeList = new ArrayList<>(); + InsertRowNode firstRow = + new InsertRowNode( + new PlanNodeId("plan node 1"), + device, + false, + new String[] {"s1", "failed"}, + new TSDataType[] {TSDataType.DOUBLE, TSDataType.FLOAT}, + 1000L, + new Object[] {1.0, 2f}, + false); + firstRow.markFailedMeasurement(1); + insertRowNodeList.add(firstRow); + insertRowNodeList.add( + new InsertRowNode( + new PlanNodeId("plan node 1"), + device, + false, + new String[] {"s2"}, + new TSDataType[] {TSDataType.INT64}, + 2000L, + new Object[] {300L}, + false)); + + node.setInsertRowNodeList(insertRowNodeList); + + Assert.assertArrayEquals(new String[] {"s1", "s2"}, node.getMeasurements()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java index 8cf0ce6e36e49..a2a407a66d451 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java @@ -30,6 +30,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; @@ -235,6 +236,135 @@ public void testRelationalSerializedSizeWithFailedMeasurement() { Assert.assertEquals(insertTabletNode.serializedSize(), byteBuffer.position()); } + @Test + public void testSerializedSizeWithClearedMeasurementAndRetainedColumn() + throws IllegalPathException { + InsertTabletNode insertTabletNode = getInsertTabletNodeWithSchema(); + insertTabletNode.getMeasurements()[1] = null; + insertTabletNode.getMeasurementSchemas()[1] = null; + insertTabletNode.getDataTypes()[1] = null; + + ByteBuffer byteBuffer = ByteBuffer.allocate(insertTabletNode.serializedSize()); + insertTabletNode.serializeToWAL(new WALByteBufferForTest(byteBuffer)); + + Assert.assertEquals(insertTabletNode.serializedSize(), byteBuffer.position()); + } + + @Test + public void testSerializedSizeWithRetainedMeasurementAndNullColumn() + throws IllegalPathException, IOException { + InsertTabletNode insertTabletNode = getInsertTabletNodeWithSchema(); + insertTabletNode.getColumns()[1] = null; + + byte[] bytes = new byte[insertTabletNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertTabletNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_TABLET.getNodeType(), dataInputStream.readShort()); + + InsertTabletNode tmpNode = InsertTabletNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + } + + @Test + public void testSerializeToWALWithoutMeasurementSchemas() throws Exception { + InsertTabletNode insertTabletNode = getInsertTabletNode(); + + byte[] bytes = new byte[insertTabletNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertTabletNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_TABLET.getNodeType(), dataInputStream.readShort()); + + InsertTabletNode tmpNode = InsertTabletNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals(new String[0], tmpNode.getMeasurements()); + } + + @Test + public void testSerializeToWALWithShortBitMaps() throws Exception { + InsertTabletNode insertTabletNode = getInsertTabletNodeWithSchema(); + BitMap bitMap = new BitMap(insertTabletNode.getRowCount()); + bitMap.mark(0); + insertTabletNode.setBitMaps(new BitMap[] {bitMap}); + + byte[] bytes = new byte[insertTabletNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertTabletNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_TABLET.getNodeType(), dataInputStream.readShort()); + + InsertTabletNode tmpNode = InsertTabletNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "\u6e7f\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + Assert.assertNotNull(tmpNode.getBitMaps()); + Assert.assertTrue(tmpNode.getBitMaps()[0].isMarked(0)); + } + + @Test + public void testRelationalSerializedSizeWithRetainedMeasurementAndNullColumn() { + RelationalInsertTabletNode insertTabletNode = getRelationalInsertTabletNodeWithSchema("table1"); + insertTabletNode.getColumns()[1] = null; + + ByteBuffer byteBuffer = ByteBuffer.allocate(insertTabletNode.serializedSize()); + insertTabletNode.serializeToWAL(new WALByteBufferForTest(byteBuffer)); + + Assert.assertEquals(insertTabletNode.serializedSize(), byteBuffer.position()); + } + + @Test + public void testRelationalDeserializeFromWALSkipsRetainedMeasurementWithNullCategory() + throws IOException { + RelationalInsertTabletNode insertTabletNode = getRelationalInsertTabletNodeWithSchema("table1"); + insertTabletNode.getColumnCategories()[1] = null; + + byte[] bytes = new byte[insertTabletNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertTabletNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals( + PlanNodeType.RELATIONAL_INSERT_TABLET.getNodeType(), dataInputStream.readShort()); + + RelationalInsertTabletNode tmpNode = + RelationalInsertTabletNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals(new String[] {"s1", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + Assert.assertArrayEquals( + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.TAG, + TsTableColumnCategory.FIELD + }, + tmpNode.getColumnCategories()); + } + + @Test + public void testDeserializeFromWALWithMarkedFailedMeasurementOnly() + throws IllegalPathException, IOException { + InsertTabletNode insertTabletNode = getInsertTabletNodeWithSchema(); + insertTabletNode.markFailedMeasurement(1); + + byte[] bytes = new byte[insertTabletNode.serializedSize()]; + WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.wrap(bytes)); + insertTabletNode.serializeToWAL(walBuffer); + Assert.assertFalse(walBuffer.getBuffer().hasRemaining()); + + DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes)); + Assert.assertEquals(PlanNodeType.INSERT_TABLET.getNodeType(), dataInputStream.readShort()); + + InsertTabletNode tmpNode = InsertTabletNode.deserializeFromWAL(dataInputStream); + Assert.assertArrayEquals( + new String[] {"\u6e29\u5ea6", "s3", "s4", "s5"}, tmpNode.getMeasurements()); + } + @Test public void testInitTabletValuesWithAllTypes() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java index 01857cb5f8aa9..6d5185af4ba7b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java @@ -260,6 +260,47 @@ public void testSplitInsertTablet() throws IllegalPathException { } } + @Test + public void testSplitInsertTabletSkipsClearedMeasurementWithRetainedColumn() + throws IllegalPathException { + InsertTabletNode insertTabletNode = new InsertTabletNode(new PlanNodeId("plan node 1")); + + insertTabletNode.setTargetPath(new PartialPath("root.sg1.d1")); + insertTabletNode.setMeasurements(new String[] {"s0", null}); + insertTabletNode.setTimes( + new long[] {-200, -101, 1, 60, 120, 180, 270, 290, 360, 375, 440, 470}); + insertTabletNode.setDataTypes(new TSDataType[] {TSDataType.INT32, TSDataType.INT32}); + insertTabletNode.setColumns( + new Object[] { + new int[] {-20, -10, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}, + new int[] {-2, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10} + }); + insertTabletNode.setRowCount(insertTabletNode.getTimes().length); + final BitMap[] bitMaps = new BitMap[] {null, new BitMap(insertTabletNode.getRowCount())}; + bitMaps[1].mark(2); + insertTabletNode.setBitMaps(bitMaps); + + DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); + dataPartitionQueryParam.setDeviceID( + insertTabletNode.getTargetPath().getIDeviceIDAsFullDevice()); + dataPartitionQueryParam.setTimePartitionSlotList(insertTabletNode.getTimePartitionSlots()); + + DataPartition dataPartition = + getDataPartition(Collections.singletonList(dataPartitionQueryParam)); + Analysis analysis = new Analysis(); + analysis.setDataPartitionInfo(dataPartition); + + List insertTabletNodeList = insertTabletNode.splitByPartition(analysis); + + Assert.assertEquals(6, insertTabletNodeList.size()); + for (WritePlanNode insertNode : insertTabletNodeList) { + InsertTabletNode tabletNode = (InsertTabletNode) insertNode; + Assert.assertNotNull(tabletNode.getColumns()[0]); + Assert.assertNull(tabletNode.getColumns()[1]); + Assert.assertNull(tabletNode.getBitMaps()); + } + } + @Test public void testSplitRelationalInsertTablet() throws IllegalPathException { RelationalInsertTabletNode relationalInsertTabletNode = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java index e5ed818323bbd..082156e134efc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java @@ -22,12 +22,20 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Test; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -82,6 +90,65 @@ public void testInsertRowNode_markSameTwice_idempotent() throws IllegalPathExcep assertFalse(node.isMeasurementFailed(1)); } + @Test + public void testInsertRowNode_clearedMeasurementWithRetainedValue_isFailed() + throws IllegalPathException { + InsertRowNode node = buildInsertRowNode(new String[] {"s0", "s1"}); + node.getMeasurements()[0] = null; + + assertTrue(node.isMeasurementFailed(0)); + assertNull(node.composeTimeValuePair(0)); + } + + @Test + public void testInsertRowNode_retainedMeasurementWithNullValueDoesNotComposeLastCacheValue() + throws IllegalPathException { + InsertRowNode node = buildInsertRowNode(new String[] {"s0", "s1"}); + node.getValues()[0] = null; + + assertFalse(node.isMeasurementFailed(0)); + assertNull(node.composeTimeValuePair(0)); + } + + @Test + public void testInsertRowNode_nullMeasurements_nullSafe() throws IllegalPathException { + InsertRowNode node = buildInsertRowNode(new String[] {"s0"}); + node.setMeasurements(null); + + assertTrue(node.isMeasurementFailed(0)); + assertFalse(node.hasValidMeasurements()); + assertTrue(node.allMeasurementFailed()); + } + + @Test + public void testRelationalInsertRowNode_nonFieldColumnsDoNotComposeLastCacheValue() + throws IllegalPathException { + RelationalInsertRowNode node = buildRelationalInsertRowNode(); + + assertNull(node.composeTimeValuePair(0)); + assertNull(node.composeTimeValuePair(1)); + assertNotNull(node.composeTimeValuePair(2)); + } + + @Test + public void testRelationalInsertRowNode_deviceIDSkipsMissingTagValue() + throws IllegalPathException { + RelationalInsertRowNode node = buildRelationalInsertRowNodeWithTwoTags(new Object[] {"tag0"}); + + assertArrayEquals(new Object[] {"table1", "tag0"}, node.getDeviceID().getSegments()); + } + + @Test + public void testRelationalInsertRowsNode_deviceIDSkipsMissingTagValue() + throws IllegalPathException { + RelationalInsertRowNode rowNode = + buildRelationalInsertRowNodeWithTwoTags(new Object[] {"tag0"}); + RelationalInsertRowsNode rowsNode = new RelationalInsertRowsNode(new PlanNodeId("test")); + rowsNode.addOneInsertRowNode(rowNode, 0); + + assertArrayEquals(new Object[] {"table1", "tag0"}, rowsNode.getDeviceID(0).getSegments()); + } + // ----------------------------------------------------------------------- // InsertTabletNode // ----------------------------------------------------------------------- @@ -115,6 +182,85 @@ public void testInsertTabletNode_markLastFailed_lastReturnsTrue() throws Illegal assertTrue(node.isMeasurementFailed(1)); } + @Test + public void testInsertTabletNode_clearedMeasurementWithRetainedColumn_isFailed() + throws IllegalPathException { + InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"}); + node.getMeasurements()[0] = null; + + assertTrue(node.isMeasurementFailed(0)); + assertNull(node.composeLastTimeValuePair(0)); + } + + @Test + public void testInsertTabletNode_retainedMeasurementWithNullColumnDoesNotComposeLastCacheValue() + throws IllegalPathException { + InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"}); + node.getColumns()[0] = null; + + assertFalse(node.isMeasurementFailed(0)); + assertNull(node.composeLastTimeValuePair(0)); + } + + @Test + public void testInsertTabletNode_markFailedMeasurementHandlesShortColumns() + throws IllegalPathException { + InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"}); + node.setColumns(new Object[] {new int[] {1, 2}}); + + node.markFailedMeasurement(1); + + assertTrue(node.isMeasurementFailed(1)); + assertNull(node.getDataType(1)); + } + + @Test + public void testInsertTabletNode_getDataTypeReturnsNullForShortDataTypes() + throws IllegalPathException { + InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"}); + node.setDataTypes(new TSDataType[] {TSDataType.INT32}); + + assertNotNull(node.getDataType(0)); + assertNull(node.getDataType(1)); + } + + @Test + public void testInsertTabletNode_composeLastTimeValuePairHandlesShortBitMaps() + throws IllegalPathException { + InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"}); + node.setBitMaps(new BitMap[0]); + + assertNotNull(node.composeLastTimeValuePair(1)); + } + + @Test + public void testInsertTabletNode_equalsHandlesShortDataTypes() throws IllegalPathException { + InsertTabletNode left = buildInsertTabletNode(new String[] {"s0", "s1"}); + InsertTabletNode right = buildInsertTabletNode(new String[] {"s0", "s1"}); + left.setDataTypes(new TSDataType[] {TSDataType.INT32}); + right.setDataTypes(new TSDataType[] {TSDataType.INT32}); + + assertFalse(left.equals(right)); + } + + @Test + public void testRelationalInsertTabletNode_nonFieldColumnsDoNotComposeLastCacheValue() + throws IllegalPathException { + RelationalInsertTabletNode node = buildRelationalInsertTabletNode(); + + assertNull(node.composeLastTimeValuePair(0)); + assertNull(node.composeLastTimeValuePair(1)); + assertNotNull(node.composeLastTimeValuePair(2)); + } + + @Test + public void testRelationalInsertTabletNode_deviceIDSkipsMissingTagColumn() + throws IllegalPathException { + RelationalInsertTabletNode node = buildRelationalInsertTabletNodeWithTwoTags(); + + assertArrayEquals(new Object[] {"table1", "tag0"}, node.getDeviceID(0).getSegments()); + } + // ----------------------------------------------------------------------- // Helpers // ----------------------------------------------------------------------- @@ -144,6 +290,59 @@ private static InsertRowNode buildInsertRowNode(String[] measurementNames) return node; } + private static RelationalInsertRowNode buildRelationalInsertRowNode() + throws IllegalPathException { + String[] measurements = {"tag0", "attr0", "field0"}; + TSDataType[] dataTypes = {TSDataType.STRING, TSDataType.STRING, TSDataType.INT32}; + MeasurementSchema[] schemas = { + new MeasurementSchema("tag0", TSDataType.STRING), + new MeasurementSchema("attr0", TSDataType.STRING), + new MeasurementSchema("field0", TSDataType.INT32) + }; + Object[] values = { + new Binary("tag".getBytes(StandardCharsets.UTF_8)), + new Binary("attr".getBytes(StandardCharsets.UTF_8)), + 1 + }; + return new RelationalInsertRowNode( + new PlanNodeId("test"), + new PartialPath("table1", false), + true, + measurements, + dataTypes, + schemas, + 1L, + values, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.ATTRIBUTE, TsTableColumnCategory.FIELD + }); + } + + private static RelationalInsertRowNode buildRelationalInsertRowNodeWithTwoTags(Object[] values) + throws IllegalPathException { + String[] measurements = {"tag0", "tag1", "field0"}; + TSDataType[] dataTypes = {TSDataType.STRING, TSDataType.STRING, TSDataType.INT32}; + MeasurementSchema[] schemas = { + new MeasurementSchema("tag0", TSDataType.STRING), + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("field0", TSDataType.INT32) + }; + return new RelationalInsertRowNode( + new PlanNodeId("test"), + new PartialPath("table1", false), + true, + measurements, + dataTypes, + schemas, + 1L, + values, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD + }); + } + private static InsertTabletNode buildInsertTabletNode(String[] measurementNames) throws IllegalPathException { int n = measurementNames.length; @@ -171,4 +370,60 @@ private static InsertTabletNode buildInsertTabletNode(String[] measurementNames) rowCount); return node; } + + private static RelationalInsertTabletNode buildRelationalInsertTabletNode() + throws IllegalPathException { + String[] measurements = {"tag0", "attr0", "field0"}; + TSDataType[] dataTypes = {TSDataType.STRING, TSDataType.STRING, TSDataType.INT32}; + MeasurementSchema[] schemas = { + new MeasurementSchema("tag0", TSDataType.STRING), + new MeasurementSchema("attr0", TSDataType.STRING), + new MeasurementSchema("field0", TSDataType.INT32) + }; + Object[] columns = { + new Binary[] {new Binary("tag".getBytes(StandardCharsets.UTF_8))}, + new Binary[] {new Binary("attr".getBytes(StandardCharsets.UTF_8))}, + new int[] {1} + }; + return new RelationalInsertTabletNode( + new PlanNodeId("test"), + new PartialPath("table1", false), + true, + measurements, + dataTypes, + schemas, + new long[] {1L}, + null, + columns, + 1, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.ATTRIBUTE, TsTableColumnCategory.FIELD + }); + } + + private static RelationalInsertTabletNode buildRelationalInsertTabletNodeWithTwoTags() + throws IllegalPathException { + String[] measurements = {"tag0", "tag1", "field0"}; + TSDataType[] dataTypes = {TSDataType.STRING, TSDataType.STRING, TSDataType.INT32}; + MeasurementSchema[] schemas = { + new MeasurementSchema("tag0", TSDataType.STRING), + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("field0", TSDataType.INT32) + }; + Object[] columns = {new Binary[] {new Binary("tag0".getBytes(StandardCharsets.UTF_8))}}; + return new RelationalInsertTabletNode( + new PlanNodeId("test"), + new PartialPath("table1", false), + true, + measurements, + dataTypes, + schemas, + new long[] {1L}, + null, + columns, + 1, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD + }); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java new file mode 100644 index 0000000000000..b6b26cacc8ecc --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertStatementPartialInsertTest.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.crud; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.TypeFactory; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class InsertStatementPartialInsertTest { + + @Test + public void testInsertRowStatementGetPathsSkipsFailedMeasurements() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.markFailedMeasurement(0, new RuntimeException("failed")); + + Assert.assertEquals(1, statement.getPaths().size()); + Assert.assertEquals("root.sg.d1.s2", statement.getPaths().get(0).getFullPath()); + } + + @Test + public void testInsertTabletStatementGetPathsSkipsFailedMeasurements() throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.markFailedMeasurement(1, new RuntimeException("failed")); + + Assert.assertEquals(1, statement.getPaths().size()); + Assert.assertEquals("root.sg.d1.s1", statement.getPaths().get(0).getFullPath()); + } + + @Test + public void testInsertRowsOfOneDeviceStatementSkipsFailedMeasurements() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.markFailedMeasurement(0, new RuntimeException("failed")); + final InsertRowsOfOneDeviceStatement rowsOfOneDeviceStatement = + new InsertRowsOfOneDeviceStatement(); + rowsOfOneDeviceStatement.setInsertRowStatementList(Arrays.asList(statement)); + + Assert.assertArrayEquals(new String[] {"s2"}, rowsOfOneDeviceStatement.getMeasurements()); + Assert.assertEquals(1, rowsOfOneDeviceStatement.getPaths().size()); + Assert.assertEquals("root.sg.d1.s2", rowsOfOneDeviceStatement.getPaths().get(0).getFullPath()); + } + + @Test + public void testInsertRowStatementMarkFailedMeasurementHandlesMissingValue() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setValues(new Object[] {1}); + + statement.markFailedMeasurement(1, new RuntimeException("failed")); + + Assert.assertNull(statement.getMeasurements()[1]); + Assert.assertNull(statement.getDataTypes()[1]); + + statement.removeAllFailedMeasurementMarks(); + + Assert.assertEquals("s2", statement.getMeasurements()[1]); + Assert.assertEquals(TSDataType.INT64, statement.getDataTypes()[1]); + } + + @Test + public void testInsertTabletStatementMarkFailedMeasurementHandlesMissingColumn() + throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.setColumns(new Object[] {new int[] {1, 2}}); + + statement.markFailedMeasurement(1, new RuntimeException("failed")); + + Assert.assertNull(statement.getMeasurements()[1]); + Assert.assertNull(statement.getDataTypes()[1]); + + statement.removeAllFailedMeasurementMarks(); + + Assert.assertEquals("s2", statement.getMeasurements()[1]); + Assert.assertEquals(TSDataType.INT64, statement.getDataTypes()[1]); + } + + @Test + public void testHasValidMeasurementsIgnoresNonFieldColumns() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.ATTRIBUTE}); + + Assert.assertFalse(statement.hasValidMeasurements()); + + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD}); + + Assert.assertTrue(statement.hasValidMeasurements()); + } + + @Test + public void testInsertTabletStatementGetFirstValueOfIndexReturnsNullForNullColumn() + throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.getColumns()[0] = null; + + Assert.assertNull(statement.getFirstValueOfIndex(0)); + } + + @Test + public void testInsertTabletStatementTagAndAttributeIndicesSkipNullColumn() throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.ATTRIBUTE}); + statement.getColumns()[0] = null; + + Assert.assertTrue(statement.getTagColumnIndices().isEmpty()); + Assert.assertEquals(Arrays.asList(1), statement.getAttrColumnIndices()); + Assert.assertEquals(Arrays.asList("s2"), statement.getAttributeColumnNameList()); + } + + @Test + public void testInsertRowStatementTableDeviceIDHandlesMissingTagValue() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD, TsTableColumnCategory.TAG}); + statement.getTagColumnIndices(); + statement.setValues(new Object[] {1}); + + Assert.assertArrayEquals( + new Object[] {"root.sg.d1"}, statement.getTableDeviceID().getSegments()); + } + + @Test + public void testInsertTabletStatementTableDeviceIDUsesTagColumnBitmap() throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.setDataTypes(new TSDataType[] {TSDataType.INT32, TSDataType.STRING}); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD, TsTableColumnCategory.TAG}); + statement.setColumns( + new Object[] { + new int[] {1, 2}, + new Binary[] { + new Binary("tag1", StandardCharsets.UTF_8), new Binary("tag2", StandardCharsets.UTF_8) + } + }); + statement.setBitMaps(new BitMap[] {new BitMap(2, new byte[] {1}), new BitMap(2)}); + + Assert.assertArrayEquals( + new Object[] {"root.sg.d1", "tag1"}, statement.getTableDeviceID(0).getSegments()); + } + + @Test + public void testInsertTabletStatementTableDeviceIDHandlesMissingTagColumn() throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD, TsTableColumnCategory.TAG}); + statement.getTagColumnIndices(); + statement.setColumns(new Object[] {new int[] {1, 2}}); + + Assert.assertArrayEquals( + new Object[] {"root.sg.d1"}, statement.getTableDeviceID(0).getSegments()); + } + + @Test + public void testGetDataTypeReturnsNullForShortTypeArray() throws Exception { + final InsertRowStatement rowStatement = createInsertRowStatement(); + rowStatement.setDataTypes(new TSDataType[] {TSDataType.INT32}); + Assert.assertEquals(TSDataType.INT32, rowStatement.getDataType(0)); + Assert.assertNull(rowStatement.getDataType(1)); + + final InsertTabletStatement tabletStatement = createInsertTabletStatement(); + tabletStatement.setDataTypes(new TSDataType[] {TSDataType.INT32}); + Assert.assertEquals(TSDataType.INT32, tabletStatement.getDataType(0)); + Assert.assertNull(tabletStatement.getDataType(1)); + } + + @Test + public void testBaseSettersExpandShortArrays() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setDataTypes(new TSDataType[] {TSDataType.INT32}); + statement.setMeasurementSchemas( + new MeasurementSchema[] {new MeasurementSchema("s1", TSDataType.INT32)}); + statement.setColumnCategories(new TsTableColumnCategory[] {TsTableColumnCategory.FIELD}); + + statement.setDataType(TSDataType.INT64, 1); + statement.setMeasurementSchema(new MeasurementSchema("s2", TSDataType.INT64), 1); + statement.setColumnCategory(TsTableColumnCategory.TAG, 1); + + Assert.assertEquals(TSDataType.INT64, statement.getDataType(1)); + Assert.assertEquals("s2", statement.getMeasurementSchemas()[1].getMeasurementName()); + Assert.assertEquals(TsTableColumnCategory.TAG, statement.getColumnCategories()[1]); + } + + @Test + public void testInsertColumnWithNullDataTypesKeepsArraysAligned() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setDataTypes(null); + + final ColumnSchema columnSchema = + new ColumnSchema( + "tag1", TypeFactory.getType(TSDataType.STRING), false, TsTableColumnCategory.TAG); + statement.insertColumn(1, columnSchema); + + Assert.assertEquals(3, statement.getMeasurements().length); + Assert.assertEquals(3, statement.getDataTypes().length); + Assert.assertEquals(3, statement.getValues().length); + Assert.assertNull(statement.getDataType(0)); + Assert.assertEquals(TSDataType.STRING, statement.getDataType(1)); + Assert.assertNull(statement.getDataType(2)); + } + + @Test + public void testRebuildArraysAfterExpansionHandlesShortValueArrays() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setDataTypes(new TSDataType[] {TSDataType.INT32}); + statement.setValues(new Object[] {1}); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD}); + + statement.rebuildArraysAfterExpansion(new int[] {-1, 0, 1}, new String[] {"tag1", "s1", "s2"}); + + Assert.assertArrayEquals(new String[] {"tag1", "s1", "s2"}, statement.getMeasurements()); + Assert.assertArrayEquals(new Object[] {null, 1, null}, statement.getValues()); + Assert.assertEquals(TSDataType.STRING, statement.getDataType(0)); + Assert.assertEquals(TSDataType.INT32, statement.getDataType(1)); + Assert.assertNull(statement.getDataType(2)); + Assert.assertEquals(TsTableColumnCategory.TAG, statement.getColumnCategories()[0]); + } + + @Test + public void testInsertRowStatementSwapColumnInvalidatesTableDeviceId() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setColumnCategories( + new TsTableColumnCategory[] {TsTableColumnCategory.TAG, TsTableColumnCategory.TAG}); + statement.setValues(new Object[] {"tag1", "tag2"}); + + Assert.assertArrayEquals( + new Object[] {"root.sg.d1", "tag1", "tag2"}, statement.getTableDeviceID().getSegments()); + + statement.swapColumn(0, 1); + + Assert.assertArrayEquals( + new Object[] {"root.sg.d1", "tag2", "tag1"}, statement.getTableDeviceID().getSegments()); + } + + @Test + public void testDeviceMeasurementMapSkipsMissingRowValues() throws Exception { + final InsertRowStatement statement = createInsertRowStatement(); + statement.setValues(new Object[] {1}); + + final Map>> result = + statement.getMapFromDeviceToMeasurementAndIndex(); + + Assert.assertEquals(1, result.get(statement.getDevicePath()).size()); + Assert.assertEquals("s1", result.get(statement.getDevicePath()).get(0).left); + } + + @Test + public void testDeviceMeasurementMapSkipsMissingTabletColumns() throws Exception { + final InsertTabletStatement statement = createInsertTabletStatement(); + statement.setColumns(new Object[] {new int[] {1, 2}}); + + final Map>> result = + statement.getMapFromDeviceToMeasurementAndIndex(); + + Assert.assertEquals(1, result.get(statement.getDevicePath()).size()); + Assert.assertEquals("s1", result.get(statement.getDevicePath()).get(0).left); + } + + private static InsertRowStatement createInsertRowStatement() throws Exception { + final InsertRowStatement statement = new InsertRowStatement(); + statement.setDevicePath(new PartialPath("root.sg.d1")); + statement.setMeasurements(new String[] {"s1", "s2"}); + statement.setDataTypes(new TSDataType[] {TSDataType.INT32, TSDataType.INT64}); + statement.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }); + statement.setValues(new Object[] {1, 2L}); + statement.setTime(1L); + return statement; + } + + private static InsertTabletStatement createInsertTabletStatement() throws Exception { + final InsertTabletStatement statement = new InsertTabletStatement(); + statement.setDevicePath(new PartialPath("root.sg.d1")); + statement.setMeasurements(new String[] {"s1", "s2"}); + statement.setDataTypes(new TSDataType[] {TSDataType.INT32, TSDataType.INT64}); + statement.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }); + statement.setColumns(new Object[] {new int[] {1, 2}, new long[] {3L, 4L}}); + statement.setTimes(new long[] {1L, 2L}); + statement.setRowCount(2); + return statement; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 3aced7e9fe11f..f5ece33a7787e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -456,12 +456,12 @@ public void testIoTDBTabletWriteAndSyncClose() new QueryId("test_write").genPlanNodeId(), new PartialPath("root.vehicle.d0"), false, - measurements, - dataTypes, - measurementSchemas, + measurements.clone(), + dataTypes.clone(), + measurementSchemas.clone(), times, null, - columns, + columns.clone(), times.length); dataRegion.insertTablet(insertTabletNode2); @@ -667,6 +667,8 @@ public void testAllMeasurementsFailedTabletWriteAndSyncClose() null, columns, times.length); + insertTabletNode1.markFailedMeasurement(0); + insertTabletNode1.markFailedMeasurement(1); insertTabletNode1.setFailedMeasurementNumber(2); dataRegion.insertTablet(insertTabletNode1); @@ -683,13 +685,15 @@ public void testAllMeasurementsFailedTabletWriteAndSyncClose() new QueryId("test_write").genPlanNodeId(), new PartialPath("root.vehicle.d0"), false, - measurements, - dataTypes, - measurementSchemas, + measurements.clone(), + dataTypes.clone(), + measurementSchemas.clone(), times, null, - columns, + columns.clone(), times.length); + insertTabletNode2.markFailedMeasurement(0); + insertTabletNode2.markFailedMeasurement(1); insertTabletNode2.setFailedMeasurementNumber(2); dataRegion.insertTablet(insertTabletNode2); @@ -743,6 +747,7 @@ public void testAllMeasurementsFailedRecordSeqAndUnSeqSyncClose() TSRecord record = new TSRecord(deviceId, j); record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); InsertRowNode rowNode = buildInsertRowNodeByTSRecord(record); + rowNode.markFailedMeasurement(0); rowNode.setFailedMeasurementNumber(1); dataRegion.insert(rowNode); dataRegion.syncCloseAllWorkingTsFileProcessors(); @@ -752,6 +757,7 @@ public void testAllMeasurementsFailedRecordSeqAndUnSeqSyncClose() TSRecord record = new TSRecord(deviceId, j); record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); InsertRowNode rowNode = buildInsertRowNodeByTSRecord(record); + rowNode.markFailedMeasurement(0); rowNode.setFailedMeasurementNumber(1); dataRegion.insert(rowNode); dataRegion.syncCloseAllWorkingTsFileProcessors(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java index 2f4a85d5840a2..bdad14c20e33f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -112,6 +113,20 @@ public void testInsertAlignedRow_oneFailedMeasurement_pointsInsertedNotNegative( assertEquals(1, memTable.getTotalPointsNum()); } + @Test + public void testInsertAlignedRow_markedFailedMeasurementOnly_pointsInsertedMatchesWrittenPoints() + throws IllegalPathException { + InsertRowNode node = + buildAlignedInsertRowNode( + new String[] {"s0", "s1"}, new Object[] {1, 2}, -1 /* no failure */); + node.markFailedMeasurement(0); + + int points = memTable.insertAlignedRow(node); + + assertEquals(1, points); + assertEquals(1, memTable.getTotalPointsNum()); + } + /** All measurements fail → insertAlignedRow returns 0 early (schemaList is empty). */ @Test public void testInsertAlignedRow_allMeasurementsFailed_pointsInsertedIsZero() @@ -132,6 +147,67 @@ public void testInsertAlignedRow_allMeasurementsFailed_pointsInsertedIsZero() assertEquals(0, memTable.getTotalPointsNum()); } + @Test + public void testInsertAlignedRow_tableNonFieldAndFailedMeasurementsNotCountedAsSeries() + throws IllegalPathException { + InsertRowNode node = + buildAlignedInsertRowNode( + new String[] {"tag1", "attr1", "s0", "s1"}, + new Object[] {1, 2, 3, 4}, + -1 /* no failure */); + node.setColumnCategories( + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + node.markFailedMeasurement(3); + + int points = memTable.insertAlignedRow(node); + + assertEquals(1, points); + assertEquals(1, memTable.getTotalPointsNum()); + assertEquals(1, memTable.getSeriesNumber()); + } + + @Test + public void testInsertRow_tableNonFieldAndFailedMeasurementsNotCountedAsSeries() + throws IllegalPathException { + InsertRowNode node = + new InsertRowNode( + new PlanNodeId("test"), + new PartialPath("root.sg.d1"), + false, + new String[] {"tag1", "attr1", "s0", "s1"}, + new TSDataType[] { + TSDataType.INT32, TSDataType.INT32, TSDataType.INT32, TSDataType.INT32 + }, + new MeasurementSchema[] { + new MeasurementSchema("tag1", TSDataType.INT32), + new MeasurementSchema("attr1", TSDataType.INT32), + new MeasurementSchema("s0", TSDataType.INT32), + new MeasurementSchema("s1", TSDataType.INT32) + }, + 1L, + new Object[] {1, 2, 3, 4}, + false); + node.setColumnCategories( + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + node.markFailedMeasurement(3); + + int points = memTable.insert(node); + + assertEquals(1, points); + assertEquals(1, memTable.getTotalPointsNum()); + assertEquals(1, memTable.getSeriesNumber()); + } + // ========================================================================= // insertTablet – failed measurement must be skipped in null-point counting // ========================================================================= @@ -179,6 +255,43 @@ public void testInsertTablet_oneFailedMeasurement_withBitmap_pointsInsertedNotNe assertEquals(3, memTable.getTotalPointsNum()); } + @Test + public void testInsertTablet_markedFailedMeasurementOnly_pointsInsertedMatchesWrittenPoints() + throws IllegalPathException, WriteProcessException { + int rowCount = 3; + InsertTabletNode node = + buildInsertTabletNode(new String[] {"s0", "s1"}, rowCount, null, -1 /* no failure */); + node.markFailedMeasurement(0); + + int points = memTable.insertTablet(node, 0, rowCount); + + assertEquals(3, points); + assertEquals(3, memTable.getTotalPointsNum()); + } + + @Test + public void testInsertTablet_tableNonFieldAndFailedMeasurementsNotCountedAsSeries() + throws IllegalPathException, WriteProcessException { + int rowCount = 3; + InsertTabletNode node = + buildInsertTabletNode( + new String[] {"tag1", "attr1", "s0", "s1"}, rowCount, null, -1 /* no failure */); + node.setColumnCategories( + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + node.markFailedMeasurement(3); + + int points = memTable.insertTablet(node, 0, rowCount); + + assertEquals(rowCount, points); + assertEquals(rowCount, memTable.getTotalPointsNum()); + assertEquals(1, memTable.getSeriesNumber()); + } + /** All measurements fail → pointsInserted == 0. formula: (2-2)*3 - 0 = 0 */ @Test public void testInsertTablet_allMeasurementsFailed_pointsInsertedIsZero() diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java index 8e03776231eae..5595390752a39 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java @@ -283,6 +283,29 @@ public void testAlignedSeries() throws IOException, QueryProcessException, Metad } } + @Test + public void testNonAlignedMemChunkGroupSerializedSizeWithNonAsciiMeasurement() + throws IOException { + String measurement = "\u6e29\u5ea6"; + WritableMemChunk series = + new WritableMemChunk( + new MeasurementSchema(measurement, TSDataType.INT32, TSEncoding.PLAIN)); + series.writeNonAlignedPoint(1, 1); + + WritableMemChunkGroup group = new WritableMemChunkGroup(); + group.getMemChunkMap().put(measurement, series); + + WALByteBufferForTest walBuffer = + new WALByteBufferForTest(ByteBuffer.allocate(group.serializedSize())); + group.serializeToWAL(walBuffer); + Assert.assertEquals(group.serializedSize(), walBuffer.getBuffer().position()); + + DataInputStream inputStream = + new DataInputStream(new ByteArrayInputStream(walBuffer.getBuffer().array())); + WritableMemChunkGroup deserialized = WritableMemChunkGroup.deserialize(inputStream); + Assert.assertTrue(deserialized.getMemChunkMap().containsKey(measurement)); + } + private WritableMemChunk createWritableMemChunkFromBytes(WritableMemChunk series) throws IOException { int serializedSize = series.serializedSize(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java index 3cc50496fb64f..2609cb1acb5e6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.DataRegionException; @@ -36,6 +37,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.storageengine.dataregion.DataRegionInfo; import org.apache.iotdb.db.storageengine.dataregion.DataRegionTest; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -56,6 +58,7 @@ import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; import org.apache.tsfile.read.reader.IPointReader; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -68,6 +71,7 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -898,6 +902,174 @@ record = new TSRecord("root.vehicle.d2", 102); Assert.assertEquals(memTable1.memSize(), memTable2.memSize()); } + @Test + public void testAlignedRamCostIgnoresRelationalNonFieldAndNullFieldColumns() + throws IllegalPathException, WriteProcessException, IOException { + TsFileProcessor relationalProcessor = + new TsFileProcessor( + storageGroup, + SystemFileFactory.INSTANCE.getFile(filePath), + sgInfo, + this::closeTsFileProcessor, + (tsFileProcessor, updateMap, systemFlushTime) -> {}, + true); + TsFileProcessorInfo relationalInfo = new TsFileProcessorInfo(sgInfo); + relationalProcessor.setTsFileProcessorInfo(relationalInfo); + this.sgInfo.initTsFileProcessorInfo(relationalProcessor); + SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, relationalProcessor); + + RelationalInsertRowNode relationalNode = + new RelationalInsertRowNode( + new PlanNodeId("relational"), + new PartialPath("table1", false), + true, + new String[] {"tag1", "attr1", "s1", "s2"}, + new TSDataType[] {TSDataType.TEXT, TSDataType.TEXT, TSDataType.INT32, TSDataType.INT64}, + new MeasurementSchema[] { + new MeasurementSchema("tag1", TSDataType.TEXT), + new MeasurementSchema("attr1", TSDataType.TEXT), + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }, + 1L, + new Object[] { + new Binary("tag-value".getBytes(StandardCharsets.UTF_8)), + new Binary("attr-value".getBytes(StandardCharsets.UTF_8)), + 1, + null + }, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + relationalProcessor.insert(relationalNode, new long[5]); + + TsFileProcessor fieldOnlyProcessor = + new TsFileProcessor( + storageGroup, + SystemFileFactory.INSTANCE.getFile(filePath), + sgInfo, + this::closeTsFileProcessor, + (tsFileProcessor, updateMap, systemFlushTime) -> {}, + true); + TsFileProcessorInfo fieldOnlyInfo = new TsFileProcessorInfo(sgInfo); + fieldOnlyProcessor.setTsFileProcessorInfo(fieldOnlyInfo); + this.sgInfo.initTsFileProcessorInfo(fieldOnlyProcessor); + SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, fieldOnlyProcessor); + + InsertRowNode fieldOnlyNode = + new InsertRowNode( + new PlanNodeId("field-only"), + new PartialPath(deviceId), + true, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32, TSDataType.INT64}, + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }, + 1L, + new Object[] {1, null}, + false); + fieldOnlyProcessor.insert(fieldOnlyNode, new long[5]); + + IMemTable relationalMemTable = relationalProcessor.getWorkMemTable(); + IMemTable fieldOnlyMemTable = fieldOnlyProcessor.getWorkMemTable(); + Assert.assertEquals( + fieldOnlyMemTable.getTVListsRamCost(), relationalMemTable.getTVListsRamCost()); + Assert.assertEquals(fieldOnlyInfo.getMemCost(), relationalInfo.getMemCost()); + Assert.assertEquals(fieldOnlyMemTable.memSize(), relationalMemTable.memSize()); + Assert.assertEquals(1, relationalMemTable.getTotalPointsNum()); + Assert.assertEquals(1, relationalMemTable.getSeriesNumber()); + } + + @Test + public void testNonAlignedRamCostIgnoresRelationalNonFieldAndNullFieldColumns() + throws IllegalPathException, WriteProcessException, IOException { + TsFileProcessor relationalProcessor = + new TsFileProcessor( + storageGroup, + SystemFileFactory.INSTANCE.getFile(filePath), + sgInfo, + this::closeTsFileProcessor, + (tsFileProcessor, updateMap, systemFlushTime) -> {}, + true); + TsFileProcessorInfo relationalInfo = new TsFileProcessorInfo(sgInfo); + relationalProcessor.setTsFileProcessorInfo(relationalInfo); + this.sgInfo.initTsFileProcessorInfo(relationalProcessor); + SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, relationalProcessor); + + RelationalInsertRowNode relationalNode = + new RelationalInsertRowNode( + new PlanNodeId("relational"), + new PartialPath("table1", false), + false, + new String[] {"tag1", "attr1", "s1", "s2"}, + new TSDataType[] {TSDataType.TEXT, TSDataType.TEXT, TSDataType.INT32, TSDataType.INT64}, + new MeasurementSchema[] { + new MeasurementSchema("tag1", TSDataType.TEXT), + new MeasurementSchema("attr1", TSDataType.TEXT), + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }, + 1L, + new Object[] { + new Binary("tag-value".getBytes(StandardCharsets.UTF_8)), + new Binary("attr-value".getBytes(StandardCharsets.UTF_8)), + 1, + null + }, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + relationalProcessor.insert(relationalNode, new long[5]); + + TsFileProcessor fieldOnlyProcessor = + new TsFileProcessor( + storageGroup, + SystemFileFactory.INSTANCE.getFile(filePath), + sgInfo, + this::closeTsFileProcessor, + (tsFileProcessor, updateMap, systemFlushTime) -> {}, + true); + TsFileProcessorInfo fieldOnlyInfo = new TsFileProcessorInfo(sgInfo); + fieldOnlyProcessor.setTsFileProcessorInfo(fieldOnlyInfo); + this.sgInfo.initTsFileProcessorInfo(fieldOnlyProcessor); + SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, fieldOnlyProcessor); + + InsertRowNode fieldOnlyNode = + new InsertRowNode( + new PlanNodeId("field-only"), + new PartialPath(deviceId), + false, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32, TSDataType.INT64}, + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }, + 1L, + new Object[] {1, null}, + false); + fieldOnlyProcessor.insert(fieldOnlyNode, new long[5]); + + IMemTable relationalMemTable = relationalProcessor.getWorkMemTable(); + IMemTable fieldOnlyMemTable = fieldOnlyProcessor.getWorkMemTable(); + Assert.assertEquals( + fieldOnlyMemTable.getTVListsRamCost(), relationalMemTable.getTVListsRamCost()); + Assert.assertEquals(fieldOnlyInfo.getMemCost(), relationalInfo.getMemCost()); + Assert.assertEquals(fieldOnlyMemTable.memSize(), relationalMemTable.memSize()); + Assert.assertEquals(1, relationalMemTable.getTotalPointsNum()); + Assert.assertEquals(1, relationalMemTable.getSeriesNumber()); + } + @Test public void testRamCostInsertSameDataBy2Ways() throws MetadataException, WriteProcessException, IOException { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java index 4ae6268ebce8e..5cc56bb016222 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverterTest.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils; @@ -34,6 +35,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; @@ -104,6 +106,33 @@ public void testConvertRelationalInsertTabletNodeWithSingleMatchedColumn() { Assert.assertSame(node.getColumns()[2], tablet.getValues()[1]); } + @Test + public void testConvertRelationalInsertTabletNodeSkipsNullMatchedFieldColumn() { + final ConsensusLogToTabletConverter converter = createConverter("m1"); + + final RelationalInsertTabletNode node = StatementTestUtils.genInsertTabletNode(3, 10); + node.getColumns()[2] = null; + + Assert.assertTrue(converter.convert(node).isEmpty()); + } + + @Test + public void testConvertRelationalInsertTabletNodeSkipsNullTagColumn() { + final ConsensusLogToTabletConverter converter = createConverter("m1"); + + final RelationalInsertTabletNode node = StatementTestUtils.genInsertTabletNode(3, 10); + node.getColumns()[0] = null; + final List tablets = converter.convert(node); + + Assert.assertEquals(1, tablets.size()); + final Tablet tablet = tablets.get(0); + Assert.assertEquals(1, tablet.getSchemas().size()); + Assert.assertEquals("m1", tablet.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals(ColumnCategory.FIELD, tablet.getColumnTypes().get(0)); + Assert.assertArrayEquals( + new double[] {10.0, 11.0, 12.0}, (double[]) tablet.getValues()[0], 0.0); + } + @Test public void testConvertRelationalInsertRowNodeKeepsTagColumnsForMatchedField() { final ConsensusLogToTabletConverter converter = createConverter("m1"); @@ -177,6 +206,35 @@ public void testConvertInsertRowsNodeKeepsOrderWhenGroupingTreeRows() Assert.assertEquals(1, tablets.get(2).getRowSize()); } + @Test + public void testConvertTreeInsertTabletNodeSkipsNullColumn() throws IllegalPathException { + final ConsensusLogToTabletConverter converter = createTreeConverter(); + final InsertTabletNode node = + new InsertTabletNode( + new PlanNodeId("tablet"), + new PartialPath("root.sg.d1"), + false, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32, TSDataType.DOUBLE}, + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.DOUBLE) + }, + new long[] {1L, 2L}, + null, + new Object[] {new int[] {1, 2}, null}, + 2); + + final List tablets = converter.convert(node); + + Assert.assertEquals(1, tablets.size()); + final Tablet tablet = tablets.get(0); + Assert.assertEquals("root.sg.d1", tablet.getDeviceId()); + Assert.assertEquals(1, tablet.getSchemas().size()); + Assert.assertEquals("s1", tablet.getSchemas().get(0).getMeasurementName()); + Assert.assertArrayEquals(new int[] {1, 2}, (int[]) tablet.getValues()[0]); + } + private static ConsensusLogToTabletConverter createConverter(final String columnPattern) { return new ConsensusLogToTabletConverter( null, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java index 7af2a9775fb0f..9bdbba74c09f8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/MemUtilsTest.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.tsfile.common.conf.TSFileConfig; @@ -91,6 +92,74 @@ public void getRecordSizeWithInsertAlignedRowNodeTest() { Assert.assertEquals(sizeSum, MemUtils.getAlignedRowRecordSize(dataTypes, row, null)); } + @Test + public void getAlignedRowRecordSizeWithSkippedSlotsTest() { + Object[] row = {null, 1, 2L}; + List dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT32); + + int sizeSum = 8 + 4; + sizeSum += TSDataType.INT32.getDataTypeSize(); + + Assert.assertEquals( + sizeSum, + MemUtils.getAlignedRowRecordSize( + dataTypes, + row, + new TsTableColumnCategory[] { + TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD, TsTableColumnCategory.TAG + })); + } + + @Test + public void getRowRecordSizeWithSkippedSlotsTest() { + Object[] row = {1, 2, 3L, null}; + List dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT64); + + int sizeSum = 8 + TSDataType.INT64.getDataTypeSize(); + + Assert.assertEquals( + sizeSum, + MemUtils.getRowRecordSize( + dataTypes, + row, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.ATTRIBUTE, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + })); + } + + @Test + public void getRowRecordSizeWithShortColumnCategoriesTest() { + Object[] row = {1, 2L}; + List dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT32); + + int sizeSum = 8 + TSDataType.INT32.getDataTypeSize(); + + Assert.assertEquals( + sizeSum, + MemUtils.getRowRecordSize( + dataTypes, row, new TsTableColumnCategory[] {TsTableColumnCategory.FIELD})); + } + + @Test + public void getAlignedRowRecordSizeWithShortColumnCategoriesTest() { + Object[] row = {1, 2L}; + List dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT32); + + int sizeSum = 8 + 4 + TSDataType.INT32.getDataTypeSize(); + + Assert.assertEquals( + sizeSum, + MemUtils.getAlignedRowRecordSize( + dataTypes, row, new TsTableColumnCategory[] {TsTableColumnCategory.FIELD})); + } + @Test public void getRecordSizeWithInsertTableNodeTest() throws IllegalPathException { PartialPath device = new PartialPath("root.sg.d1"); @@ -125,6 +194,14 @@ public void getRecordSizeWithInsertTableNodeTest() throws IllegalPathException { null, columns, 1); + insertNode.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64), + new MeasurementSchema("s3", TSDataType.FLOAT), + new MeasurementSchema("s4", TSDataType.DOUBLE), + new MeasurementSchema("s5", TSDataType.TEXT) + }); Assert.assertEquals(sizeSum, MemUtils.getTabletSize(insertNode, 0, 1)); } @@ -175,6 +252,56 @@ public void getRecordSizeWithInsertAlignedTableNodeTest() throws IllegalPathExce Assert.assertEquals(sizeSum, MemUtils.getAlignedTabletSize(insertNode, 0, 1, null)); } + @Test + public void getAlignedTabletSizeWithMarkedFailedMeasurementTest() throws IllegalPathException { + PartialPath device = new PartialPath("root.sg.d1"); + String[] measurements = {"s1", "s2"}; + Object[] columns = {new int[] {1, 2}, new long[] {3, 4}}; + TSDataType[] dataTypes = {TSDataType.INT32, TSDataType.INT64}; + InsertTabletNode insertNode = + new InsertTabletNode( + new PlanNodeId(""), + device, + true, + measurements, + dataTypes, + new long[] {1, 2}, + null, + columns, + 2); + insertNode.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT64) + }); + insertNode.markFailedMeasurement(0); + + long sizeSum = 2L * TSDataType.INT64.getDataTypeSize(); + sizeSum += 2L * (8L + 4L); + Assert.assertEquals(sizeSum, MemUtils.getAlignedTabletSize(insertNode, 0, 2, null)); + } + + @Test + public void getTabletSizeWithShortValueArraysTest() throws IllegalPathException { + PartialPath device = new PartialPath("root.sg.d1"); + InsertTabletNode insertNode = + new InsertTabletNode( + new PlanNodeId(""), + device, + false, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32}, + new long[] {1}, + null, + new Object[] {new int[] {1}}, + 1); + insertNode.setMeasurementSchemas( + new MeasurementSchema[] {new MeasurementSchema("s1", TSDataType.INT32)}); + + long sizeSum = 8 + TSDataType.INT32.getDataTypeSize(); + Assert.assertEquals(sizeSum, MemUtils.getTabletSize(insertNode, 0, 1)); + } + /** This method tests MemUtils.getStringMem() and MemUtils.getDataPointMem() */ @Test public void getMemSizeTest() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java index eb08a20f283a9..271f84ac65283 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfo.java @@ -135,7 +135,7 @@ public int getMeasurementCount() { } public TSDataType getType(int index) { - if (dataTypes == null) { + if (dataTypes == null || index < 0 || index >= dataTypes.length) { return null; } return dataTypes[index]; @@ -170,6 +170,13 @@ public TSDataType getTypeForFirstValue(int index) { *

The delegate is optional. If absent, this method is a no-op. */ public void toLowerCase() { + if (measurements != null) { + for (int i = 0; i < measurements.length; i++) { + if (measurements[i] != null) { + measurements[i] = measurements[i].toLowerCase(); + } + } + } if (toLowerCaseAction != null) { toLowerCaseAction.run(); } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java new file mode 100644 index 0000000000000..0645ef3bab1d1 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/schema/table/InsertNodeMeasurementInfoTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.table; + +import org.apache.tsfile.enums.TSDataType; +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class InsertNodeMeasurementInfoTest { + + @Test + public void testToLowerCaseUpdatesLocalMeasurementsAndDelegate() { + final String[] measurementInfoMeasurements = new String[] {"TAG1", null, "S1"}; + final String[] statementMeasurements = new String[] {"TAG1", "ATTR1", "S1"}; + final AtomicBoolean delegateCalled = new AtomicBoolean(false); + + final InsertNodeMeasurementInfo measurementInfo = + new InsertNodeMeasurementInfo( + "TABLE1", + null, + measurementInfoMeasurements, + null, + index -> null, + () -> { + delegateCalled.set(true); + for (int i = 0; i < statementMeasurements.length; i++) { + statementMeasurements[i] = statementMeasurements[i].toLowerCase(); + } + }, + null, + null, + null, + null); + + measurementInfo.toLowerCase(); + + assertArrayEquals(new String[] {"tag1", null, "s1"}, measurementInfoMeasurements); + assertArrayEquals(new String[] {"tag1", "attr1", "s1"}, statementMeasurements); + assertTrue(delegateCalled.get()); + } + + @Test + public void testGetTypeReturnsNullForShortDataTypes() { + final InsertNodeMeasurementInfo measurementInfo = + new InsertNodeMeasurementInfo( + "table1", + null, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32}, + index -> null, + null, + null, + null, + null, + null); + + assertEquals(TSDataType.INT32, measurementInfo.getType(0)); + assertNull(measurementInfo.getType(1)); + } +}