From bd4a22b51c422603d0d7db182fb2610167ba8002 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 3 Feb 2026 18:22:00 +0800 Subject: [PATCH 1/5] Support same measurement name and type with different encoding and compression This PR allows measurements with the same name and type to have different encoding and compression types during TsFile loading. Previously, duplicate measurements would throw an exception. Now, if measurements have the same name and type but different encoding/compression, they will be deduplicated by keeping only one schema entry. Changes: - Add updateDevice2TimeSeries method in LoadTsFileTreeSchemaCache to support updating the device-to-timeseries mapping - Modify makeSureNoDuplicatedMeasurementsInDevices in TreeSchemaAutoCreatorAndVerifier to allow same measurement with different encoding/compression by deduplicating instead of throwing an exception - Only throw exception when duplicate measurements have different data types --- .../load/LoadTsFileTreeSchemaCache.java | 4 +++ .../TreeSchemaAutoCreatorAndVerifier.java | 34 ++++++++++++++++--- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java index d0335d1b3bd64..115c4f83765fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java @@ -229,6 +229,10 @@ public boolean shouldFlushAlignedCache() { return tsFileDevice2IsAlignedMemoryUsageSizeInBytes >= FLUSH_ALIGNED_CACHE_MEMORY_SIZE_IN_BYTES; } + public void updateDevice2TimeSeries(Map> newDevice2TimeSeries) { + currentBatchDevice2TimeSeriesSchemas = newDevice2TimeSeries; + } + public void clearTimeSeries() { currentBatchDevice2TimeSeriesSchemas.clear(); block.reduceMemoryUsage(batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java index c174bd7942b81..2c49073d18caa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java @@ -252,18 +252,44 @@ private void handleException(Exception e, String statementString) throws Semanti } private void makeSureNoDuplicatedMeasurementsInDevices() throws LoadAnalyzeException { + boolean hasDuplicates = false; + final Map> deduplicatedDevice2TimeSeries = new HashMap<>(); + for (final Map.Entry> entry : schemaCache.getDevice2TimeSeries().entrySet()) { final IDeviceID device = entry.getKey(); final Map measurement2Schema = new HashMap<>(); + boolean deviceHasDuplicates =false; + for (final MeasurementSchema timeseriesSchema : entry.getValue()) { final String measurement = timeseriesSchema.getMeasurementName(); - if (measurement2Schema.containsKey(measurement)) { - throw new LoadAnalyzeException( - String.format("Duplicated measurements %s in device %s.", measurement, device)); + final MeasurementSchema existingSchema = measurement2Schema.get(measurement); + + if (existingSchema != null) { + if (existingSchema.getType() != timeseriesSchema.getType()) { + throw new LoadAnalyzeException( + String.format("Duplicated measurements %s in device %s.", measurement, device)); + } + deviceHasDuplicates = true; + hasDuplicates = true; + } else { + measurement2Schema.put(measurement, timeseriesSchema); + } + } + + if (deviceHasDuplicates) { + deduplicatedDevice2TimeSeries.put(device, new HashSet<>(measurement2Schema.values())); + } + } + + if (hasDuplicates) { + for (final Map.Entry> entry : + schemaCache.getDevice2TimeSeries().entrySet()) { + if (!deduplicatedDevice2TimeSeries.containsKey(entry.getKey())) { + deduplicatedDevice2TimeSeries.put(entry.getKey(), entry.getValue()); } - measurement2Schema.put(measurement, timeseriesSchema); } + schemaCache.updateDevice2TimeSeries(deduplicatedDevice2TimeSeries); } } From b431844f8dfa17a046dbce4e566686e0c937b1d4 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 3 Feb 2026 18:25:51 +0800 Subject: [PATCH 2/5] fix --- .../plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java index 2c49073d18caa..e0ae4a6857a85 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java @@ -259,7 +259,7 @@ private void makeSureNoDuplicatedMeasurementsInDevices() throws LoadAnalyzeExcep schemaCache.getDevice2TimeSeries().entrySet()) { final IDeviceID device = entry.getKey(); final Map measurement2Schema = new HashMap<>(); - boolean deviceHasDuplicates =false; + boolean deviceHasDuplicates = false; for (final MeasurementSchema timeseriesSchema : entry.getValue()) { final String measurement = timeseriesSchema.getMeasurementName(); From 260d91ef606491392a8f23693d51c5e4625578a3 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 3 Feb 2026 18:34:21 +0800 Subject: [PATCH 3/5] fix --- .../plan/analyze/load/LoadTsFileTreeSchemaCache.java | 4 ---- .../analyze/load/TreeSchemaAutoCreatorAndVerifier.java | 8 +++----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java index 115c4f83765fa..d0335d1b3bd64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java @@ -229,10 +229,6 @@ public boolean shouldFlushAlignedCache() { return tsFileDevice2IsAlignedMemoryUsageSizeInBytes >= FLUSH_ALIGNED_CACHE_MEMORY_SIZE_IN_BYTES; } - public void updateDevice2TimeSeries(Map> newDevice2TimeSeries) { - currentBatchDevice2TimeSeriesSchemas = newDevice2TimeSeries; - } - public void clearTimeSeries() { currentBatchDevice2TimeSeriesSchemas.clear(); block.reduceMemoryUsage(batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java index e0ae4a6857a85..6bc68d97d4e7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java @@ -283,13 +283,11 @@ private void makeSureNoDuplicatedMeasurementsInDevices() throws LoadAnalyzeExcep } if (hasDuplicates) { + Map> device2TimeSeries = schemaCache.getDevice2TimeSeries(); for (final Map.Entry> entry : - schemaCache.getDevice2TimeSeries().entrySet()) { - if (!deduplicatedDevice2TimeSeries.containsKey(entry.getKey())) { - deduplicatedDevice2TimeSeries.put(entry.getKey(), entry.getValue()); - } + deduplicatedDevice2TimeSeries.entrySet()) { + device2TimeSeries.put(entry.getKey(), new HashSet<>(entry.getValue())); } - schemaCache.updateDevice2TimeSeries(deduplicatedDevice2TimeSeries); } } From 919f7ffab75182aa93b13dc9a6c3077c93a50599 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 3 Feb 2026 19:03:52 +0800 Subject: [PATCH 4/5] add IT --- .../apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index 928c7875b19ab..43a9b5b24f4ca 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -1058,6 +1058,64 @@ public void testLoadLocally() throws Exception { } } + @Test + public void testLoadWithSameMeasurementNameDifferentDevice() throws Exception { + final String device = "root.sg.test_0.device_1"; + MeasurementSchema measurement = + new MeasurementSchema("temperature", TSDataType.DOUBLE, TSEncoding.GORILLA); + + final long writtenPoint1; + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "same-measurement-1.tsfile"))) { + generator.registerTimeseries(device, Collections.singletonList(measurement)); + generator.generateData(device, 1000, PARTITION_INTERVAL, false); + writtenPoint1 = generator.getTotalNumber(); + } + + measurement = + new MeasurementSchema("temperature", TSDataType.DOUBLE, TSEncoding.PLAIN); + final long writtenPoint2; + try (final TsFileGenerator generator = + new TsFileGenerator(new File(tmpDir, "same-measurement-2.tsfile"))) { + generator.registerTimeseries(device, Collections.singletonList(measurement)); + generator.generateData(device, 2000, PARTITION_INTERVAL/10000, false); + writtenPoint2 = generator.getTotalNumber(); + } + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + + statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); + + try (final ResultSet resultSet = + statement.executeQuery("select count(**) from root.sg.**")) { + if (resultSet.next()) { + final long sg1Count = resultSet.getLong("count(root.sg.test_0.device_1.temperature)"); + Assert.assertEquals(writtenPoint1 + writtenPoint2, sg1Count); + } else { + Assert.fail("This ResultSet is empty."); + } + } + + try (final ResultSet resultSet = + statement.executeQuery("show timeseries root.sg.**")) { + int count = 0; + Set expectedPaths = new HashSet<>(); + expectedPaths.add(device + "." + measurement.getMeasurementName()); + while (resultSet.next()) { + String path = resultSet.getString(ColumnHeaderConstant.TIMESERIES); + Assert.assertTrue( + "Unexpected timeseries path: " + path, + expectedPaths.contains(path)); + expectedPaths.remove(path); + count++; + } + Assert.assertEquals(1, count); + Assert.assertTrue("Not all expected timeseries found", expectedPaths.isEmpty()); + } + } + } + @Test @Ignore("Load with conversion is currently banned") public void testLoadWithConvertOnTypeMismatchForTreeModel() throws Exception { From 41d3206b94b946007204e7b5b7e59aa28e20a00e Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 3 Feb 2026 19:07:55 +0800 Subject: [PATCH 5/5] spotless --- .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 15 +++++---------- .../load/TreeSchemaAutoCreatorAndVerifier.java | 2 +- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index 43a9b5b24f4ca..19a4214bed2f1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -1072,13 +1072,12 @@ public void testLoadWithSameMeasurementNameDifferentDevice() throws Exception { writtenPoint1 = generator.getTotalNumber(); } - measurement = - new MeasurementSchema("temperature", TSDataType.DOUBLE, TSEncoding.PLAIN); + measurement = new MeasurementSchema("temperature", TSDataType.DOUBLE, TSEncoding.PLAIN); final long writtenPoint2; try (final TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "same-measurement-2.tsfile"))) { generator.registerTimeseries(device, Collections.singletonList(measurement)); - generator.generateData(device, 2000, PARTITION_INTERVAL/10000, false); + generator.generateData(device, 2000, PARTITION_INTERVAL / 10000, false); writtenPoint2 = generator.getTotalNumber(); } @@ -1087,8 +1086,7 @@ public void testLoadWithSameMeasurementNameDifferentDevice() throws Exception { statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath())); - try (final ResultSet resultSet = - statement.executeQuery("select count(**) from root.sg.**")) { + try (final ResultSet resultSet = statement.executeQuery("select count(**) from root.sg.**")) { if (resultSet.next()) { final long sg1Count = resultSet.getLong("count(root.sg.test_0.device_1.temperature)"); Assert.assertEquals(writtenPoint1 + writtenPoint2, sg1Count); @@ -1097,16 +1095,13 @@ public void testLoadWithSameMeasurementNameDifferentDevice() throws Exception { } } - try (final ResultSet resultSet = - statement.executeQuery("show timeseries root.sg.**")) { + try (final ResultSet resultSet = statement.executeQuery("show timeseries root.sg.**")) { int count = 0; Set expectedPaths = new HashSet<>(); expectedPaths.add(device + "." + measurement.getMeasurementName()); while (resultSet.next()) { String path = resultSet.getString(ColumnHeaderConstant.TIMESERIES); - Assert.assertTrue( - "Unexpected timeseries path: " + path, - expectedPaths.contains(path)); + Assert.assertTrue("Unexpected timeseries path: " + path, expectedPaths.contains(path)); expectedPaths.remove(path); count++; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java index 6bc68d97d4e7d..29d4f1be07b9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java @@ -285,7 +285,7 @@ private void makeSureNoDuplicatedMeasurementsInDevices() throws LoadAnalyzeExcep if (hasDuplicates) { Map> device2TimeSeries = schemaCache.getDevice2TimeSeries(); for (final Map.Entry> entry : - deduplicatedDevice2TimeSeries.entrySet()) { + deduplicatedDevice2TimeSeries.entrySet()) { device2TimeSeries.put(entry.getKey(), new HashSet<>(entry.getValue())); } }