From 92935307856b93f369a5a6282567306846bbe643 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 14 Oct 2026 12:11:34 +0800 Subject: [PATCH 01/29] Document exchange fan-in implementation scope --- ...2026-10-14-exchange-fanin-budget-design.md | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md diff --git a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md new file mode 100644 index 0000000000000..b2bc07d1ddbb0 --- /dev/null +++ b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md @@ -0,0 +1,71 @@ + + +# Exchange 无序扇入与输入输出预算 + +基线：Apache master `aec1ad6139cb4dd1450725f53418cf40c26efb4e`。 +依据：[技术设计文档](https://timechor.feishu.cn/docx/EpDFdmS94ovMCWxluEOc3WGDn8d)，修订 861。 + +## 已确认范围 + +实现 FI 输入输出共享账目、Collect/MappingCollect 任意就绪消费，以及由无序汇集引起的计划属性修正。 +聚合使用 HashAggregation：只清空实际输入已失去连续性的 preGroupedSymbols；有效分支内的流式聚合保留。 +MergeSort、Join、ASOF、CrossJoin 只作依赖验证，不修改其消费算法，不新增进度保障机制。 +TableTopK、CountMerge 的任意就绪消费留到后续；有序输入标志仍需匹配实际输入。 + +## 额度与账目 + +F 为配置的单 FI 交换预算。INPUT=F/2，OUTPUT=F-INPUT；同 FI 的所有 pipeline 继承相同固定值。 +远端 SourceHandle 计入接收 FI 的 INPUT；SinkChannel 和 SharedTsBlockQueue 计入生产 FI 的 OUTPUT。 +本地共享队列只计费一次，创建先后不改变所有者。 + +MemoryPool 维护 query→FI→planNode 明细，注册时固定 side。 +FI 两侧使用 AtomicLong；planNode 字节明细通过 compute 增减。 +申请、回滚、释放均对应修改全局池、FI 单侧与 planNode 三层账目。 +额度判断使用本次增加后的单侧总数；稳定快照中的总数等于同侧明细之和。 +保留现有最高优先级查询的零计费兜底与交换组件的收发、等待、释放机制。 + +## Collect 生命周期 + +AnyChildBlocked 缓存直接 child.isBlocked 的 Future，每个 pending 周期只有一个 listener。 +新轮次重定向现有 listener；发布后检查完成状态，覆盖注册与完成交错。 +回调只记录就绪或错误，在锁外完成选择 Future；所有 child 方法由 Driver 线程调用。 +重复 isBlocked 与输出背压保留选择；取消对外等待不向 child 传播。 + +一次 next 消耗一次选择。null/空块刷新等待，不表示结束。 +hasNext=false 后检查 isFinished，确认结束才关闭 child 并移除记录。 +MappingCollect 按实际 childId 映射。关闭唤醒待完成选择，忽略迟到回调，逐一尝试关闭全部输入。 +保留内存 R 为各存活 child 保留内存之和；峰值为 max(R,max(R-r_i+p_i))；返回大小为 max(b_i)。 + +## 计划属性 + +多路 Collect/Union 打断分支连续性和排序；单路可透传。 +Filter 与符号透传 Project 不恢复属性；有效 Sort/MergeSort 可以重新建立属性。 +在 SINGLE/PARTIAL/FINAL 实际 child 组装后校验聚合，包括 DISTINCT/mask、普通分阶段聚合及 Union 下推。 +Group 在分区连续性失效时改为完整 Sort。复用 master 的 StreamSort 前缀校验和 Window/Fill 修复。 +SortElimination 在各汇集分支独立优化，并阻止跨无序汇集删除恢复排序。 + +## 验证 + +覆盖共享额度并发、INPUT/OUTPUT 隔离、所有者、奇数 F、超大申请、回滚释放取消及稳定账目一致性； +覆盖 100/1000 child、A 阻塞 B 就绪、listener 复用、注册竞争、输出背压、空块、结束、异常取消、关闭和映射； +覆盖文档 SQL 对应的聚合、Group/Window、StreamSort、Union 与 Fill 计划及实际结果。 +运行 Spotless、英文/中文全 reactor test-compile、针对性单测，以及本分支新增或修改的 IT。 From a1cc520cbde67c9e1cf89388efcde4ffad2099d1 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 14 Oct 2026 12:47:59 +0800 Subject: [PATCH 02/29] Share fragment exchange budgets and consume ready Collect inputs --- ...2026-10-14-exchange-fanin-budget-design.md | 28 ++ .../it/query/recent/IoTDBCollectFanInIT.java | 276 +++++++++++ .../operator/process/AnyChildBlocked.java | 242 ++++++++++ .../operator/process/CollectOperator.java | 136 ++++-- .../process/MappingCollectOperator.java | 56 ++- .../operator/process/CollectOperatorTest.java | 442 ++++++++++++++++++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 2 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 2 + .../exchange/SharedTsBlockQueue.java | 9 +- .../exchange/sink/ShuffleSinkHandle.java | 4 +- .../execution/exchange/sink/SinkChannel.java | 9 +- .../exchange/source/SourceHandle.java | 9 +- .../execution/memory/MemoryPool.java | 195 ++++++-- .../execution/memory/MemorySide.java | 31 ++ .../DataNodeTableOperatorGenerator.java | 4 +- .../planner/LocalExecutionPlanContext.java | 46 +- .../plan/planner/LocalExecutionPlanner.java | 4 +- .../plan/planner/OperatorTreeGenerator.java | 6 +- .../TableDistributedPlanGenerator.java | 122 ++++- .../optimizations/ParallelizeGrouping.java | 16 + .../optimizations/SortElimination.java | 25 + .../exchange/MPPDataExchangeManagerTest.java | 53 +++ .../execution/exchange/SourceHandleTest.java | 3 +- .../execution/memory/MemoryPoolTest.java | 115 ++++- .../LocalExecutionPlanContextBudgetTest.java | 78 ++++ .../planner/CollectInputPropertyTest.java | 149 ++++++ .../ParallelizeGroupingTest.java | 54 +++ 27 files changed, 1968 insertions(+), 148 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java create mode 100644 iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java create mode 100644 iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java diff --git a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md index b2bc07d1ddbb0..a817039cab4b9 100644 --- a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md +++ b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md @@ -69,3 +69,31 @@ SortElimination 在各汇集分支独立优化，并阻止跨无序汇集删除 覆盖 100/1000 child、A 阻塞 B 就绪、listener 复用、注册竞争、输出背压、空块、结束、异常取消、关闭和映射； 覆盖文档 SQL 对应的聚合、Group/Window、StreamSort、Union 与 Fill 计划及实际结果。 运行 Spotless、英文/中文全 reactor test-compile、针对性单测，以及本分支新增或修改的 IT。 + +## 实现验证记录（2026-10-14） + +第一版实现已完成。单元测试 86 项通过；IoTDBCollectFanInIT 的 6 项测试在单节点和三 DataNode 集群中分别通过。 +集成验证包含普通与 DISTINCT 聚合、Union 分阶段聚合、嵌套窗口、Fill、排序/TopK、Join 依赖、树模型以及树表视图。 + +构建先执行以下干净构建，生成当前分支的节点包并安装本地依赖： + +```sh +mvn -o clean install -DskipTests -pl integration-test,iotdb-core/confignode,iotdb-client/cli -am -P with-integration-tests +``` + +全 reactor 中文编译与英文编译/回归均通过；英文 test 阶段包含完整 test-compile： + +```sh +mvn -o test-compile -DskipTests -P with-zh-locale +mvn -o test -Dtest=CollectOperatorTest,MemoryPoolTest,LocalExecutionPlanContextBudgetTest,CollectInputPropertyTest,SharedTsBlockQueueTest,SourceHandleTest,SinkChannelTest,LocalSourceHandleTest,LocalSinkChannelTest,MPPDataExchangeManagerTest,ParallelizeGroupingTest,TableFunctionSortTest -Dsurefire.failIfNoSpecifiedTests=false +``` + +两个集成环境均只运行本分支新增的测试类： + +```sh +mvn -o verify -DskipUTs -Dit.test=IoTDBCollectFanInIT -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -am -P TableSimpleIT -P with-integration-tests +mvn -o verify -DskipUTs -Dit.test=IoTDBCollectFanInIT -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -am -P TableClusterIT -P with-integration-tests +``` + +使用离线模式是为了避免远端 SNAPSHOT ZIP 下载停滞；节点执行的是当前分支重新打包的代码。 +Spotless、Checkstyle、新增文件许可证及中英文消息键名/占位符一致性检查通过。 diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java new file mode 100644 index 0000000000000..adc72b6046e50 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCollectFanInIT { + private static final String DATABASE = "collect_fan_in"; + private static final int DEVICES = 24; + private static final int ROWS = 8; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setMaxTsBlockLineNumber(2) + .setSortBufferSize(128 * 1024) + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDataRegionPerDataNode(3); + EnvFactory.getEnv().initClusterEnvironment(); + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE); + statement.execute("USE " + DATABASE); + for (String table : new String[] {"readings1", "readings2"}) { + statement.execute( + "CREATE TABLE " + table + " (device STRING TAG, zone STRING TAG, value INT32 FIELD)"); + for (int device = 0; device < DEVICES; device++) { + for (int value = 1; value <= ROWS; value++) { + statement.execute( + "INSERT INTO " + + table + + " (time,device,zone,value) VALUES (" + + value + + ",'" + + device(device) + + "'," + + (zone(device) == null ? "NULL" : "'" + zone(device) + "'") + + "," + + value + + ")"); + } + } + } + statement.execute("FLUSH"); + } + } + + @Test + public void testSharedExchangePathsForTreeAndTreeToTable() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.collect_tree"); + for (int device = 0; device < DEVICES; device++) { + String path = "root.collect_tree.z" + device % 3 + "." + device(device); + statement.execute("CREATE ALIGNED TIMESERIES " + path + "(value INT32)"); + for (int value = 1; value <= ROWS; value++) { + statement.execute( + "INSERT INTO " + path + "(time,value) ALIGNED VALUES (" + value + "," + value + ")"); + } + } + try (ResultSet result = + statement.executeQuery("SELECT value FROM root.collect_tree.*.* ALIGN BY DEVICE")) { + int rows = 0; + while (result.next()) { + assertEquals(result.getLong("Time"), result.getLong("value")); + rows++; + } + assertEquals(DEVICES * ROWS, rows); + } + } + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + statement.execute( + "CREATE VIEW tree_readings(zone STRING TAG, device STRING TAG, value INT32 FIELD) AS root.collect_tree.**"); + try (ResultSet result = + statement.executeQuery( + "SELECT device, SUM(value) FROM (" + + "SELECT device, value FROM tree_readings UNION ALL SELECT device, value FROM tree_readings" + + ") u GROUP BY device ORDER BY device")) { + for (int device = 0; device < DEVICES; device++) { + assertTrue(result.next()); + assertEquals(device(device), result.getString(1)); + assertEquals(72.0, result.getDouble(2), 0.0); + } + assertFalse(result.next()); + } + } + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static String device(int id) { + return "d" + (id < 10 ? "0" : "") + id; + } + + private static String zone(int id) { + return id % 3 == 0 ? null : "z" + id % 3; + } + + @Test + public void testGroupedAggregationAfterGlobalWindow() throws Exception { + checkGroups( + "SELECT device, zone, value FROM (SELECT device, zone, value, " + + "COUNT(*) OVER () AS n FROM readings1) t WHERE n > 0", + 1); + } + + @Test + public void testUnionPartialFinalAndDistinctMask() throws Exception { + String input = + "SELECT device, zone, value FROM readings1 UNION ALL " + + "SELECT device, zone, value FROM readings2"; + checkGroups(input, 2); + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + // Without DISTINCT, this query exercises branch PARTIAL and post-Collect FINAL aggregation. + try (ResultSet result = + statement.executeQuery( + "SELECT device, zone, SUM(value) FROM (" + + input + + ") u GROUP BY device, zone ORDER BY device")) { + for (int device = 0; device < DEVICES; device++) { + assertTrue(result.next()); + assertEquals(device(device), result.getString(1)); + assertEquals(zone(device), result.getString(2)); + assertEquals(72.0, result.getDouble(3), 0.0); + } + assertFalse(result.next()); + } + } + } + + @Test + public void testOrderedFanInAndJoinDependencies() throws Exception { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + for (String limit : new String[] {"", " LIMIT 13"}) { + try (ResultSet result = + statement.executeQuery( + "SELECT device, value FROM (SELECT device, value FROM readings1 UNION ALL " + + "SELECT device, value FROM readings2) u ORDER BY device, value" + + limit)) { + int expectedRows = limit.isEmpty() ? DEVICES * ROWS * 2 : 13; + for (int row = 0; row < expectedRows; row++) { + assertTrue(result.next()); + assertEquals(device(row / (ROWS * 2)), result.getString(1)); + assertEquals(row / 2 % ROWS + 1, result.getInt(2)); + } + assertFalse(result.next()); + } + } + try (ResultSet result = + statement.executeQuery( + "SELECT COUNT(*) FROM readings1 a JOIN readings2 b " + + "ON a.device = b.device AND a.time = b.time WHERE a.value = b.value")) { + assertTrue(result.next()); + assertEquals(DEVICES * ROWS, result.getLong(1)); + assertFalse(result.next()); + } + } + } + + private void checkGroups(String input, int copies) throws Exception { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + try (ResultSet result = + statement.executeQuery( + "SELECT device, zone, SUM(value), " + + "COUNT(*), COUNT(DISTINCT value) FROM (" + + input + + ") u GROUP BY device, zone ORDER BY device")) { + for (int device = 0; device < DEVICES; device++) { + assertTrue(result.next()); + assertEquals(device(device), result.getString(1)); + assertEquals(zone(device), result.getString(2)); + assertEquals(36.0 * copies, result.getDouble(3), 0.0); + assertEquals((long) ROWS * copies, result.getLong(4)); + assertEquals(ROWS, result.getLong(5)); + } + assertFalse(result.next()); + } + } + } + + @Test + public void testNestedWindowAfterCollectRestoresCompletePartitions() throws Exception { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + try (ResultSet result = + statement.executeQuery( + "SELECT device, zone, value, " + + "SUM(value) OVER (PARTITION BY device, zone ORDER BY value) AS running_sum FROM (" + + "SELECT device, zone, value, COUNT(*) OVER () AS n FROM readings1) t " + + "WHERE n > 0 ORDER BY device, value")) { + for (int device = 0; device < DEVICES; device++) { + for (int value = 1; value <= ROWS; value++) { + assertTrue(result.next()); + assertEquals(device(device), result.getString(1)); + assertEquals(zone(device), result.getString(2)); + assertEquals(value, result.getInt(3)); + assertEquals((double) value * (value + 1) / 2, result.getDouble(4), 0.0); + } + } + assertFalse(result.next()); + } + } + } + + @Test + public void testOrderByAfterGlobalWindowAndConstantFill() throws Exception { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + try (ResultSet result = + statement.executeQuery( + "SELECT device, zone, value, COUNT(*) OVER () AS n " + + "FROM readings1 FILL METHOD CONSTANT 0 ORDER BY device, value DESC")) { + for (int device = 0; device < DEVICES; device++) { + for (int value = ROWS; value > 0; value--) { + assertTrue(result.next()); + assertEquals(device(device), result.getString(1)); + assertEquals(value, result.getInt(3)); + assertEquals((long) DEVICES * ROWS, result.getLong(4)); + } + } + assertFalse(result.next()); + } + } + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java new file mode 100644 index 0000000000000..c0397bf1ff5a4 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.process; + +import org.apache.iotdb.calc.execution.operator.Operator; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.utils.Accountable; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; + +import static com.google.common.util.concurrent.Futures.getDone; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; + +/** + * Observes direct children without calling operators from completion callbacks. The driver owns + * selection/invalidation; the lock protects listener records shared with asynchronous completions. + */ +final class AnyChildBlocked implements Accountable { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(AnyChildBlocked.class); + private static final long LISTENER_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ChildFutureListener.class); + private static final long ROUND_SIZE = + RamUsageEstimator.shallowSizeOfInstance(WaitRound.class) + + RamUsageEstimator.shallowSizeOfInstance(SettableFuture.class); + + private final List children; + private final Object lock = new Object(); + private final ChildFutureListener[] listeners; + private WaitRound currentRound; + private Throwable firstFailure; + private boolean closed; + private int nextChild; + + AnyChildBlocked(List children) { + this.children = children; + this.listeners = new ChildFutureListener[children.size()]; + } + + ListenableFuture getAnyChild() { + WaitRound round; + synchronized (lock) { + if (currentRound != null) { + return currentRound.selected; + } + round = new WaitRound(); + currentRound = round; + for (ChildFutureListener listener : listeners) { + if (listener != null) { + listener.target = round; + } + } + } + + // Activate every live direct child, even if an earlier one has already become ready. Only + // invalidated records call child.isBlocked(); a long-pending future keeps its one listener. + for (int offset = 0; offset < children.size(); offset++) { + int childId = (nextChild + offset) % children.size(); + Operator child; + ChildFutureListener listener; + synchronized (lock) { + if (closed) { + break; + } + child = children.get(childId); + listener = listeners[childId]; + } + if (child == null) { + continue; + } + if (listener == null) { + ListenableFuture observedFuture = child.isBlocked(); + synchronized (lock) { + if (closed) { + break; + } + listener = new ChildFutureListener(childId, observedFuture, round); + listeners[childId] = listener; + } + // The target is published before registration. addListener also handles a future that + // finishes between isBlocked and registration, including inline completion. + observedFuture.addListener(listener, directExecutor()); + } else if (listener.observedFuture.isDone()) { + // A completed record may have notified a previous round. Recheck it for this round. + listener.run(); + } + } + + Throwable failure; + boolean isClosed; + synchronized (lock) { + failure = firstFailure; + isClosed = closed; + } + if (isClosed || children.isEmpty()) { + round.selected.set(-1); + } else if (failure != null) { + round.selected.setException(failure); + } + return round.selected; + } + + void checkFailure() throws Exception { + Throwable failure; + synchronized (lock) { + failure = firstFailure; + } + if (failure instanceof Exception) { + throw (Exception) failure; + } + if (failure instanceof Error) { + throw (Error) failure; + } + if (failure != null) { + throw new RuntimeException(failure); + } + } + + void invalidateChild(int childId) { + synchronized (lock) { + listeners[childId] = null; + if (currentRound != null) { + currentRound.valid = false; + currentRound = null; + } + nextChild = (childId + 1) % children.size(); + } + } + + void finishChild(int childId) { + invalidateChild(childId); + } + + void close() { + WaitRound round; + synchronized (lock) { + closed = true; + round = currentRound; + currentRound = null; + if (round != null) { + round.valid = false; + } + for (ChildFutureListener listener : listeners) { + if (listener != null) { + listener.target = null; + } + } + Arrays.fill(listeners, null); + } + if (round != null) { + round.selected.set(-1); + } + } + + @Override + public long ramBytesUsed() { + synchronized (lock) { + long size = + INSTANCE_SIZE + + RamUsageEstimator.shallowSizeOfInstance(Object.class) + + RamUsageEstimator.shallowSizeOf(listeners); + for (ChildFutureListener listener : listeners) { + if (listener != null) { + size += LISTENER_SIZE; + } + } + return size + (currentRound == null ? 0 : ROUND_SIZE); + } + } + + private static final class WaitRound { + private final SettableFuture selected = SettableFuture.create(); + private boolean valid = true; + } + + private final class ChildFutureListener implements Runnable { + private final int childId; + private final ListenableFuture observedFuture; + private WaitRound target; + + private ChildFutureListener(int childId, ListenableFuture observedFuture, WaitRound target) { + this.childId = childId; + this.observedFuture = observedFuture; + this.target = target; + } + + @Override + public void run() { + Throwable failure = null; + try { + getDone(observedFuture); + } catch (ExecutionException e) { + failure = e.getCause(); + } catch (CancellationException e) { + failure = e; + } + WaitRound round; + synchronized (lock) { + if (closed || listeners[childId] != this) { + return; + } + if (failure != null && firstFailure == null) { + firstFailure = failure; + } + round = target; + if (round == null || !round.valid || round != currentRound) { + return; + } + failure = firstFailure; + } + // Completing a selection can invoke downstream listeners synchronously. Do not hold lock. + if (failure == null) { + round.selected.set(childId); + } else { + round.selected.setException(failure); + } + } + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java index 921d273d04e8a..2cc5deec59ea2 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java @@ -30,60 +30,102 @@ import java.util.List; +import static com.google.common.util.concurrent.Futures.getDone; +import static com.google.common.util.concurrent.Futures.nonCancellationPropagating; + public class CollectOperator implements ProcessOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(CollectOperator.class); private final CommonOperatorContext operatorContext; protected final List children; - private boolean inited = false; - - protected int currentIndex; + private final AnyChildBlocked anyChildBlocked; + private ListenableFuture selectionFuture; + private ListenableFuture exposedFuture; + private int remainingChildren; + private boolean closed; public CollectOperator(CommonOperatorContext operatorContext, List children) { this.operatorContext = operatorContext; this.children = children; - this.currentIndex = 0; + this.anyChildBlocked = new AnyChildBlocked(children); + this.remainingChildren = children.size(); } @Override - public boolean hasNext() throws Exception { - return currentIndex < children.size(); + public boolean hasNext() { + return !closed && remainingChildren > 0; } @Override public TsBlock next() throws Exception { - if (children.get(currentIndex).hasNextWithTimer()) { - return children.get(currentIndex).nextWithTimer(); - } else { - closeCurrentChild(currentIndex); - currentIndex++; + anyChildBlocked.checkFailure(); + if (!hasNext()) { + return null; + } + if (selectionFuture == null) { + isBlocked(); + } + anyChildBlocked.checkFailure(); + if (!selectionFuture.isDone()) { + return null; + } + int childId = getDone(selectionFuture); + // Consume this selection before invoking any operator method. Another next must acquire a + // new readiness observation even when this advancement returns null or an empty block. + selectionFuture = null; + exposedFuture = null; + if (childId < 0 || closed || children.get(childId) == null) { return null; } + Operator child = children.get(childId); + try { + if (child.hasNextWithTimer()) { + TsBlock block = child.nextWithTimer(); + return block == null ? null : transformBlock(childId, block); + } + if (child.isFinished()) { + closeCurrentChild(childId); + } + return null; + } finally { + anyChildBlocked.invalidateChild(childId); + } + } + + protected TsBlock transformBlock(int childId, TsBlock block) { + return block; } protected void closeCurrentChild(int index) throws Exception { - children.get(index).close(); + Operator child = children.get(index); + if (child == null) { + return; + } + // Retire the record even if closing its subtree throws. + anyChildBlocked.finishChild(index); children.set(index, null); + remainingChildren--; + child.close(); } @Override public ListenableFuture isBlocked() { - if (!inited) { - inited = true; - for (Operator child : children) { - child.isBlocked(); - } - } - if (currentIndex >= children.size()) { + if (!hasNext()) { return NOT_BLOCKED; } - return children.get(currentIndex).isBlocked(); + if (selectionFuture == null) { + selectionFuture = anyChildBlocked.getAnyChild(); + } + if (exposedFuture == null || exposedFuture.isCancelled()) { + exposedFuture = nonCancellationPropagating(selectionFuture); + } + return exposedFuture; } @Override - public boolean isFinished() throws Exception { - return currentIndex >= children.size(); + public boolean isFinished() { + return !hasNext(); } @Override @@ -93,35 +135,67 @@ public CommonOperatorContext getOperatorContext() { @Override public void close() throws Exception { - for (int i = currentIndex, n = children.size(); i < n; i++) { - Operator currentChild = children.get(i); - if (currentChild != null) { + if (closed) { + return; + } + closed = true; + anyChildBlocked.close(); + selectionFuture = null; + exposedFuture = null; + Exception firstException = null; + for (int i = 0; i < children.size(); i++) { + try { closeCurrentChild(i); + } catch (Exception e) { + if (firstException == null) { + firstException = e; + } else if (firstException != e) { + firstException.addSuppressed(e); + } } } + if (firstException != null) { + throw firstException; + } } @Override public long calculateMaxPeekMemory() { - long maxPeekMemory = 0; + long retained = calculateRetainedSizeAfterCallingNext(); + long peak = retained; for (Operator child : children) { - maxPeekMemory = Math.max(maxPeekMemory, child.calculateMaxPeekMemoryWithCounter()); + if (child != null) { + peak = + Math.max( + peak, + retained + - child.calculateRetainedSizeAfterCallingNext() + + child.calculateMaxPeekMemoryWithCounter()); + } } - return maxPeekMemory; + return peak; } @Override public long calculateMaxReturnSize() { long maxReturnSize = 0; for (Operator child : children) { - maxReturnSize = Math.max(maxReturnSize, child.calculateMaxReturnSize()); + if (child != null) { + maxReturnSize = Math.max(maxReturnSize, child.calculateMaxReturnSize()); + } } return maxReturnSize; } @Override public long calculateRetainedSizeAfterCallingNext() { - return 0L; + long retained = 0; + for (Operator child : children) { + if (child != null) { + retained += child.calculateRetainedSizeAfterCallingNext(); + } + } + return retained; } @TestOnly @@ -132,7 +206,9 @@ public List getChildren() { @Override public long ramBytesUsed() { return INSTANCE_SIZE + + anyChildBlocked.ramBytesUsed() + children.stream() + .filter(java.util.Objects::nonNull) .mapToLong(MemoryEstimationHelper::getEstimatedSizeOfAccountableObject) .sum() + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/MappingCollectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/MappingCollectOperator.java index 8bc1c18e9e66a..02850bf0caf16 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/MappingCollectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/MappingCollectOperator.java @@ -45,37 +45,43 @@ public MappingCollectOperator( List> mappings) { super(operatorContext, children); this.mappings = mappings; - outputColumnsCount = mappings.get(0).size(); + outputColumnsCount = mappings.isEmpty() ? 0 : mappings.get(0).size(); } @Override - public TsBlock next() throws Exception { - if (children.get(currentIndex).hasNextWithTimer()) { - TsBlock tsBlock = children.get(currentIndex).nextWithTimer(); - if (tsBlock == null) { - return null; - } else { - Column[] columns = new Column[outputColumnsCount]; - List mapping = mappings.get(currentIndex); - for (int i = 0; i < columns.length; i++) { - columns[i] = tsBlock.getColumn(mapping.get(i)); - } - return TsBlock.wrapBlocksWithoutCopy( - tsBlock.getPositionCount(), - new RunLengthEncodedColumn( - CommonOperatorUtils.TIME_COLUMN_TEMPLATE, tsBlock.getPositionCount()), - columns); - } - } else { - closeCurrentChild(currentIndex); - currentIndex++; - return null; + protected TsBlock transformBlock(int childId, TsBlock tsBlock) { + Column[] columns = new Column[outputColumnsCount]; + List mapping = mappings.get(childId); + for (int i = 0; i < columns.length; i++) { + columns[i] = tsBlock.getColumn(mapping.get(i)); } + return TsBlock.wrapBlocksWithoutCopy( + tsBlock.getPositionCount(), + new RunLengthEncodedColumn( + CommonOperatorUtils.TIME_COLUMN_TEMPLATE, tsBlock.getPositionCount()), + columns); } + @Override protected void closeCurrentChild(int index) throws Exception { - children.get(index).close(); - children.set(index, null); - mappings.set(index, null); + try { + super.closeCurrentChild(index); + } finally { + mappings.set(index, null); + } + } + + @Override + public long ramBytesUsed() { + long size = + super.ramBytesUsed() + + INSTANCE_SIZE + - RamUsageEstimator.shallowSizeOfInstance(CollectOperator.class); + for (List mapping : mappings) { + if (mapping != null) { + size += RamUsageEstimator.shallowSizeOf(mapping.toArray()); + } + } + return size; } } diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java new file mode 100644 index 0000000000000..abf7a82acf3d9 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java @@ -0,0 +1,442 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.process; + +import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; +import org.apache.iotdb.calc.execution.operator.Operator; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; + +import com.google.common.util.concurrent.ForwardingListenableFuture; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class CollectOperatorTest { + private static TsBlock block(int... values) { + TsBlockBuilder builder = + new TsBlockBuilder(Collections.nCopies(values.length, TSDataType.INT32)); + builder.getTimeColumnBuilder().writeLong(0); + for (int i = 0; i < values.length; i++) { + builder.getColumnBuilder(i).writeInt(values[i]); + } + builder.declarePosition(); + return builder.build(); + } + + private static CollectOperator collect(Operator... children) { + return new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(children))); + } + + @Test + public void testReadyChildAndStableSelectionDuringBackpressure() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + b.ready(); + CollectOperator collect = collect(a, b); + ListenableFuture selected = collect.isBlocked(); + assertTrue(selected.isDone()); + a.ready(); + assertSame(selected, collect.isBlocked()); + assertSame(b.output, collect.next()); + assertEquals(0, a.advances); + assertEquals(1, b.advances); + assertTrue(collect.isBlocked().isDone()); + assertSame(a.output, collect.next()); + collect.close(); + } + + @Test + public void testThousandChildrenAndOneListenerPerPendingPeriod() throws Exception { + for (int count : new int[] {100, 1000}) { + List children = new ArrayList<>(); + for (int i = 0; i < count; i++) { + children.add(new Child(block(i))); + } + CollectOperator collect = new CollectOperator(new TestContext(), children); + Child ready = (Child) children.get(count - 1); + for (int round = 0; round < 5; round++) { + ListenableFuture wait = collect.isBlocked(); + assertFalse(wait.isDone()); + for (int repeat = 0; repeat < 10; repeat++) { + assertSame(wait, collect.isBlocked()); + } + ready.ready(); + wait.get(5, TimeUnit.SECONDS); + assertSame(ready.output, collect.next()); + ready.reset(); + } + for (int i = 0; i < count - 1; i++) { + Child child = (Child) children.get(i); + assertEquals(1, child.waitCalls); + assertEquals(1, child.wait.registrations.get()); + } + collect.close(); + } + } + + @Test + public void testPendingListenerRetargetedAndStaleCompletionIgnored() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + CollectOperator collect = collect(a, b); + ListenableFuture first = collect.isBlocked(); + b.ready(); + assertSame(b.output, collect.next()); + Runnable stale = b.wait.listener; + b.reset(); + ListenableFuture second = collect.isBlocked(); + assertFalse(second.isDone()); + stale.run(); + assertFalse(second.isDone()); + a.ready(); + second.get(5, TimeUnit.SECONDS); + assertEquals(1, a.wait.registrations.get()); + assertSame(a.output, collect.next()); + assertTrue(first.isDone()); + collect.close(); + } + + @Test + public void testNullEmptyAndUnfinishedChildRefreshReadiness() throws Exception { + Child child = new Child(null); + child.ready(); + CollectOperator collect = collect(child); + assertTrue(collect.isBlocked().isDone()); + assertNull(collect.next()); + child.reset(); + assertFalse(collect.isBlocked().isDone()); + child.output = new TsBlockBuilder(Collections.singletonList(TSDataType.INT32)).build(); + child.ready(); + assertTrue(collect.next().isEmpty()); + child.reset(); + child.hasNext = false; + child.ready(); + assertNull(collect.next()); + assertFalse(collect.isFinished()); + assertEquals(0, child.closes); + child.finished = true; + assertNull(collect.next()); + assertTrue(collect.isFinished()); + assertEquals(1, child.closes); + assertTrue(collect.isBlocked().isDone()); + } + + @Test + public void testCancellationDoesNotPropagateToChild() throws Exception { + Child child = new Child(block(1)); + CollectOperator collect = collect(child); + ListenableFuture exposed = collect.isBlocked(); + assertTrue(exposed.cancel(true)); + assertFalse(child.wait.isCancelled()); + ListenableFuture replacement = collect.isBlocked(); + assertFalse(replacement.isCancelled()); + child.ready(); + replacement.get(5, TimeUnit.SECONDS); + assertSame(child.output, collect.next()); + collect.close(); + } + + @Test + public void testFailureOfUnselectedChildIsNotLost() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + a.ready(); + CollectOperator collect = collect(a, b); + assertTrue(collect.isBlocked().isDone()); + IOException expected = new IOException("child failure"); + b.wait.delegate.setException(expected); + try { + collect.next(); + fail("must propagate the failed child even when another child was selected"); + } catch (IOException e) { + assertSame(expected, e); + } finally { + collect.close(); + } + assertEquals(0, a.advances); + } + + @Test + public void testChildCancellationPropagates() throws Exception { + Child child = new Child(block(1)); + CollectOperator collect = collect(child); + ListenableFuture wait = collect.isBlocked(); + child.wait.cancel(false); + assertTrue(wait.isDone()); + try { + collect.next(); + fail("must propagate child cancellation"); + } catch (CancellationException expected) { + assertEquals(0, child.advances); + } finally { + collect.close(); + } + } + + @Test + public void testCompletionDuringRegistrationAndOnAnotherThread() throws Exception { + Child a = new Child(block(1)); + a.wait.beforeRegistration = a::ready; + CollectOperator collect = collect(a); + assertTrue(collect.isBlocked().isDone()); + assertSame(a.output, collect.next()); + a.reset(); + ListenableFuture pending = collect.isBlocked(); + Thread completion = new Thread(a::ready); + completion.start(); + pending.get(5, TimeUnit.SECONDS); + completion.join(); + assertSame(a.output, collect.next()); + collect.close(); + } + + @Test + public void testCloseWakesWaitAndClosesEveryChildDespiteException() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + a.closeFailure = new IOException("close failure"); + CollectOperator collect = collect(a, b); + ListenableFuture pending = collect.isBlocked(); + try { + collect.close(); + fail("must propagate close failure"); + } catch (IOException e) { + assertSame(a.closeFailure, e); + } + pending.get(5, TimeUnit.SECONDS); + assertEquals(1, a.closes); + assertEquals(1, b.closes); + a.ready(); + b.ready(); + assertTrue(collect.isFinished()); + assertNull(collect.next()); + collect.close(); + assertEquals(1, b.closes); + } + + @Test + public void testMappingUsesActualChildIdentity() throws Exception { + Child a = new Child(block(10, 11)); + Child b = new Child(block(20, 21)); + b.ready(); + List> mappings = new ArrayList<>(); + mappings.add(Collections.singletonList(0)); + mappings.add(Collections.singletonList(1)); + MappingCollectOperator collect = + new MappingCollectOperator( + new TestContext(), new ArrayList<>(Arrays.asList(a, b)), mappings); + assertTrue(collect.isBlocked().isDone()); + assertEquals(21, collect.next().getColumn(0).getInt(0)); + a.ready(); + assertEquals(10, collect.next().getColumn(0).getInt(0)); + collect.close(); + assertNull(mappings.get(0)); + assertNull(mappings.get(1)); + } + + @Test + public void testMemoryIncludesAllLiveRetainedStates() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + a.retained = 30; + a.peak = 100; + a.returnSize = 10; + b.retained = 50; + b.peak = 80; + b.returnSize = 20; + CollectOperator collect = collect(a, b); + assertEquals(80, collect.calculateRetainedSizeAfterCallingNext()); + assertEquals(150, collect.calculateMaxPeekMemory()); + assertEquals(20, collect.calculateMaxReturnSize()); + assertTrue(collect.ramBytesUsed() > 0); + a.hasNext = false; + a.finished = true; + a.ready(); + collect.next(); + assertEquals(50, collect.calculateRetainedSizeAfterCallingNext()); + assertEquals(80, collect.calculateMaxPeekMemory()); + collect.close(); + assertEquals(0, collect.calculateMaxPeekMemory()); + assertEquals(0, collect.calculateMaxReturnSize()); + assertEquals(0, collect.calculateRetainedSizeAfterCallingNext()); + } + + private static final class CountingFuture + extends ForwardingListenableFuture.SimpleForwardingListenableFuture { + private final SettableFuture delegate; + private final AtomicInteger registrations = new AtomicInteger(); + private Runnable beforeRegistration; + private Runnable listener; + + private CountingFuture() { + this(SettableFuture.create()); + } + + private CountingFuture(SettableFuture delegate) { + super(delegate); + this.delegate = delegate; + } + + @Override + public void addListener(Runnable listener, Executor executor) { + this.listener = listener; + registrations.incrementAndGet(); + if (beforeRegistration != null) { + beforeRegistration.run(); + } + super.addListener(listener, executor); + } + } + + private static final class TestContext extends CommonOperatorContext { + private TestContext() { + super(0, new PlanNodeId("collect"), "test"); + } + + @Override + public MemoryReservationManager getMemoryReservationContext() { + return null; + } + + @Override + public int getFragmentId() { + return 0; + } + + @Override + public int getPipelineId() { + return 0; + } + + @Override + public long ramBytesUsed() { + return 0; + } + } + + private static final class Child implements Operator { + private final Thread driver = Thread.currentThread(); + private final CommonOperatorContext context = new TestContext(); + private CountingFuture wait = new CountingFuture(); + private TsBlock output; + private boolean hasNext = true; + private boolean finished; + private int advances; + private int waitCalls; + private int closes; + private IOException closeFailure; + private long retained, peak, returnSize; + + private Child(TsBlock output) { + this.output = output; + } + + private void ready() { + wait.delegate.set(null); + } + + private void reset() { + wait = new CountingFuture(); + } + + private void checkDriver() { + assertSame(driver, Thread.currentThread()); + } + + @Override + public CommonOperatorContext getOperatorContext() { + return context; + } + + @Override + public ListenableFuture isBlocked() { + checkDriver(); + waitCalls++; + return wait; + } + + @Override + public boolean hasNext() { + checkDriver(); + return hasNext; + } + + @Override + public TsBlock next() { + checkDriver(); + advances++; + return output; + } + + @Override + public boolean isFinished() { + checkDriver(); + return finished; + } + + @Override + public void close() throws Exception { + checkDriver(); + closes++; + if (closeFailure != null) throw closeFailure; + } + + @Override + public long calculateMaxPeekMemory() { + return peak; + } + + @Override + public long calculateMaxReturnSize() { + return returnSize; + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return retained; + } + + @Override + public long ramBytesUsed() { + return 0; + } + } +} diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index c89034e8831df..f28a007654ca8 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4051,4 +4051,6 @@ private DataNodeQueryMessages() {} "Dispatch LOAD piece: loadId={}, regionId={}, pieceIndex={}, tsFile={}, dataSize={}, replicaSet={}"; public static final String LOG_DISPATCH_LOAD_PIECE_SUCCESS_LOADID_ARG_REGIONID_ARG_PIECEINDEX_ARG_DATASIZE_ARG_75F7AEFE = "Dispatch LOAD piece success: loadId={}, regionId={}, pieceIndex={}, dataSize={}"; + public static final String EXCEPTION_PLAN_NODE_ARG_IS_ALREADY_REGISTERED_ON_MEMORY_SIDE_ARG_NOT_ARG_4C1BB625 = "Plan node %s is already registered on memory side %s, not %s"; + public static final String EXCEPTION_MEMORY_RESERVATION_IS_NOT_REGISTERED_FOR_QUERY_ARG_FRAGMENT_ARG_PLAN_NODE_ARG_67A4B81F = "Memory reservation is not registered for query %s, fragment %s, plan node %s"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index a3307bd07cb42..57d2ef87150c8 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4806,4 +4806,6 @@ private DataNodeQueryMessages() {} "下发 LOAD piece：loadId={}，regionId={}，pieceIndex={}，tsFile={}，dataSize={}，replicaSet={}"; public static final String LOG_DISPATCH_LOAD_PIECE_SUCCESS_LOADID_ARG_REGIONID_ARG_PIECEINDEX_ARG_DATASIZE_ARG_75F7AEFE = "下发 LOAD piece 成功：loadId={}，regionId={}，pieceIndex={}，dataSize={}"; + public static final String EXCEPTION_PLAN_NODE_ARG_IS_ALREADY_REGISTERED_ON_MEMORY_SIDE_ARG_NOT_ARG_4C1BB625 = "PlanNode %s 已注册到内存侧 %s，不能注册到 %s"; + public static final String EXCEPTION_MEMORY_RESERVATION_IS_NOT_REGISTERED_FOR_QUERY_ARG_FRAGMENT_ARG_PLAN_NODE_ARG_67A4B81F = "查询 %s、Fragment %s、PlanNode %s 尚未注册内存账目"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java index 5812d4e37cdd3..cb8c681e4e9da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.source.LocalSourceHandle; import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; +import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; @@ -85,7 +86,8 @@ public class SharedTsBlockQueue { private LocalSinkChannel sinkChannel; private long maxBytesCanReserve = - IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); + MemorySide.OUTPUT.getMaxBytes( + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance()); private final boolean isHighestPriority; private volatile Throwable abortedCause = null; @@ -265,7 +267,10 @@ public ListenableFuture add(TsBlock tsBlock) { localMemoryManager .getQueryPool() .registerPlanNodeIdToQueryMemoryMap( - localFragmentInstanceId.queryId, fullFragmentInstanceId, localPlanNodeId); + localFragmentInstanceId.queryId, + fullFragmentInstanceId, + localPlanNodeId, + MemorySide.OUTPUT); alreadyRegistered = true; } MemoryReservationResult reserveResult = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/ShuffleSinkHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/ShuffleSinkHandle.java index e54062158a3f1..1de3ccb897df7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/ShuffleSinkHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/ShuffleSinkHandle.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager; +import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.metric.DataExchangeCostMetricSet; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; @@ -76,7 +77,8 @@ public class ShuffleSinkHandle implements ISinkHandle { /** max bytes this ShuffleSinkHandle can reserve. */ private long maxBytesCanReserve = - IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); + MemorySide.OUTPUT.getMaxBytes( + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance()); private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ShuffleSinkHandle.class) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java index d24a1552a5b22..0dec1cbf07f26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager.SinkListener; import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; +import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.metric.DataExchangeCostMetricSet; import org.apache.iotdb.db.queryengine.metric.DataExchangeCountMetricSet; import org.apache.iotdb.db.utils.SetThreadName; @@ -131,7 +132,8 @@ private TsBlockInfo(TsBlock tsBlock, ByteBuffer serializedTsBlock, long tsBlockS /** max bytes this SinkChannel can reserve. */ private long maxBytesCanReserve = - IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); + MemorySide.OUTPUT.getMaxBytes( + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance()); private final boolean isHighestPriority; @@ -235,7 +237,10 @@ public SinkChannel( localMemoryManager .getQueryPool() .registerPlanNodeIdToQueryMemoryMap( - localFragmentInstanceId.queryId, fullFragmentInstanceId, localPlanNodeId); + localFragmentInstanceId.queryId, + fullFragmentInstanceId, + localPlanNodeId, + MemorySide.OUTPUT); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java index aed3f9533fb1d..b1dc2781ed658 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager.SourceHandleListener; import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; +import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.metric.DataExchangeCostMetricSet; import org.apache.iotdb.db.queryengine.metric.DataExchangeCountMetricSet; import org.apache.iotdb.db.utils.SetThreadName; @@ -117,7 +118,8 @@ public class SourceHandle implements ISourceHandle { /** max bytes this SourceHandle can reserve. */ private long maxBytesCanReserve = - IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); + MemorySide.INPUT.getMaxBytes( + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance()); /** * this is set to true after calling isBlocked() at least once which indicates that this @@ -223,7 +225,10 @@ public SourceHandle( localMemoryManager .getQueryPool() .registerPlanNodeIdToQueryMemoryMap( - localFragmentInstanceId.queryId, fullFragmentInstanceId, localPlanNodeId); + localFragmentInstanceId.queryId, + fullFragmentInstanceId, + localPlanNodeId, + MemorySide.INPUT); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index b2d35568fa31c..56c9974764a5f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -40,6 +40,7 @@ import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import static com.google.common.util.concurrent.Futures.immediateVoidFuture; @@ -57,8 +58,8 @@ public static class MemoryReservationFuture extends AbstractFuture { private final long bytesToReserve; /** - * MemoryReservationFuture is created when SinkHandle or SourceHandle tries to reserve memory - * from pool. This field is max Bytes that SinkHandle or SourceHandle can reserve. + * MemoryReservationFuture is created when an exchange handle tries to reserve memory from the + * pool. This limit is shared by all plan nodes registered on the same side of that fragment. */ private final long maxBytesCanReserve; @@ -154,8 +155,32 @@ public long getReservedBytes() { private final IMemoryBlock memoryBlock; private final long maxBytesPerFragmentInstance; - /** queryId -> fragmentInstanceId -> planNodeId -> bytesReserved. */ - private final Map>> queryMemoryReservations = + private static class PlanNodeReservation { + private final MemorySide side; + private volatile long bytesReserved; + + private PlanNodeReservation(MemorySide side) { + this.side = side; + } + + @Override + public String toString() { + return side + ":" + bytesReserved; + } + } + + private static class FragmentMemoryReservation { + private final Map planNodes = new ConcurrentHashMap<>(); + private final AtomicLong inputUsedBytes = new AtomicLong(); + private final AtomicLong outputUsedBytes = new AtomicLong(); + + private AtomicLong getUsedBytes(MemorySide side) { + return side == MemorySide.INPUT ? inputUsedBytes : outputUsedBytes; + } + } + + /** queryId -> fragmentInstanceId -> FI side totals and planNode details. */ + private final Map> queryMemoryReservations = new ConcurrentHashMap<>(); private final Queue> memoryReservationFutures = @@ -204,12 +229,27 @@ public int getMemoryReservationSize() { * planNodeId to queryMemoryReservationsMap first. */ public void registerPlanNodeIdToQueryMemoryMap( - String queryId, String fragmentInstanceId, String planNodeId) { + String queryId, String fragmentInstanceId, String planNodeId, MemorySide side) { + java.util.Objects.requireNonNull(side); synchronized (queryMemoryReservations) { queryMemoryReservations .computeIfAbsent(queryId, x -> new ConcurrentHashMap<>()) - .computeIfAbsent(fragmentInstanceId, x -> new ConcurrentHashMap<>()) - .putIfAbsent(planNodeId, 0L); + .computeIfAbsent(fragmentInstanceId, x -> new FragmentMemoryReservation()) + .planNodes + .compute( + planNodeId, + (key, reservation) -> { + if (reservation != null && reservation.side != side) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_PLAN_NODE_ARG_IS_ALREADY_REGISTERED_ON_MEMORY_SIDE_ARG_NOT_ARG_4C1BB625, + planNodeId, + reservation.side, + side)); + } + return reservation == null ? new PlanNodeReservation(side) : reservation; + }); } } @@ -224,15 +264,19 @@ public void registerPlanNodeIdToQueryMemoryMap( */ public void deRegisterFragmentInstanceFromQueryMemoryMap( String queryId, String fragmentInstanceId, boolean forceDeregister) { - Map> queryRelatedMemory = queryMemoryReservations.get(queryId); + Map queryRelatedMemory = + queryMemoryReservations.get(queryId); if (queryRelatedMemory != null) { - Map fragmentRelatedMemory = queryRelatedMemory.get(fragmentInstanceId); + FragmentMemoryReservation fragmentRelatedMemory = queryRelatedMemory.get(fragmentInstanceId); boolean hasPotentialMemoryLeak = false; // fragmentRelatedMemory could be null if the FI has not reserved any memory(For example, // next() of root operator returns no data) if (fragmentRelatedMemory != null) { hasPotentialMemoryLeak = - fragmentRelatedMemory.values().stream().anyMatch(value -> value != 0L); + fragmentRelatedMemory.inputUsedBytes.get() != 0L + || fragmentRelatedMemory.outputUsedBytes.get() != 0L + || fragmentRelatedMemory.planNodes.values().stream() + .anyMatch(value -> value.bytesReserved != 0L); } if (!forceDeregister && hasPotentialMemoryLeak) { // If hasPotentialMemoryLeak is true, it means that LocalSourceChannel/LocalSourceHandles @@ -252,9 +296,9 @@ public void deRegisterFragmentInstanceFromQueryMemoryMap( } if (hasPotentialMemoryLeak) { // hasPotentialMemoryLeak means that fragmentRelatedMemory is not null - List> invalidEntryList = - fragmentRelatedMemory.entrySet().stream() - .filter(entry -> entry.getValue() != 0L) + List> invalidEntryList = + fragmentRelatedMemory.planNodes.entrySet().stream() + .filter(entry -> entry.getValue().bytesReserved != 0L) .collect(Collectors.toList()); throw new MemoryLeakException( String.format( @@ -390,25 +434,26 @@ public void free(String queryId, String fragmentInstanceId, String planNodeId, l Validate.notNull(queryId, DataNodeQueryMessages.EXCEPTION_QUERYID_CAN_NOT_BE_NULL_DOT_16639DBE); Validate.isTrue(bytes > 0L); - try { - queryMemoryReservations - .get(queryId) - .get(fragmentInstanceId) - .computeIfPresent( - planNodeId, - (k, reservedMemory) -> { - if (reservedMemory < bytes) { - throw new IllegalArgumentException( - DataNodeQueryMessages.FREE_MORE_MEMORY_THAN_HAS_BEEN_RESERVED); - } - return reservedMemory - bytes; - }); - } catch (NullPointerException e) { + Map fragments = queryMemoryReservations.get(queryId); + FragmentMemoryReservation fragment = + fragments == null ? null : fragments.get(fragmentInstanceId); + if (fragment == null) { throw new IllegalArgumentException( DataNodeQueryMessages - .QUERY_EXCEPTION_RELATEDMEMORYRESERVED_CAN_T_BE_NULL_WHEN_FREEING_MEMORY_C80009F2, - e); + .QUERY_EXCEPTION_RELATEDMEMORYRESERVED_CAN_T_BE_NULL_WHEN_FREEING_MEMORY_C80009F2); } + fragment.planNodes.compute( + planNodeId, + (key, reservation) -> { + requireReservation(reservation, queryId, fragmentInstanceId, planNodeId); + if (reservation.bytesReserved < bytes) { + throw new IllegalArgumentException( + DataNodeQueryMessages.FREE_MORE_MEMORY_THAN_HAS_BEEN_RESERVED); + } + reservation.bytesReserved -= bytes; + fragment.getUsedBytes(reservation.side).addAndGet(-bytes); + return reservation; + }); memoryBlock.release(bytes); @@ -440,16 +485,34 @@ public void free(String queryId, String fragmentInstanceId, String planNodeId, l } public long getQueryMemoryReservedBytes(String queryId) { - if (!queryMemoryReservations.containsKey(queryId)) { + Map fragments = queryMemoryReservations.get(queryId); + if (fragments == null) { return 0L; } long sum = 0; - for (Map map : queryMemoryReservations.get(queryId).values()) { - sum = sum + map.values().stream().reduce(0L, Long::sum); + for (FragmentMemoryReservation fragment : fragments.values()) { + sum += fragment.inputUsedBytes.get() + fragment.outputUsedBytes.get(); } return sum; } + public long getFragmentMemoryReservedBytes( + String queryId, String fragmentInstanceId, MemorySide side) { + Map fragments = queryMemoryReservations.get(queryId); + FragmentMemoryReservation fragment = + fragments == null ? null : fragments.get(fragmentInstanceId); + return fragment == null ? 0L : fragment.getUsedBytes(side).get(); + } + + public long getPlanNodeMemoryReservedBytes( + String queryId, String fragmentInstanceId, String planNodeId) { + Map fragments = queryMemoryReservations.get(queryId); + FragmentMemoryReservation fragment = + fragments == null ? null : fragments.get(fragmentInstanceId); + PlanNodeReservation reservation = fragment == null ? null : fragment.planNodes.get(planNodeId); + return reservation == null ? 0L : reservation.bytesReserved; + } + public long getReservedBytes() { return memoryBlock.getUsedMemoryInBytes(); } @@ -460,22 +523,68 @@ public boolean tryReserve( String planNodeId, long bytesToReserve, long maxBytesCanReserve) { + FragmentMemoryReservation fragment = + getFragmentReservation(queryId, fragmentInstanceId, planNodeId); + PlanNodeReservation reservation = + requireReservation( + fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); long tryUsedBytes = memoryBlock.forceAllocateWithoutLimitation(bytesToReserve); - long queryRemainingBytes = - maxBytesCanReserve - - queryMemoryReservations - .get(queryId) - .get(fragmentInstanceId) - .merge(planNodeId, bytesToReserve, Long::sum); - return tryUsedBytes <= memoryBlock.getTotalMemorySizeInBytes() && queryRemainingBytes >= 0; + long sideUsedBytes = fragment.getUsedBytes(reservation.side).addAndGet(bytesToReserve); + fragment.planNodes.compute( + planNodeId, + (key, value) -> { + value.bytesReserved += bytesToReserve; + return value; + }); + return tryUsedBytes <= memoryBlock.getTotalMemorySizeInBytes() + && sideUsedBytes <= maxBytesCanReserve; } private void rollbackReserve( String queryId, String fragmentInstanceId, String planNodeId, long bytesToReserve) { - queryMemoryReservations - .get(queryId) - .get(fragmentInstanceId) - .merge(planNodeId, -bytesToReserve, Long::sum); + FragmentMemoryReservation fragment = + getFragmentReservation(queryId, fragmentInstanceId, planNodeId); + fragment.planNodes.compute( + planNodeId, + (key, reservation) -> { + reservation.bytesReserved -= bytesToReserve; + fragment.getUsedBytes(reservation.side).addAndGet(-bytesToReserve); + return reservation; + }); memoryBlock.release(bytesToReserve); } + + private FragmentMemoryReservation getFragmentReservation( + String queryId, String fragmentInstanceId, String planNodeId) { + Map fragments = queryMemoryReservations.get(queryId); + FragmentMemoryReservation fragment = + fragments == null ? null : fragments.get(fragmentInstanceId); + if (fragment == null) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_MEMORY_RESERVATION_IS_NOT_REGISTERED_FOR_QUERY_ARG_FRAGMENT_ARG_PLAN_NODE_ARG_67A4B81F, + queryId, + fragmentInstanceId, + planNodeId)); + } + return fragment; + } + + private PlanNodeReservation requireReservation( + PlanNodeReservation reservation, + String queryId, + String fragmentInstanceId, + String planNodeId) { + if (reservation == null) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_MEMORY_RESERVATION_IS_NOT_REGISTERED_FOR_QUERY_ARG_FRAGMENT_ARG_PLAN_NODE_ARG_67A4B81F, + queryId, + fragmentInstanceId, + planNodeId)); + } + return reservation; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java new file mode 100644 index 0000000000000..a59ef20d26848 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.memory; + +/** The exchange budget charged by a registered plan node. */ +public enum MemorySide { + INPUT, + OUTPUT; + + public long getMaxBytes(long fragmentExchangeMaxBytes) { + long inputMaxBytes = fragmentExchangeMaxBytes / 2; + return this == INPUT ? inputMaxBytes : fragmentExchangeMaxBytes - inputMaxBytes; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index cb2f6f02acb57..fd539be7467e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -343,7 +343,7 @@ public Operator visitIdentitySink(IdentitySinkNode node, LocalExecutionPlanConte Operator child = node.getChildren().get(0).accept(this, context); List children = new ArrayList<>(1); children.add(child); - sinkHandle.setMaxBytesCanReserve(context.getMaxBytesOneHandleCanReserve()); + sinkHandle.setMaxBytesCanReserve(context.getOutputMaxBytes()); context.getDriverContext().setSink(sinkHandle); return new IdentitySinkOperator( operatorContext, children, downStreamChannelIndex, sinkHandle); @@ -385,7 +385,7 @@ public Operator visitTableExchange(ExchangeNode node, LocalExecutionPlanContext if (!isSameNode) { context.addExchangeSumNum(1); } - sourceHandle.setMaxBytesCanReserve(context.getMaxBytesOneHandleCanReserve()); + sourceHandle.setMaxBytesCanReserve(context.getInputMaxBytes()); ExchangeOperator exchangeOperator = new ExchangeOperator(operatorContext, sourceHandle, node.getUpstreamPlanNodeId()); context.addExchangeOperator(exchangeOperator); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java index 7a5f52749aa4a..bfed28ccc4ca4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java @@ -32,8 +32,10 @@ import org.apache.iotdb.db.queryengine.execution.driver.DriverContext; import org.apache.iotdb.db.queryengine.execution.driver.SchemaDriverContext; import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISink; +import org.apache.iotdb.db.queryengine.execution.exchange.source.PipelineSourceHandle; import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; import org.apache.iotdb.db.queryengine.plan.analyze.TemplatedInfo; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; @@ -47,8 +49,6 @@ import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.annotation.Nullable; @@ -70,7 +70,6 @@ // Attention: We should use thread-safe data structure for members that are shared by all pipelines public class LocalExecutionPlanContext implements ITableOperatorGeneratorContext { - private static final Logger LOGGER = LoggerFactory.getLogger(LocalExecutionPlanContext.class); // Save operators in this pipeline, a new one will be created when creating another pipeline private final DriverContext driverContext; private final AtomicInteger nextOperatorId; @@ -83,6 +82,8 @@ public class LocalExecutionPlanContext implements ITableOperatorGeneratorContext private List pipelineDriverFactories; private List exchangeOperatorList = new ArrayList<>(); private int exchangeSumNum = 0; + private final long inputMaxBytes; + private final long outputMaxBytes; private List cachedDataTypes; @@ -110,6 +111,10 @@ public LocalExecutionPlanContext( TypeProvider typeProvider, FragmentInstanceContext instanceContext, DataNodeQueryContext dataNodeQueryContext) { + long fragmentExchangeMaxBytes = + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); + this.inputMaxBytes = MemorySide.INPUT.getMaxBytes(fragmentExchangeMaxBytes); + this.outputMaxBytes = MemorySide.OUTPUT.getMaxBytes(fragmentExchangeMaxBytes); this.typeProvider = typeProvider; this.allSensorsMap = new ConcurrentHashMap<>(); this.nextOperatorId = new AtomicInteger(0); @@ -121,6 +126,8 @@ public LocalExecutionPlanContext( // For creating subContext, differ from parent context mainly in driver context public LocalExecutionPlanContext(LocalExecutionPlanContext parentContext) { + this.inputMaxBytes = parentContext.inputMaxBytes; + this.outputMaxBytes = parentContext.outputMaxBytes; this.nextOperatorId = parentContext.nextOperatorId; this.typeProvider = parentContext.typeProvider; this.allSensorsMap = parentContext.allSensorsMap; @@ -142,6 +149,10 @@ public LocalExecutionPlanContext( final TypeProvider typeProvider, final FragmentInstanceContext instanceContext, final ISchemaRegion schemaRegion) { + long fragmentExchangeMaxBytes = + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); + this.inputMaxBytes = MemorySide.INPUT.getMaxBytes(fragmentExchangeMaxBytes); + this.outputMaxBytes = MemorySide.OUTPUT.getMaxBytes(fragmentExchangeMaxBytes); this.allSensorsMap = new ConcurrentHashMap<>(); this.typeProvider = typeProvider; this.nextOperatorId = new AtomicInteger(0); @@ -247,10 +258,12 @@ public void setExchangeSumNum(int exchangeSumNum) { this.exchangeSumNum = exchangeSumNum; } - public long getMaxBytesOneHandleCanReserve() { - long maxBytesPerFI = - IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); - return exchangeSumNum == 0 ? maxBytesPerFI : maxBytesPerFI / exchangeSumNum; + public long getInputMaxBytes() { + return inputMaxBytes; + } + + public long getOutputMaxBytes() { + return outputMaxBytes; } public void addExchangeSumNum(int addValue) { @@ -261,16 +274,16 @@ public void addExchangeOperator(ExchangeOperator exchangeOperator) { this.exchangeOperatorList.add(exchangeOperator); } - public void setMaxBytesOneHandleCanReserve() { - long maxBytesOneHandleCanReserve = getMaxBytesOneHandleCanReserve(); - LOGGER.debug( - DataNodeQueryMessages - .MAXBYTESONEHANDLECANRESERVE_FOR_EXCHANGEOPERATOR_IS_ARG_EXCHANGESUMNUM_IS_ARG, - maxBytesOneHandleCanReserve, - exchangeSumNum); + public void setExchangeMemoryLimits() { exchangeOperatorList.forEach( - exchangeOperator -> - exchangeOperator.getSourceHandle().setMaxBytesCanReserve(maxBytesOneHandleCanReserve)); + exchangeOperator -> { + exchangeOperator + .getSourceHandle() + .setMaxBytesCanReserve( + exchangeOperator.getSourceHandle() instanceof PipelineSourceHandle + ? outputMaxBytes + : inputMaxBytes); + }); } public Set getAllSensors(IDeviceID deviceId, String sensorId) { @@ -303,6 +316,7 @@ public void setISink(ISink sink) { checkArgument( driverContext.getSink() == null, DataNodeQueryMessages.EXCEPTION_THERE_MUST_BE_AT_MOST_ONE_SINKNODE_A965AFE7); + sink.setMaxBytesCanReserve(outputMaxBytes); driverContext.setSink(sink); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java index feb6b0cb74afd..1a85cb148c4f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java @@ -133,7 +133,7 @@ public List plan( context.getTimePartitions().ifPresent(instanceContext::setTimePartitions); // set maxBytes one SourceHandle can reserve after visiting the whole tree - context.setMaxBytesOneHandleCanReserve(); + context.setExchangeMemoryLimits(); return context.getPipelineDriverFactories(); } @@ -166,7 +166,7 @@ public List plan( context.addPipelineDriverFactory(root, context.getDriverContext(), 0); // set maxBytes one SourceHandle can reserve after visiting the whole tree - context.setMaxBytesOneHandleCanReserve(); + context.setExchangeMemoryLimits(); return context.getPipelineDriverFactories(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index b6a57db5445d9..d643ec03752e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -2577,7 +2577,7 @@ public Operator visitExchange(ExchangeNode node, LocalExecutionPlanContext conte if (!isSameNode) { context.addExchangeSumNum(1); } - sourceHandle.setMaxBytesCanReserve(context.getMaxBytesOneHandleCanReserve()); + sourceHandle.setMaxBytesCanReserve(context.getInputMaxBytes()); ExchangeOperator exchangeOperator = new ExchangeOperator(operatorContext, sourceHandle, node.getUpstreamPlanNodeId()); context.addExchangeOperator(exchangeOperator); @@ -2616,7 +2616,7 @@ public Operator visitIdentitySink(IdentitySinkNode node, LocalExecutionPlanConte node.getPlanNodeId().getId(), context.getInstanceContext()); List children = dealWithConsumeChildrenOneByOneNode(node, context); - sinkHandle.setMaxBytesCanReserve(context.getMaxBytesOneHandleCanReserve()); + sinkHandle.setMaxBytesCanReserve(context.getOutputMaxBytes()); context.getDriverContext().setSink(sinkHandle); return new IdentitySinkOperator(operatorContext, children, downStreamChannelIndex, sinkHandle); @@ -2658,7 +2658,7 @@ public Operator visitShuffleSink(ShuffleSinkNode node, LocalExecutionPlanContext localInstanceId.toThrift(), node.getPlanNodeId().getId(), context.getInstanceContext()); - sinkHandle.setMaxBytesCanReserve(context.getMaxBytesOneHandleCanReserve()); + sinkHandle.setMaxBytesCanReserve(context.getOutputMaxBytes()); context.getDriverContext().setSink(sinkHandle); return new ShuffleHelperOperator(operatorContext, children, downStreamChannelIndex, sinkHandle); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 66f6da892f3e8..d7461431824e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -630,11 +630,27 @@ public List visitTopK(TopKNode node, PlanContext context) { if (canTopKEliminated(node.getOrderingScheme(), node.getCount(), childrenNodes.get(0))) { return childrenNodes; } - node.setChildren(Collections.singletonList(childrenNodes.get(0))); - return Collections.singletonList(node); + TopKNode physical = + new TopKNode( + node.getPlanNodeId(), + childrenNodes, + node.getOrderingScheme(), + node.getCount(), + node.getOutputSymbols(), + canSortEliminated( + node.getOrderingScheme(), + nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()))); + physical.setTopKRuntimeFilterSourceId(node.getTopKRuntimeFilterSourceId()); + return Collections.singletonList(physical); } - TopKNode newTopKNode = (TopKNode) node.clone(); + TopKNode newTopKNode = + new TopKNode( + node.getPlanNodeId(), + node.getOrderingScheme(), + node.getCount(), + node.getOutputSymbols(), + true); // root TopK is not a runtime filter producer. newTopKNode.setTopKRuntimeFilterSourceId(null); for (PlanNode child : childrenNodes) { @@ -649,7 +665,8 @@ public List visitTopK(TopKNode node, PlanContext context) { node.getOrderingScheme(), node.getCount(), node.getOutputSymbols(), - node.isChildrenDataInOrder()); + canSortEliminated( + node.getOrderingScheme(), nodeOrderingMap.get(child.getPlanNodeId()))); regionTopK.setTopKRuntimeFilterSourceId(node.getTopKRuntimeFilterSourceId()); newChild = regionTopK; } @@ -689,6 +706,12 @@ public List visitGroup(GroupNode node, PlanContext context) { for (PlanNode child : childrenNodes) { if (canSortEliminated(node.getOrderingScheme(), nodeOrderingMap.get(child.getPlanNodeId()))) { result.add(child); + } else if (groupingInvalidatedByCollect( + child, node.getOrderingScheme().getOrderBy().subList(0, node.getPartitionKeyCount()))) { + SortNode sort = + new SortNode(queryId.genPlanNodeId(), child, node.getOrderingScheme(), false, false); + nodeOrderingMap.put(sort.getPlanNodeId(), sort.getOrderingScheme()); + result.add(sort); } else { GroupNode subGroupNode = new GroupNode( @@ -1100,7 +1123,7 @@ public List visitExternalTsFileAggregationScan( if (needFinalAggregation) { OrderingScheme childOrdering = nodeOrderingMap.get(result.get(0).getPlanNodeId()); finalAggregation.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, result)); - return Collections.singletonList(finalAggregation); + return Collections.singletonList(validateAggregationGrouping(finalAggregation)); } return result; } @@ -2045,7 +2068,7 @@ public List visitAggregation(AggregationNode node, PlanContext context } List childrenNodes = node.getChild().accept(this, context); - OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); + OrderingScheme childOrdering = commonOrdering(childrenNodes); if (node.isStreamable()) { // Child has Ordering, we need to check if it is the Ordering we expected if (childOrdering != null) { @@ -2118,7 +2141,7 @@ public List visitAggregation(AggregationNode node, PlanContext context if (physicalAggregation.isStreamable() && childOrdering != null) { nodeOrderingMap.put(planNodeId, expectedOrderingSchema); } - return aggregationNode; + return validateAggregationGrouping(aggregationNode); }) .collect(Collectors.toList()); @@ -2128,10 +2151,10 @@ public List visitAggregation(AggregationNode node, PlanContext context new CollectNode(queryId.genPlanNodeId(), aggregationNodes.get(0).getOutputSymbols()); collectNode.setChildren(aggregationNodes); splitResult.left.setChild(collectNode); - return Collections.singletonList(splitResult.left); + return Collections.singletonList(validateAggregationGrouping(splitResult.left)); } - return Collections.singletonList(physicalAggregation); + return Collections.singletonList(validateAggregationGrouping(physicalAggregation)); } // We cannot do multi-stage Aggregate if any aggregation-function is distinct. @@ -2144,7 +2167,9 @@ public List visitAggregation(AggregationNode node, PlanContext context nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()), childrenNodes); node.setChild(physicalChild); return Collections.singletonList( - withRateFunctionInputOrdering(node, nodeOrderingMap.get(physicalChild.getPlanNodeId()))); + validateAggregationGrouping( + withRateFunctionInputOrdering( + node, nodeOrderingMap.get(physicalChild.getPlanNodeId())))); } Pair splitResult = split(node, symbolAllocator, queryId); AggregationNode intermediate = splitResult.right; @@ -2167,13 +2192,73 @@ public List visitAggregation(AggregationNode node, PlanContext context if (node.isStreamable() && childOrdering != null) { nodeOrderingMap.put(planNodeId, expectedOrderingSchema); } - return aggregationNode; + return validateAggregationGrouping(aggregationNode); }) .collect(Collectors.toList()); splitResult.left.setChild( mergeChildrenViaCollectOrMergeSort( nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()), childrenNodes)); - return Collections.singletonList(splitResult.left); + return Collections.singletonList(validateAggregationGrouping(splitResult.left)); + } + + private AggregationNode validateAggregationGrouping(AggregationNode node) { + if (!node.getPreGroupedSymbols().isEmpty() + && groupingInvalidatedByCollect(node.getChild(), node.getPreGroupedSymbols())) { + node.setPreGroupedSymbols(ImmutableList.of()); + nodeOrderingMap.remove(node.getPlanNodeId()); + } + return node; + } + + /** Whether an unordered fan-in invalidates a previously established grouping property. */ + private boolean groupingInvalidatedByCollect(PlanNode node, List grouping) { + if (grouping.isEmpty()) { + return false; + } + if ((node instanceof CollectNode || node instanceof UnionNode) + && node.getChildren().size() > 1) { + return true; + } + OrderingScheme ordering = nodeOrderingMap.get(node.getPlanNodeId()); + if (ordering != null && prefixMatched(ordering, grouping)) { + return false; + } + // A completed aggregation emits each full grouping key once, even when it hashes its input. + if (node instanceof AggregationNode) { + AggregationNode aggregation = (AggregationNode) node; + if (!aggregation.getStep().isOutputPartial() + && aggregation.getGroupingSetCount() == 1 + && grouping.containsAll(aggregation.getGroupingKeys())) { + return false; + } + } + if (node instanceof ProjectNode) { + ProjectNode project = (ProjectNode) node; + List inputs = new ArrayList<>(); + for (Symbol symbol : grouping) { + Expression expression = project.getAssignments().get(symbol); + if (!(expression instanceof SymbolReference)) { + // A computed key cannot inherit a branch-local continuity proof. + return containsUnorderedCollect(project.getChild()); + } + inputs.add(Symbol.from(expression)); + } + return groupingInvalidatedByCollect(project.getChild(), inputs); + } + for (PlanNode child : node.getChildren()) { + if (groupingInvalidatedByCollect(child, grouping)) { + return true; + } + } + return false; + } + + private boolean containsUnorderedCollect(PlanNode node) { + if ((node instanceof CollectNode || node instanceof UnionNode) + && node.getChildren().size() > 1) { + return true; + } + return node.getChildren().stream().anyMatch(this::containsUnorderedCollect); } private static AggregationNode withRateFunctionInputOrdering( @@ -2301,7 +2386,8 @@ public List visitAggregationTableScan( throw new IllegalStateException( DataNodeQueryMessages.LIST_PLANNODE_SIZE_SHOULD_1_BUT_NOW_IS); } - resultTableScanNodeList = Collections.singletonList(finalAggregation); + resultTableScanNodeList = + Collections.singletonList(validateAggregationGrouping(finalAggregation)); } return resultTableScanNodeList; @@ -2460,7 +2546,7 @@ private List constructSpilledAggregationTableScanByRegionReplicaSet( mergeChildrenViaCollectOrMergeSort( nodeOrderingMap.get(result.get(0).getPlanNodeId()), result)); } - return Collections.singletonList(finalAggregation); + return Collections.singletonList(validateAggregationGrouping(finalAggregation)); } return result; } @@ -2677,7 +2763,8 @@ public List visitAggregationTreeDeviceViewScan( throw new IllegalStateException( DataNodeQueryMessages.LIST_PLANNODE_SIZE_SHOULD_1_BUT_NOW_IS); } - resultTableScanNodeList = Collections.singletonList(finalAggregation); + resultTableScanNodeList = + Collections.singletonList(validateAggregationGrouping(finalAggregation)); } return resultTableScanNodeList; @@ -2863,7 +2950,7 @@ private List constructSpilledAggregationTreeDeviceViewScanByRegionRepl mergeChildrenViaCollectOrMergeSort( nodeOrderingMap.get(result.get(0).getPlanNodeId()), result)); } - return Collections.singletonList(finalAggregation); + return Collections.singletonList(validateAggregationGrouping(finalAggregation)); } return result; } @@ -4007,6 +4094,9 @@ public NodeDistribution getNodeDistribution(PlanNodeId nodeId) { public void clearExpectedOrderingScheme() { expectedOrderingScheme = null; hasSortProperty = false; + // Grouping pushdown depends on this ordering. A boundary such as a global window + // must not pass an outer Group's partitioning request to its scans. + pushDownGrouping = false; } public void setExpectedOrderingScheme(OrderingScheme expectedOrderingScheme) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java index 66c4eff6e3e03..ac3fc65fe6683 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGrouping.java @@ -34,6 +34,7 @@ import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TableFunctionProcessorNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; @@ -101,6 +102,21 @@ public PlanNode visitPlan(PlanNode node, Context context) { return newNode; } + @Override + public PlanNode visitUnion(UnionNode node, Context context) { + if (node.getChildren().size() <= 1) { + return visitPlan(node, context); + } + if (!context.canSkip()) { + context.canOptimized = TO_SORT; + } + // Neither branch may establish a property for the interleaved Union output. + return node.replaceChildren( + node.getChildren().stream() + .map(child -> child.accept(this, new Context(null, 0))) + .collect(Collectors.toList())); + } + /** We need to make sure: context#partitionKey can match the prefix of childOrderSchema */ private void checkPrefixMatch(Context context, List childOrder) { if (context.canSkip()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java index 64ef1b8c2adc6..c35f66128052e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.OrderingScheme; import org.apache.iotdb.commons.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.FillNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.PatternRecognitionNode; @@ -31,6 +32,7 @@ import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TableFunctionProcessorNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKRankingNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneTableScanColumns; @@ -38,6 +40,7 @@ import com.google.common.collect.ImmutableSet; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -75,6 +78,28 @@ public PlanNode visitPlan(PlanNode node, Context context) { return newNode; } + @Override + public PlanNode visitCollect(CollectNode node, Context context) { + return visitUnorderedFanIn(node, context); + } + + @Override + public PlanNode visitUnion(UnionNode node, Context context) { + return visitUnorderedFanIn(node, context); + } + + private PlanNode visitUnorderedFanIn(PlanNode node, Context context) { + if (node.getChildren().size() <= 1) { + return visitPlan(node, context); + } + List children = new ArrayList<>(); + for (PlanNode child : node.getChildren()) { + children.add(child.accept(this, new Context())); + } + context.addCannotEliminateSort(true); + return node.replaceChildren(children); + } + @Override public PlanNode visitProject(ProjectNode node, Context context) { Context newContext = new Context(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java index ce49104248123..a5573159a94e5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java @@ -40,9 +40,11 @@ import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine; import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; +import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; import org.apache.iotdb.mpp.rpc.thrift.TNewDataBlockEvent; +import org.apache.tsfile.read.common.block.TsBlock; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -180,6 +182,13 @@ public void testCreateLocalSinkHandle() { Assert.assertEquals( ((LocalSinkChannel) shuffleSinkHandle.getChannel(0)).getSharedTsBlockQueue(), ((LocalSourceHandle) localSourceHandle).getSharedTsBlockQueue()); + assertProducerOwnsQueue( + spyMemoryPool, + shuffleSinkHandle, + localSourceHandle, + localFragmentInstanceId, + remoteFragmentInstanceId, + localPlanNodeId); } @Test @@ -237,5 +246,49 @@ public void testCreateLocalSourceHandle() { Assert.assertEquals( ((LocalSinkChannel) shuffleSinkHandle.getChannel(0)).getSharedTsBlockQueue(), ((LocalSourceHandle) localSourceHandle).getSharedTsBlockQueue()); + assertProducerOwnsQueue( + spyMemoryPool, + shuffleSinkHandle, + localSourceHandle, + localFragmentInstanceId, + remoteFragmentInstanceId, + localPlanNodeId); + } + + private static void assertProducerOwnsQueue( + MemoryPool pool, + ISinkHandle sink, + ISourceHandle source, + TFragmentInstanceId producer, + TFragmentInstanceId consumer, + String planNodeId) { + SharedTsBlockQueue queue = ((LocalSourceHandle) source).getSharedTsBlockQueue(); + sink.setMaxBytesCanReserve(512); + source.setMaxBytesCanReserve(128); + Assert.assertEquals(512, queue.getMaxBytesCanReserve()); + String producerId = + FragmentInstanceId.createFragmentInstanceIdFromTFragmentInstanceId(producer); + String consumerId = + FragmentInstanceId.createFragmentInstanceIdFromTFragmentInstanceId(consumer); + TsBlock block = Utils.createMockTsBlock(256); + synchronized (queue) { + Assert.assertTrue(queue.add(block).isDone()); + Assert.assertEquals(256, pool.getReservedBytes()); + Assert.assertEquals( + 256, pool.getPlanNodeMemoryReservedBytes(producer.queryId, producerId, planNodeId)); + Assert.assertEquals( + 256, + pool.getFragmentMemoryReservedBytes(producer.queryId, producerId, MemorySide.OUTPUT)); + Assert.assertEquals( + 0, pool.getFragmentMemoryReservedBytes(producer.queryId, producerId, MemorySide.INPUT)); + Assert.assertEquals( + 0, pool.getFragmentMemoryReservedBytes(consumer.queryId, consumerId, MemorySide.INPUT)); + Assert.assertEquals( + 0, pool.getFragmentMemoryReservedBytes(consumer.queryId, consumerId, MemorySide.OUTPUT)); + Assert.assertSame(block, queue.remove()); + Assert.assertEquals(0, pool.getReservedBytes()); + Assert.assertEquals( + 0, pool.getFragmentMemoryReservedBytes(producer.queryId, producerId, MemorySide.OUTPUT)); + } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java index 2d9e5683cfdfb..3cfebb1ba295c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java @@ -66,9 +66,10 @@ public class SourceHandleTest { public static void beforeClass() { maxBytesPerFI = IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); + // INPUT receives half of the fragment budget, allowing five buffered blocks. IoTDBDescriptor.getInstance() .getMemoryConfig() - .setMaxBytesPerFragmentInstance(5 * MOCK_TSBLOCK_SIZE); + .setMaxBytesPerFragmentInstance(10 * MOCK_TSBLOCK_SIZE); } @AfterClass diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index 5f2285e4ca910..19fc0f17d3367 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -50,7 +50,8 @@ public class MemoryPoolTest { public void before() { MemoryManager memoryManager = Mockito.spy(new MemoryManager(1024L)); pool = new MemoryPool("test", memoryManager, 512L); - pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID); + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, MemorySide.INPUT); } @Test @@ -87,7 +88,7 @@ public void testDeregisterDoesNotRemoveReregisteredQueryMemory() throws Exceptio // Replace the query map while the delayed cleanup still refers to the old one. pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); Assert.assertEquals(0, pool.getQueryMemoryReservationSize()); - pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "result", PLAN_NODE_ID); + pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "result", PLAN_NODE_ID, MemorySide.INPUT); Assert.assertTrue( pool.tryReserveForTest(QUERY_ID, "result", PLAN_NODE_ID, 256L, Long.MAX_VALUE)); } @@ -123,7 +124,7 @@ public void testDeregisterMissingFragmentPreservesReservedMemory() { @Test public void testDeregisterPreservesOtherFragments() { - pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "other", PLAN_NODE_ID); + pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "other", PLAN_NODE_ID, MemorySide.INPUT); Assert.assertTrue( pool.tryReserveForTest(QUERY_ID, "other", PLAN_NODE_ID, 256L, Long.MAX_VALUE)); @@ -518,4 +519,112 @@ public void testReserveWithPriorityShowQueriesWithinAvailable() { Assert.assertTrue(r.getFuture().isDone()); Assert.assertEquals(256L, pool.getReservedBytes()); } + + @Test + public void testSharedInputBudgetAndIndependentOutput() { + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, "output", MemorySide.OUTPUT); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 200, 256, false) + .isReserveSuccess()); + MemoryReservationResult pending = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 100, 256, false); + Assert.assertFalse(pending.isReserveSuccess()); + Assert.assertFalse(pending.getFuture().isDone()); + Assert.assertEquals( + 200, pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.INPUT)); + Assert.assertEquals( + 0, pool.getPlanNodeMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2")); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, "output", 256, 256, false) + .isReserveSuccess()); + Assert.assertEquals(456, pool.getReservedBytes()); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 200); + Assert.assertTrue(pending.getFuture().isDone()); + Assert.assertEquals( + 100, pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.INPUT)); + Assert.assertEquals( + 256, + pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.OUTPUT)); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 100); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, "output", 256); + Assert.assertEquals(0, pool.getReservedBytes()); + Assert.assertEquals(0, pool.getQueryMemoryReservedBytes(QUERY_ID)); + } + + @Test + public void testSideRegistrationCannotChange() { + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, MemorySide.INPUT); + try { + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, MemorySide.OUTPUT); + Assert.fail("conflicting side must be rejected"); + } catch (IllegalArgumentException expected) { + Assert.assertEquals(0, pool.getReservedBytes()); + } + } + + @Test + public void testOddBudgetAndSingleRequestBoundary() { + Assert.assertEquals(256, MemorySide.INPUT.getMaxBytes(513)); + Assert.assertEquals(257, MemorySide.OUTPUT.getMaxBytes(513)); + try { + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 257, 256, false); + Assert.fail("request exceeds the input budget"); + } catch (IllegalArgumentException expected) { + Assert.assertEquals(0, pool.getReservedBytes()); + Assert.assertEquals(0, pool.getQueryMemoryReservedBytes(QUERY_ID)); + } + } + + @Test + public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { + java.util.concurrent.ExecutorService executor = + java.util.concurrent.Executors.newFixedThreadPool(16); + java.util.concurrent.CountDownLatch start = new java.util.concurrent.CountDownLatch(1); + java.util.concurrent.CountDownLatch attempted = new java.util.concurrent.CountDownLatch(16); + java.util.concurrent.CountDownLatch release = new java.util.concurrent.CountDownLatch(1); + java.util.List> tasks = new java.util.ArrayList<>(); + try { + for (int i = 0; i < 16; i++) { + String id = "pipeline-" + i; + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, id, MemorySide.INPUT); + tasks.add( + executor.submit( + () -> { + start.await(); + boolean reserved = + pool.tryReserveForTest(QUERY_ID, FRAGMENT_INSTANCE_ID, id, 32, 128); + attempted.countDown(); + release.await(); + if (reserved) pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, id, 32); + return null; + })); + } + start.countDown(); + Assert.assertTrue(attempted.await(10, TimeUnit.SECONDS)); + long details = 0; + for (int i = 0; i < 16; i++) { + details += + pool.getPlanNodeMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, "pipeline-" + i); + } + Assert.assertTrue(details > 0 && details <= 128); + Assert.assertEquals( + details, + pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.INPUT)); + Assert.assertEquals(details, pool.getReservedBytes()); + release.countDown(); + for (java.util.concurrent.Future task : tasks) task.get(10, TimeUnit.SECONDS); + Assert.assertEquals(0, pool.getReservedBytes()); + Assert.assertEquals(0, pool.getQueryMemoryReservedBytes(QUERY_ID)); + } finally { + start.countDown(); + release.countDown(); + executor.shutdownNow(); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java new file mode 100644 index 0000000000000..68bff035fe2c3 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; +import org.apache.iotdb.db.queryengine.common.PlanFragmentId; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISink; +import org.apache.iotdb.db.queryengine.execution.exchange.source.ISourceHandle; +import org.apache.iotdb.db.queryengine.execution.exchange.source.PipelineSourceHandle; +import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; +import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class LocalExecutionPlanContextBudgetTest { + @Test + public void testFixedBudgetsInheritedByPipelinesAndWiredBySide() { + long previous = + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); + try { + IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(513); + FragmentInstanceContext instance = mock(FragmentInstanceContext.class); + when(instance.getId()) + .thenReturn(new FragmentInstanceId(new PlanFragmentId(new QueryId("budget"), 0), "0")); + LocalExecutionPlanContext context = + new LocalExecutionPlanContext(new TypeProvider(), instance, (DataNodeQueryContext) null); + context.setExchangeSumNum(500); + assertEquals(256, context.getInputMaxBytes()); + assertEquals(257, context.getOutputMaxBytes()); + IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(1024); + LocalExecutionPlanContext child = context.createSubContext(); + assertEquals(256, child.getInputMaxBytes()); + assertEquals(257, child.getOutputMaxBytes()); + ISink sink = mock(ISink.class); + child.setISink(sink); + verify(sink).setMaxBytesCanReserve(257); + ISourceHandle remote = mock(ISourceHandle.class); + PipelineSourceHandle pipeline = mock(PipelineSourceHandle.class); + ExchangeOperator remoteOperator = mock(ExchangeOperator.class); + ExchangeOperator pipelineOperator = mock(ExchangeOperator.class); + when(remoteOperator.getSourceHandle()).thenReturn(remote); + when(pipelineOperator.getSourceHandle()).thenReturn(pipeline); + context.addExchangeOperator(remoteOperator); + child.addExchangeOperator(pipelineOperator); + context.setExchangeMemoryLimits(); + verify(remote).setMaxBytesCanReserve(256); + verify(pipeline).setMaxBytesCanReserve(257); + } finally { + IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(previous); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java new file mode 100644 index 0000000000000..3021363139c72 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner; + +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GroupNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.StreamSortNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.WindowNode; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class CollectInputPropertyTest { + private static PlanNode plan(String sql) { + PlanTester tester = new PlanTester(); + tester.createPlan(sql); + return tester.getFragmentPlan(0); + } + + private static List nodes(PlanNode node, Class type) { + List result = new ArrayList<>(); + if (type.isInstance(node)) result.add(type.cast(node)); + for (PlanNode child : node.getChildren()) result.addAll(nodes(child, type)); + return result; + } + + @Test + public void testAggregationAboveGlobalWindowHashesInterleavedGroups() { + for (String aggregate : new String[] {"SUM(s1)", "COUNT(DISTINCT s1)"}) { + PlanNode root = + plan( + "SELECT tag1, tag2, tag3, " + + aggregate + + " FROM (" + + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " + + "FROM table1 WHERE tag1 != 'shenzhen') t WHERE n > 0 GROUP BY tag1, tag2, tag3"); + List aggregations = nodes(root, AggregationNode.class); + assertFalse(aggregations.isEmpty()); + assertFalse(nodes(root, CollectNode.class).isEmpty()); + for (AggregationNode aggregation : aggregations) { + if (!nodes(aggregation.getChild(), CollectNode.class).isEmpty()) { + assertTrue(aggregation.getPreGroupedSymbols().isEmpty()); + assertFalse(aggregation.isStreamable()); + } + } + assertTrue( + "Hash aggregation should not add a grouping sort", nodes(root, SortNode.class).isEmpty()); + } + } + + @Test + public void testPartialPreGroupingAlsoFallsBackToHash() { + PlanNode root = + plan( + "SELECT tag1, tag2, tag3, s1, COUNT(*) FROM (" + + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " + + "FROM table1 WHERE tag1 != 'shenzhen') t WHERE n > 0 GROUP BY tag1, tag2, tag3, s1"); + List aggregations = nodes(root, AggregationNode.class); + assertFalse(aggregations.isEmpty()); + assertFalse(nodes(root, CollectNode.class).isEmpty()); + assertTrue(aggregations.get(0).getPreGroupedSymbols().isEmpty()); + } + + @Test + public void testNestedWindowRestoresPartitionContiguityWithSort() { + PlanNode root = + plan( + "SELECT tag1, tag2, tag3, SUM(s1) OVER (" + + "PARTITION BY tag1, tag2, tag3 ORDER BY s1) AS running_sum " + + "FROM (SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n FROM table1) t WHERE n > 0"); + assertFalse(nodes(root, CollectNode.class).isEmpty()); + assertTrue(nodes(root, GroupNode.class).isEmpty()); + List windows = nodes(root, WindowNode.class); + assertEquals(2, windows.size()); + WindowNode outer = + windows.stream() + .filter(w -> !w.getSpecification().getPartitionBy().isEmpty()) + .findFirst() + .get(); + assertFalse(nodes(outer.getChild(), SortNode.class).isEmpty()); + } + + @Test + public void testStreamSortAfterGlobalWindowUsesFullSort() { + PlanNode root = + plan( + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " + + "FROM table1 ORDER BY tag1, tag2, tag3, s1"); + assertFalse(nodes(root, CollectNode.class).isEmpty()); + assertFalse(nodes(root, SortNode.class).isEmpty()); + assertTrue(nodes(root, StreamSortNode.class).isEmpty()); + } + + @Test + public void testConstantFillDoesNotRemoveRestoredSort() { + PlanNode root = + plan( + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n FROM table1 " + + "FILL METHOD CONSTANT 0 ORDER BY tag1, tag2, tag3"); + assertFalse(nodes(root, CollectNode.class).isEmpty()); + assertFalse(nodes(root, SortNode.class).isEmpty()); + assertTrue(nodes(root, StreamSortNode.class).isEmpty()); + } + + @Test + public void testUnionFinalAggregationDoesNotInheritBranchGrouping() { + PlanNode root = + plan( + "SELECT tag1, tag2, tag3, SUM(s1) FROM (" + + "SELECT tag1, tag2, tag3, s1 FROM table1 UNION ALL " + + "SELECT tag1, tag2, tag3, s1 FROM table2) u GROUP BY tag1, tag2, tag3"); + List aggregations = nodes(root, AggregationNode.class); + assertFalse(aggregations.isEmpty()); + boolean foundFinalCollect = false; + for (AggregationNode aggregation : aggregations) { + if (aggregation.getChild() instanceof CollectNode) { + foundFinalCollect = true; + assertTrue(aggregation.getPreGroupedSymbols().isEmpty()); + } + } + assertTrue(foundFinalCollect); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGroupingTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGroupingTest.java index e2326688112b2..b5b69f7663f03 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGroupingTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGroupingTest.java @@ -26,6 +26,9 @@ import org.apache.iotdb.commons.queryengine.plan.relational.planner.OrderingScheme; import org.apache.iotdb.commons.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GroupNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.ValuesNode; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.execution.warnings.WarningCollector; @@ -36,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExternalTsFileScanNode; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import org.junit.Test; @@ -108,4 +112,54 @@ region, new ColumnSchema("region", STRING, false, TAG), resource.closeByQueryExecution(); } } + + @Test + public void testUnionDoesNotInheritFirstBranchGrouping() { + Symbol key = new Symbol("key"); + OrderingScheme ordering = + new OrderingScheme(ImmutableList.of(key), ImmutableMap.of(key, ASC_NULLS_LAST)); + PlanNode left = + new SortNode( + new PlanNodeId("leftSort"), + new ValuesNode(new PlanNodeId("left"), ImmutableList.of(key), ImmutableList.of()), + ordering, + false, + false); + PlanNode right = + new SortNode( + new PlanNodeId("rightSort"), + new ValuesNode(new PlanNodeId("right"), ImmutableList.of(key), ImmutableList.of()), + ordering, + false, + false); + UnionNode union = + new UnionNode( + new PlanNodeId("union"), + ImmutableList.of(left, right), + ImmutableListMultimap.of(key, key, key, key), + ImmutableList.of(key)); + assertTrue( + optimizeGroup(new GroupNode(new PlanNodeId("group"), union, ordering, 1)) + instanceof SortNode); + // An explicit sort above the fan-in restores the grouping property. + SortNode restored = new SortNode(new PlanNodeId("restored"), union, ordering, false, false); + assertTrue( + optimizeGroup(new GroupNode(new PlanNodeId("group2"), restored, ordering, 1)) + instanceof GroupNode); + } + + private PlanNode optimizeGroup(GroupNode group) { + return new ParallelizeGrouping() + .optimize( + group, + new PlanOptimizer.Context( + SESSION_INFO, + new Analysis(null, Collections.emptyMap()), + TEST_MATADATA, + new MPPQueryContext(new QueryId("group_union")), + new SymbolAllocator(), + new QueryId("group_union"), + WarningCollector.NOOP, + PlanOptimizersStatsCollector.createPlanOptimizersStatsCollector())); + } } From 04c6542f0abe6886136590c9cea95db68ddeda84 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 14 Oct 2026 14:53:56 +0800 Subject: [PATCH 03/29] Fix exchange reservation wakeup race and strengthen fan-in tests --- ...2026-10-14-exchange-fanin-budget-design.md | 22 ++- .../it/query/recent/IoTDBCollectFanInIT.java | 23 +++ .../operator/process/CollectOperatorTest.java | 102 ++++++++++++ .../execution/memory/MemoryPool.java | 51 +++--- .../execution/memory/MemoryPoolTest.java | 154 +++++++++++++++++- .../planner/CollectInputPropertyTest.java | 28 ++++ 6 files changed, 353 insertions(+), 27 deletions(-) diff --git a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md index a817039cab4b9..e191622940dc7 100644 --- a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md +++ b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md @@ -72,7 +72,7 @@ SortElimination 在各汇集分支独立优化，并阻止跨无序汇集删除 ## 实现验证记录（2026-10-14） -第一版实现已完成。单元测试 86 项通过；IoTDBCollectFanInIT 的 6 项测试在单节点和三 DataNode 集群中分别通过。 +上述约定范围的实现已完成。单元测试 86 项通过；IoTDBCollectFanInIT 的 6 项测试在单节点和三 DataNode 集群中分别通过。 集成验证包含普通与 DISTINCT 聚合、Union 分阶段聚合、嵌套窗口、Fill、排序/TopK、Join 依赖、树模型以及树表视图。 构建先执行以下干净构建，生成当前分支的节点包并安装本地依赖： @@ -97,3 +97,23 @@ mvn -o verify -DskipUTs -Dit.test=IoTDBCollectFanInIT -DfailIfNoTests=false -Dfa 使用离线模式是为了避免远端 SNAPSHOT ZIP 下载停滞；节点执行的是当前分支重新打包的代码。 Spotless、Checkstyle、新增文件许可证及中英文消息键名/占位符一致性检查通过。 + +## 自查与补测（2026-10-14） + +自查复现了 MemoryPool 的等待申请漏唤醒：一次申请额度不足后，旧申请先释放全部额度， +新申请才进入等待队列；释放线程看不到新申请，导致空闲额度下仍无限等待。 +回归测试用 latch 固定这一时序，在修复前稳定失败。 +修复在申请入队后再次检查额度，并与 free/tryCancel 共用该 future 的锁，避免重复计费。 + +新增 8 项单元测试：等待申请发布竞态、取消不占用共享额度、16 路阻塞申请反复获得额度后账目归零； +空 Collect、全部阻塞时异常唤醒、并发就绪后每路恰好消费一次； +Collect 后 TopK 不误用有序输入，以及有效分支保留流式聚合。 +增强关闭异常测试，检查其他输入仍被关闭且后续异常被保留。 +新增一项集成测试，验证全局窗口后按字段倒序的 TopK 跨设备结果及完整窗口计数。 + +补测后的 118 项定向单测通过（14 个测试类，零失败、零跳过）；全 reactor 中英文编译通过。 +在前述英文回归命令中加入 `OrderingPropertyPropagationTest,TopKRuntimeFilterDistributedPlanTest`， +同时验证有效排序优化与 TopK runtime filter 不受影响。 +重新执行 `mvn -o install -DskipTests -pl integration-test,iotdb-core/confignode,iotdb-client/cli -am -P with-integration-tests` +后运行前述两条 verify 命令；IoTDBCollectFanInIT 的 7 项测试在单节点和三 DataNode 集群中分别全部通过。 +Spotless、Checkstyle、许可证与 `git diff --check` 检查通过。 diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java index adc72b6046e50..8667f01ddfd70 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java @@ -273,4 +273,27 @@ public void testOrderByAfterGlobalWindowAndConstantFill() throws Exception { } } } + + @Test + public void testTopKAfterGlobalWindowConsumesUnorderedInput() throws Exception { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + int limit = DEVICES + 5; + try (ResultSet result = + statement.executeQuery( + "SELECT device, zone, value, COUNT(*) OVER () AS n FROM readings1 " + + "ORDER BY value DESC, device LIMIT " + + limit)) { + for (int row = 0; row < limit; row++) { + assertTrue(result.next()); + assertEquals(device(row % DEVICES), result.getString(1)); + assertEquals(zone(row % DEVICES), result.getString(2)); + assertEquals(ROWS - row / DEVICES, result.getInt(3)); + assertEquals((long) DEVICES * ROWS, result.getLong(4)); + } + assertFalse(result.next()); + } + } + } } diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java index abf7a82acf3d9..b446f9c999b95 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java @@ -36,9 +36,16 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CancellationException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -233,6 +240,7 @@ public void testCloseWakesWaitAndClosesEveryChildDespiteException() throws Excep Child a = new Child(block(1)); Child b = new Child(block(2)); a.closeFailure = new IOException("close failure"); + b.closeFailure = new IOException("second close failure"); CollectOperator collect = collect(a, b); ListenableFuture pending = collect.isBlocked(); try { @@ -240,6 +248,8 @@ public void testCloseWakesWaitAndClosesEveryChildDespiteException() throws Excep fail("must propagate close failure"); } catch (IOException e) { assertSame(a.closeFailure, e); + assertEquals(1, e.getSuppressed().length); + assertSame(b.closeFailure, e.getSuppressed()[0]); } pending.get(5, TimeUnit.SECONDS); assertEquals(1, a.closes); @@ -299,6 +309,98 @@ public void testMemoryIncludesAllLiveRetainedStates() throws Exception { assertEquals(0, collect.calculateRetainedSizeAfterCallingNext()); } + @Test + public void testEmptyCollectIsAlreadyFinished() throws Exception { + CollectOperator collect = collect(); + assertFalse(collect.hasNext()); + assertTrue(collect.isFinished()); + assertTrue(collect.isBlocked().isDone()); + assertNull(collect.next()); + assertEquals(0, collect.calculateMaxPeekMemory()); + assertEquals(0, collect.calculateMaxReturnSize()); + collect.close(); + collect.close(); + } + + @Test + public void testFailureWakesAllPendingChildren() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + CollectOperator collect = collect(a, b); + ListenableFuture pending = collect.isBlocked(); + assertFalse(pending.isDone()); + IOException expected = new IOException("blocked child failure"); + b.wait.delegate.setException(expected); + try { + pending.get(5, TimeUnit.SECONDS); + fail("child failure must wake the driver"); + } catch (ExecutionException e) { + assertSame(expected, e.getCause()); + } + try { + collect.next(); + fail("child failure must propagate without advancing other children"); + } catch (IOException e) { + assertSame(expected, e); + } finally { + collect.close(); + } + assertEquals(0, a.advances); + assertEquals(0, b.advances); + assertEquals(1, a.closes); + assertEquals(1, b.closes); + } + + @Test + public void testConcurrentCompletionsConsumeEveryChildExactlyOnce() throws Exception { + int count = 16; + List children = new ArrayList<>(); + for (int i = 0; i < count; i++) children.add(new Child(block(i))); + CollectOperator collect = new CollectOperator(new TestContext(), new ArrayList<>(children)); + ListenableFuture pending = collect.isBlocked(); + assertFalse(pending.isDone()); + ExecutorService executor = Executors.newFixedThreadPool(count); + CountDownLatch start = new CountDownLatch(1); + List> completions = new ArrayList<>(); + try { + for (Child child : children) { + completions.add( + executor.submit( + () -> { + start.await(); + child.ready(); + return null; + })); + } + start.countDown(); + pending.get(5, TimeUnit.SECONDS); + for (Future completion : completions) completion.get(5, TimeUnit.SECONDS); + Set consumed = new HashSet<>(); + for (int round = 0; round < 2 * count && collect.hasNext(); round++) { + assertTrue(collect.isBlocked().isDone()); + TsBlock result = collect.next(); + if (result != null) { + int childId = result.getColumn(0).getInt(0); + assertTrue("child emitted more than once", consumed.add(childId)); + children.get(childId).hasNext = false; + children.get(childId).finished = true; + } + } + assertEquals(count, consumed.size()); + assertTrue(collect.isFinished()); + assertNull(collect.next()); + for (Child child : children) { + assertEquals(1, child.advances); + assertEquals(1, child.closes); + } + } finally { + start.countDown(); + executor.shutdownNow(); + assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + collect.close(); + } + } + private static final class CountingFuture extends ForwardingListenableFuture.SimpleForwardingListenableFuture { private final SettableFuture delegate; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index 56c9974764a5f..07eced179d94e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -365,10 +365,15 @@ public MemoryReservationResult reserveWithPriority( DataNodeQueryMessages.BLOCKED_RESERVE_REQUEST_ARG_BYTES_MEMORY_FOR_PLANNODEIDARG, bytesToReserve, planNodeId); - ListenableFuture result = + MemoryReservationFuture result = MemoryReservationFuture.create( queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve); - memoryReservationFutures.add((MemoryReservationFuture) result); + memoryReservationFutures.add(result); + // A concurrent free may have missed this request before it was published. Retry after + // publication, under the same future lock used by free and tryCancel to avoid charging twice. + if (tryCompleteReservation(result)) { + memoryReservationFutures.remove(result); + } return new MemoryReservationResult(result, false, bytesToReserve); } } @@ -463,27 +468,35 @@ public void free(String queryId, String fragmentInstanceId, String planNodeId, l Iterator> iterator = memoryReservationFutures.iterator(); while (iterator.hasNext()) { MemoryReservationFuture future = iterator.next(); - synchronized (future) { - if (future.isCancelled() || future.isDone()) { - iterator.remove(); - continue; - } - long bytesToReserve = future.getBytesToReserve(); - String curQueryId = future.getQueryId(); - String curFragmentInstanceId = future.getFragmentInstanceId(); - String curPlanNodeId = future.getPlanNodeId(); - long maxBytesCanReserve = future.getMaxBytesCanReserve(); - if (tryReserve( - curQueryId, curFragmentInstanceId, curPlanNodeId, bytesToReserve, maxBytesCanReserve)) { - future.set(null); - iterator.remove(); - } else { - rollbackReserve(curQueryId, curFragmentInstanceId, curPlanNodeId, bytesToReserve); - } + if (tryCompleteReservation(future)) { + iterator.remove(); } } } + private boolean tryCompleteReservation(MemoryReservationFuture future) { + synchronized (future) { + if (future.isDone()) { + return true; + } + long bytesToReserve = future.getBytesToReserve(); + String queryId = future.getQueryId(); + String fragmentInstanceId = future.getFragmentInstanceId(); + String planNodeId = future.getPlanNodeId(); + if (tryReserve( + queryId, + fragmentInstanceId, + planNodeId, + bytesToReserve, + future.getMaxBytesCanReserve())) { + future.set(null); + return true; + } + rollbackReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve); + return false; + } + } + public long getQueryMemoryReservedBytes(String queryId) { Map fragments = queryMemoryReservations.get(queryId); if (fragments == null) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index 19fc0f17d3367..0254fa1ac33c3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -32,8 +32,15 @@ import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.awaitility.Awaitility.await; @@ -582,12 +589,11 @@ public void testOddBudgetAndSingleRequestBoundary() { @Test public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { - java.util.concurrent.ExecutorService executor = - java.util.concurrent.Executors.newFixedThreadPool(16); - java.util.concurrent.CountDownLatch start = new java.util.concurrent.CountDownLatch(1); - java.util.concurrent.CountDownLatch attempted = new java.util.concurrent.CountDownLatch(16); - java.util.concurrent.CountDownLatch release = new java.util.concurrent.CountDownLatch(1); - java.util.List> tasks = new java.util.ArrayList<>(); + ExecutorService executor = Executors.newFixedThreadPool(16); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch attempted = new CountDownLatch(16); + CountDownLatch release = new CountDownLatch(1); + List> tasks = new ArrayList<>(); try { for (int i = 0; i < 16; i++) { String id = "pipeline-" + i; @@ -618,7 +624,7 @@ public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.INPUT)); Assert.assertEquals(details, pool.getReservedBytes()); release.countDown(); - for (java.util.concurrent.Future task : tasks) task.get(10, TimeUnit.SECONDS); + for (Future task : tasks) task.get(10, TimeUnit.SECONDS); Assert.assertEquals(0, pool.getReservedBytes()); Assert.assertEquals(0, pool.getQueryMemoryReservedBytes(QUERY_ID)); } finally { @@ -627,4 +633,138 @@ public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { executor.shutdownNow(); } } + + @Test + public void testReleaseBeforePendingReservationIsPublished() throws Exception { + pool = Mockito.spy(pool); + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256, 256, false) + .isReserveSuccess()); + CountDownLatch failedAttempt = new CountDownLatch(1); + CountDownLatch released = new CountDownLatch(1); + AtomicBoolean firstAttempt = new AtomicBoolean(true); + Mockito.doAnswer( + invocation -> { + boolean granted = (boolean) invocation.callRealMethod(); + if (!granted && firstAttempt.compareAndSet(true, false)) { + failedAttempt.countDown(); + Assert.assertTrue(released.await(10, TimeUnit.SECONDS)); + } + return granted; + }) + .when(pool) + .tryReserve(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 256, 256); + FutureTask reservation = + new FutureTask<>( + () -> + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 256, 256, false)); + Thread reserver = new Thread(reservation, "reserve-shared-input"); + reserver.start(); + try { + Assert.assertTrue(failedAttempt.await(10, TimeUnit.SECONDS)); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256); + released.countDown(); + MemoryReservationResult result = reservation.get(10, TimeUnit.SECONDS); + Assert.assertTrue( + "Releasing the last owner must wake the pending request", result.getFuture().isDone()); + result.getFuture().get(10, TimeUnit.SECONDS); + Assert.assertEquals(256, pool.getReservedBytes()); + Assert.assertEquals( + 256, + pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.INPUT)); + Assert.assertEquals( + 256, pool.getPlanNodeMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2")); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 256); + Assert.assertEquals(0, pool.getReservedBytes()); + } finally { + released.countDown(); + reserver.join(10000); + Assert.assertFalse(reserver.isAlive()); + } + } + + @Test + public void testCancelledInputDoesNotConsumeReleasedSharedBudget() throws Exception { + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, "cancelled", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, "waiting", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, "output", MemorySide.OUTPUT); + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256, 256, false); + MemoryReservationResult cancelled = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, "cancelled", 128, 256, false); + MemoryReservationResult waiting = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, "waiting", 256, 256, false); + Assert.assertFalse(cancelled.getFuture().isDone()); + Assert.assertFalse(waiting.getFuture().isDone()); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, "output", 128, 256, false) + .isReserveSuccess()); + Assert.assertEquals(128, pool.tryCancel(cancelled.getFuture())); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256); + waiting.getFuture().get(10, TimeUnit.SECONDS); + Assert.assertTrue(cancelled.getFuture().isCancelled()); + Assert.assertEquals( + 0, pool.getPlanNodeMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, "cancelled")); + Assert.assertEquals( + 256, pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.INPUT)); + Assert.assertEquals( + 128, + pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.OUTPUT)); + Assert.assertEquals(384, pool.getReservedBytes()); + Assert.assertEquals(0, pool.getMemoryReservationSize()); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, "waiting", 256); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, "output", 128); + Assert.assertEquals(0, pool.getQueryMemoryReservedBytes(QUERY_ID)); + Assert.assertEquals(0, pool.getReservedBytes()); + } + + @Test + public void testConcurrentPendingReservationsCompleteAndReleaseAllAccounting() throws Exception { + int count = 16; + ExecutorService executor = Executors.newFixedThreadPool(count); + CountDownLatch attempted = new CountDownLatch(count); + List> tasks = new ArrayList<>(); + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 128, 128, false); + try { + for (int i = 0; i < count; i++) { + String id = "waiting-" + i; + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, id, MemorySide.INPUT); + tasks.add( + executor.submit( + () -> { + for (int round = 0; round < 50; round++) { + MemoryReservationResult result = + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, id, 32, 128, false); + if (round == 0) attempted.countDown(); + result.getFuture().get(10, TimeUnit.SECONDS); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, id, result.getReservedBytes()); + } + return null; + })); + } + Assert.assertTrue(attempted.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(count, pool.getMemoryReservationSize()); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 128); + for (Future task : tasks) task.get(30, TimeUnit.SECONDS); + for (int i = 0; i < count; i++) { + Assert.assertEquals( + 0, pool.getPlanNodeMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, "waiting-" + i)); + } + Assert.assertEquals( + 0, pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.INPUT)); + Assert.assertEquals(0, pool.getReservedBytes()); + Assert.assertEquals(0, pool.getQueryMemoryReservedBytes(QUERY_ID)); + Assert.assertEquals(0, pool.getMemoryReservationSize()); + } finally { + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java index 3021363139c72..5bae986bf4762 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GroupNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.StreamSortNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.WindowNode; import org.junit.Test; @@ -146,4 +147,31 @@ public void testUnionFinalAggregationDoesNotInheritBranchGrouping() { } assertTrue(foundFinalCollect); } + + @Test + public void testTopKAboveGlobalWindowDoesNotAssumeOrderedInput() { + PlanNode root = + plan( + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n FROM table1 " + + "ORDER BY s1 DESC, tag1, tag2, tag3 LIMIT 5"); + List topKs = nodes(root, TopKNode.class); + assertFalse(topKs.isEmpty()); + boolean foundUnorderedInput = false; + for (TopKNode topK : topKs) { + if (!nodes(topK.getChildren().get(0), CollectNode.class).isEmpty()) { + foundUnorderedInput = true; + assertFalse(topK.isChildrenDataInOrder()); + } + } + assertTrue(foundUnorderedInput); + } + + @Test + public void testOrderedScanBranchesKeepStreamingAggregation() { + PlanNode root = + plan("SELECT tag1, tag2, tag3, SUM(s1 + 1) FROM table1 GROUP BY tag1, tag2, tag3"); + List aggregations = nodes(root, AggregationNode.class); + assertFalse(aggregations.isEmpty()); + assertTrue(aggregations.stream().anyMatch(AggregationNode::isStreamable)); + } } From 2a30afbb692f0b90d784eac3d383beebaf24a413 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 08:15:10 +0800 Subject: [PATCH 04/29] Fix unordered result assertions for ready Collect inputs Validate unordered results and EXPLAIN semantics independently on every DataNode while retaining ordered assertions. Add duplicate-preserving checks and deterministic regressions, and resolve receiver column names from metadata. --- .../itbase/runtime/ClusterTestResultSet.java | 12 ++ .../apache/iotdb/db/it/utils/TestUtils.java | 46 ++++-- .../it/triple/IoTDBShowReceiversTripleIT.java | 84 ++++++---- .../it/db/it/IoTDBMultiDeviceTableIT.java | 27 ++-- .../udf/IoTDBUserDefinedTableFunctionIT.java | 45 +++--- .../it/insertquery/IoTDBInsertQueryIT.java | 19 ++- .../query/recent/IoTDBClusterResultSetIT.java | 152 ++++++++++++++++++ .../it/query/recent/IoTDBCollectFanInIT.java | 25 +++ .../it/query/recent/IoTDBCteIT.java | 82 +++++----- .../query/recent/IoTDBTableAggregationIT.java | 28 ++-- .../IoTDBTopKRuntimeFilterScanPathIT.java | 29 ++-- 11 files changed, 400 insertions(+), 149 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestResultSet.java b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestResultSet.java index 98686d436be12..a9823b28572a4 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestResultSet.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestResultSet.java @@ -40,6 +40,7 @@ import java.sql.Time; import java.sql.Timestamp; import java.util.Calendar; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -68,6 +69,17 @@ public ClusterTestResultSet( this.env = env; } + /** + * Returns each node's result for assertions that do not require identical row order or plan text. + * Every returned result must be checked. The caller must close the original result set and must + * not mix per-node iteration with iteration through the cluster wrapper. + */ + public static List getNodeResultSets(ResultSet resultSet) { + return resultSet instanceof ClusterTestResultSet + ? Collections.unmodifiableList(((ClusterTestResultSet) resultSet).resultSets) + : Collections.singletonList(resultSet); + } + @Override public boolean next() throws SQLException { RequestDelegate delegate = createRequestDelegate(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java index 79a2f1c984769..04499fedf367a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java @@ -65,6 +65,7 @@ import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT; import static org.apache.iotdb.itbase.env.BaseEnv.TREE_SQL_DIALECT; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -807,6 +808,15 @@ public static void assertResultSetEqual( final String expectedHeader, final Collection expectedResult, final boolean printResult) { + if (expectedResult instanceof Set) { + List nodeResults = getNodeResultSets(actualResultSet); + if (nodeResults.size() != 1 || nodeResults.get(0) != actualResultSet) { + for (ResultSet nodeResult : nodeResults) { + assertResultSetEqual(nodeResult, expectedHeader, expectedResult, printResult); + } + return; + } + } try { final ResultSetMetaData resultSetMetaData = actualResultSet.getMetaData(); final StringBuilder header = new StringBuilder(); @@ -868,27 +878,31 @@ public static void assertResultSetEqual( String expectedHeader, Set expectedRetSet, Consumer consumer) { - try { - ResultSetMetaData resultSetMetaData = actualResultSet.getMetaData(); + assertResultSetEqual(actualResultSet, expectedHeader, expectedRetSet, false); + } + + /** Checks unordered rows on every node, preserving duplicate multiplicity. */ + public static void assertResultSetEqualUnordered( + ResultSet resultSet, String expectedHeader, List expectedRows) throws SQLException { + List sortedExpected = new ArrayList<>(expectedRows); + Collections.sort(sortedExpected); + for (ResultSet nodeResult : getNodeResultSets(resultSet)) { + ResultSetMetaData metadata = nodeResult.getMetaData(); StringBuilder header = new StringBuilder(); - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - header.append(resultSetMetaData.getColumnName(i)).append(","); + for (int i = 1; i <= metadata.getColumnCount(); i++) { + header.append(metadata.getColumnName(i)).append(","); } assertEquals(expectedHeader, header.toString()); - - Set actualRetSet = new HashSet<>(); - - while (actualResultSet.next()) { - StringBuilder builder = new StringBuilder(); - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - builder.append(actualResultSet.getString(i)).append(","); + List actualRows = new ArrayList<>(); + while (nodeResult.next()) { + StringBuilder row = new StringBuilder(); + for (int i = 1; i <= metadata.getColumnCount(); i++) { + row.append(nodeResult.getString(i)).append(","); } - actualRetSet.add(builder.toString()); + actualRows.add(row.toString()); } - assertStringSetEqual(expectedRetSet, actualRetSet); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(String.valueOf(e)); + Collections.sort(actualRows); + assertEquals(sortedExpected, actualRows); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java index 0712387775b9d..fda9935e31918 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java @@ -44,6 +44,7 @@ import java.sql.Connection; import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; @@ -54,6 +55,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; + @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT3.class}) public class IoTDBShowReceiversTripleIT { @@ -214,39 +217,44 @@ private boolean hasReceiversOnDifferentDataNodes( final String secondPipeName, final int secondReceiverDataNodeId) throws SQLException { - boolean hasFirstPipe = false; - boolean hasSecondPipe = false; - final Set receiverDataNodeIds = new HashSet<>(); try (final Connection connection = receiverEnv.getConnection( SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect); final Statement statement = connection.createStatement(); - final ResultSet resultSet = statement.executeQuery(sql)) { - while (resultSet.next()) { - final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids"); - if (!"DataNode".equals(getString(resultSet, "ReceiverNodeType", "receiver_node_type")) - || !"thrift".equals(getString(resultSet, "Protocol", "protocol")) - || pipeIds == null) { - continue; - } - final int receiverDataNodeId = getInt(resultSet, "ReceiverNodeId", "receiver_node_id"); - if (receiverDataNodeId == firstReceiverDataNodeId - && pipeIds.contains(firstPipeName + "@") - && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 - && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { - hasFirstPipe = true; - receiverDataNodeIds.add(receiverDataNodeId); + final ResultSet clusterResult = statement.executeQuery(sql)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + boolean hasFirstPipe = false; + boolean hasSecondPipe = false; + final Set receiverDataNodeIds = new HashSet<>(); + while (resultSet.next()) { + final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids"); + if (!"DataNode".equals(getString(resultSet, "ReceiverNodeType", "receiver_node_type")) + || !"thrift".equals(getString(resultSet, "Protocol", "protocol")) + || pipeIds == null) { + continue; + } + final int receiverDataNodeId = getInt(resultSet, "ReceiverNodeId", "receiver_node_id"); + if (receiverDataNodeId == firstReceiverDataNodeId + && pipeIds.contains(firstPipeName + "@") + && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 + && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { + hasFirstPipe = true; + receiverDataNodeIds.add(receiverDataNodeId); + } + if (receiverDataNodeId == secondReceiverDataNodeId + && pipeIds.contains(secondPipeName + "@") + && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 + && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { + hasSecondPipe = true; + receiverDataNodeIds.add(receiverDataNodeId); + } } - if (receiverDataNodeId == secondReceiverDataNodeId - && pipeIds.contains(secondPipeName + "@") - && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 - && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { - hasSecondPipe = true; - receiverDataNodeIds.add(receiverDataNodeId); + if (!hasFirstPipe || !hasSecondPipe || receiverDataNodeIds.size() < 2) { + return false; } } } - return hasFirstPipe && hasSecondPipe && receiverDataNodeIds.size() >= 2; + return true; } private int getDataNodeId(final DataNodeWrapper targetDataNode) throws Exception { @@ -268,20 +276,28 @@ private int getDataNodeId(final DataNodeWrapper targetDataNode) throws Exception private static String getString( final ResultSet resultSet, final String treeColumnName, final String tableColumnName) throws SQLException { - try { - return resultSet.getString(treeColumnName); - } catch (final SQLException ignored) { - return resultSet.getString(tableColumnName); - } + return resultSet.getString(getColumnIndex(resultSet, treeColumnName, tableColumnName)); } private static int getInt( final ResultSet resultSet, final String treeColumnName, final String tableColumnName) throws SQLException { - try { - return resultSet.getInt(treeColumnName); - } catch (final SQLException ignored) { - return resultSet.getInt(tableColumnName); + return resultSet.getInt(getColumnIndex(resultSet, treeColumnName, tableColumnName)); + } + + private static int getColumnIndex( + final ResultSet resultSet, final String treeColumnName, final String tableColumnName) + throws SQLException { + // SHOW RECEIVERS and information_schema.receivers expose different column names. + // Resolve from metadata because JDBC getters do not all use SQLException for unknown names. + final ResultSetMetaData metadata = resultSet.getMetaData(); + for (int column = 1; column <= metadata.getColumnCount(); column++) { + final String name = metadata.getColumnName(column); + if (treeColumnName.equals(name) || tableColumnName.equals(name)) { + return column; + } } + throw new AssertionError( + "Missing receiver column: " + treeColumnName + " / " + tableColumnName); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java index 16d92c213436d..096c1383c10d6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java @@ -37,6 +37,7 @@ import java.util.HashMap; import java.util.Map; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -269,20 +270,22 @@ private void testSelectAll() { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("use \"test\""); - Map lastTimeMap = new HashMap<>(); - try (ResultSet resultSet = statement.executeQuery(selectSql)) { - int cnt = 0; - while (resultSet.next()) { - String tag = resultSet.getString("tag1") + "_" + resultSet.getString("tag2"); - long before = lastTimeMap.getOrDefault(tag, -1L); - long cur = resultSet.getTimestamp("time").getTime(); - if (cur <= before) { - fail("time order wrong!"); + try (ResultSet clusterResult = statement.executeQuery(selectSql)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + Map lastTimeMap = new HashMap<>(); + int cnt = 0; + while (resultSet.next()) { + String tag = resultSet.getString("tag1") + "_" + resultSet.getString("tag2"); + long before = lastTimeMap.getOrDefault(tag, -1L); + long cur = resultSet.getTimestamp("time").getTime(); + if (cur <= before) { + fail("time order wrong!"); + } + lastTimeMap.put(tag, cur); + cnt++; } - lastTimeMap.put(tag, cur); - cnt++; + assertEquals(16030, cnt); } - assertEquals(16030, cnt); } } catch (Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java index 9db3acb13e098..25967b8caeb83 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java @@ -41,6 +41,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -200,37 +201,39 @@ public void testLargeResultIsSplitWithoutDataLoss() throws Exception { "large_result", "org.apache.iotdb.db.query.udf.example.relational.LargeResultTableFunction"); - Set returnedRows = new HashSet<>(); try (Connection connection = EnvFactory.getEnv().getTableConnection(); Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); - try (ResultSet resultSet = + try (ResultSet clusterResult = statement.executeQuery( "SELECT * FROM large_result(vehicle, " + LARGE_RESULT_REPEAT_COUNT + ", " + LARGE_RESULT_PAYLOAD_SIZE + ")")) { - while (resultSet.next()) { - int repeatIndex = resultSet.getInt("repeat_index"); - long time = resultSet.getLong("time"); - Assert.assertTrue(repeatIndex >= 0 && repeatIndex < LARGE_RESULT_REPEAT_COUNT); - Assert.assertEquals( - repeatIndex + ":" + "x".repeat(LARGE_RESULT_PAYLOAD_SIZE), - resultSet.getString("payload")); - assertPassThroughColumns(resultSet, time); - Assert.assertTrue( - "Duplicate result row for time " + time + " and repeat index " + repeatIndex, - returnedRows.add(time + ":" + repeatIndex)); - } - } - } + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + Set returnedRows = new HashSet<>(); + while (resultSet.next()) { + int repeatIndex = resultSet.getInt("repeat_index"); + long time = resultSet.getLong("time"); + Assert.assertTrue(repeatIndex >= 0 && repeatIndex < LARGE_RESULT_REPEAT_COUNT); + Assert.assertEquals( + repeatIndex + ":" + "x".repeat(LARGE_RESULT_PAYLOAD_SIZE), + resultSet.getString("payload")); + assertPassThroughColumns(resultSet, time); + Assert.assertTrue( + "Duplicate result row for time " + time + " and repeat index " + repeatIndex, + returnedRows.add(time + ":" + repeatIndex)); + } - long[] inputTimes = new long[] {1, 2, 3, 5}; - Assert.assertEquals(inputTimes.length * LARGE_RESULT_REPEAT_COUNT, returnedRows.size()); - for (long time : inputTimes) { - for (int repeatIndex = 0; repeatIndex < LARGE_RESULT_REPEAT_COUNT; repeatIndex++) { - Assert.assertTrue(returnedRows.contains(time + ":" + repeatIndex)); + long[] inputTimes = new long[] {1, 2, 3, 5}; + Assert.assertEquals(inputTimes.length * LARGE_RESULT_REPEAT_COUNT, returnedRows.size()); + for (long time : inputTimes) { + for (int repeatIndex = 0; repeatIndex < LARGE_RESULT_REPEAT_COUNT; repeatIndex++) { + Assert.assertTrue(returnedRows.contains(time + ":" + repeatIndex)); + } + } + } } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java index 75a7725c41498..0b43dc0e4152a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Locale; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -280,13 +281,17 @@ public void testPartialColumn() throws SQLException { statement.execute("DELETE FROM vehicle1"); statement.execute( "INSERT INTO vehicle1(time, s2, manufacturer) SELECT time, s2, manufacturer FROM vehicle0"); - resultSet = statement.executeQuery("SELECT deviceId, manufacturer, s1, s2 FROM vehicle1"); - while (resultSet.next()) { - for (int i = 1; i <= 4; i++) { - if (i % 2 == 1) { - assertNull(resultSet.getString(i)); - } else { - assertNotNull(resultSet.getString(i)); + try (ResultSet clusterResult = + statement.executeQuery("SELECT deviceId, manufacturer, s1, s2 FROM vehicle1")) { + for (ResultSet nodeResult : getNodeResultSets(clusterResult)) { + while (nodeResult.next()) { + for (int i = 1; i <= 4; i++) { + if (i % 2 == 1) { + assertNull(nodeResult.getString(i)); + } else { + assertNotNull(nodeResult.getString(i)); + } + } } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java new file mode 100644 index 0000000000000..a29c9793b72d2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.runtime.ClusterTestResultSet; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import javax.sql.rowset.CachedRowSet; +import javax.sql.rowset.RowSetMetaDataImpl; +import javax.sql.rowset.RowSetProvider; + +import java.lang.reflect.Proxy; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +/** Deterministic coverage of cross-node assertions without depending on network arrival order. */ +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBClusterResultSetIT { + + @Test + public void testUnorderedSetChecksEveryNode() throws SQLException { + try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { + TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,")); + } + try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "c"})) { + assertThrows( + AssertionError.class, + () -> TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,"))); + } + } + + @Test + public void testUnorderedSetConsumerOverloadChecksEveryNode() throws SQLException { + try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { + TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,"), message -> {}); + } + try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"a"})) { + assertThrows( + AssertionError.class, + () -> + TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,"), message -> {})); + } + } + + @Test + public void testUnorderedRowsPreserveDuplicateMultiplicity() throws SQLException { + try (ResultSet result = cluster(new String[] {"a", "a", "b"}, new String[] {"b", "a", "a"})) { + TestUtils.assertResultSetEqualUnordered(result, "value,", Arrays.asList("a,", "a,", "b,")); + } + for (String[] rows : new String[][] {{"a", "b"}, {"a", "a", "a", "b"}}) { + try (ResultSet result = cluster(new String[] {"a", "a", "b"}, rows)) { + assertThrows( + AssertionError.class, + () -> + TestUtils.assertResultSetEqualUnordered( + result, "value,", Arrays.asList("a,", "a,", "b,"))); + } + } + } + + @Test + public void testOrderedRowsStillRejectDifferentNodeOrder() throws SQLException { + try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { + assertThrows( + AssertionError.class, + () -> TestUtils.assertResultSetEqual(result, "value,", Arrays.asList("a,", "b,"))); + } + } + + @Test + public void testSingleNodeAndNullValues() throws SQLException { + try (ResultSet result = rows("a", null, "a")) { + assertEquals(1, getNodeResultSets(result).size()); + TestUtils.assertResultSetEqualUnordered(result, "value,", Arrays.asList("null,", "a,", "a,")); + } + try (ResultSet result = cluster(new String[] {"a"})) { + assertEquals(1, getNodeResultSets(result).size()); + TestUtils.assertResultSetEqual(result, "value,", Set.of("a,")); + } + } + + private static ResultSet cluster(String[]... nodeRows) throws SQLException { + List statements = new ArrayList<>(); + List endpoints = new ArrayList<>(); + for (String[] values : nodeRows) { + ResultSet result = rows(values); + Statement statement = + (Statement) + Proxy.newProxyInstance( + Statement.class.getClassLoader(), + new Class[] {Statement.class}, + (proxy, method, arguments) -> { + if (method.getName().equals("executeQuery")) { + return result; + } + throw new UnsupportedOperationException(method.getName()); + }); + statements.add(statement); + endpoints.add("node-" + endpoints.size()); + } + return new ClusterTestResultSet(statements, endpoints, "SELECT value", 10, null); + } + + private static CachedRowSet rows(String... values) throws SQLException { + CachedRowSet result = RowSetProvider.newFactory().createCachedRowSet(); + RowSetMetaDataImpl metadata = new RowSetMetaDataImpl(); + metadata.setColumnCount(1); + metadata.setColumnName(1, "value"); + metadata.setColumnType(1, Types.VARCHAR); + result.setMetaData(metadata); + for (String value : values) { + result.afterLast(); + result.moveToInsertRow(); + result.updateString(1, value); + result.insertRow(); + result.moveToCurrentRow(); + } + result.beforeFirst(); + return result; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java index 8667f01ddfd70..f9de8a0664fb4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java @@ -33,7 +33,10 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -83,6 +86,28 @@ public static void setUp() throws Exception { } } + @Test + public void testUnorderedUnionPreservesEveryDuplicateOnEachNode() throws Exception { + List expectedRows = new ArrayList<>(); + for (int device = 0; device < DEVICES; device++) { + for (int value = 1; value <= ROWS; value++) { + String row = device(device) + "," + zone(device) + "," + value + ","; + expectedRows.add(row); + expectedRows.add(row); + } + } + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + try (ResultSet result = + statement.executeQuery( + "SELECT device, zone, value FROM readings1 UNION ALL " + + "SELECT device, zone, value FROM readings1")) { + assertResultSetEqualUnordered(result, "device,zone,value,", expectedRows); + } + } + } + @Test public void testSharedExchangePathsForTreeAndTreeToTable() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java index 65d3f277e52de..e8089db62f13b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java @@ -60,6 +60,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -361,19 +362,22 @@ public void testNestExplain1() throws SQLException { statement.execute("USE testdb"); // explain - ResultSet resultSet = statement.executeQuery(sql); - ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(metaData.getColumnCount(), 1); - assertEquals(metaData.getColumnName(1), "distribution plan"); - - StringBuilder sb = new StringBuilder(); - while (resultSet.next()) { - sb.append(resultSet.getString(1)).append(System.lineSeparator()); + try (ResultSet clusterResult = statement.executeQuery(sql)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnCount(), 1); + assertEquals(metaData.getColumnName(1), "distribution plan"); + + StringBuilder sb = new StringBuilder(); + while (resultSet.next()) { + sb.append(resultSet.getString(1)).append(System.lineSeparator()); + } + String result = sb.toString(); + assertFalse(result.contains("CTE Query : 'cte1'")); + assertTrue(result.contains("CTE Query : 'cte2'")); + assertTrue(result.contains("Main Query")); + } } - String result = sb.toString(); - assertFalse(result.contains("CTE Query : 'cte1'")); - assertTrue(result.contains("CTE Query : 'cte2'")); - assertTrue(result.contains("Main Query")); } } @@ -388,19 +392,22 @@ public void testNestExplain2() throws SQLException { statement.execute("USE testdb"); // explain - ResultSet resultSet = statement.executeQuery(sql); - ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(metaData.getColumnCount(), 1); - assertEquals(metaData.getColumnName(1), "distribution plan"); - - StringBuilder sb = new StringBuilder(); - while (resultSet.next()) { - sb.append(resultSet.getString(1)).append(System.lineSeparator()); + try (ResultSet clusterResult = statement.executeQuery(sql)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnCount(), 1); + assertEquals(metaData.getColumnName(1), "distribution plan"); + + StringBuilder sb = new StringBuilder(); + while (resultSet.next()) { + sb.append(resultSet.getString(1)).append(System.lineSeparator()); + } + String result = sb.toString(); + assertTrue(result.contains("CTE Query : 'cte1'")); + assertTrue(result.contains("CTE Query : 'cte2'")); + assertTrue(result.contains("Main Query")); + } } - String result = sb.toString(); - assertTrue(result.contains("CTE Query : 'cte1'")); - assertTrue(result.contains("CTE Query : 'cte2'")); - assertTrue(result.contains("Main Query")); } } @@ -415,19 +422,22 @@ public void testExplainJoin() throws SQLException { statement.execute("USE testdb"); // explain - ResultSet resultSet = statement.executeQuery(sql); - ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(metaData.getColumnCount(), 1); - assertEquals(metaData.getColumnName(1), "distribution plan"); - - StringBuilder sb = new StringBuilder(); - while (resultSet.next()) { - sb.append(resultSet.getString(1)).append(System.lineSeparator()); + try (ResultSet clusterResult = statement.executeQuery(sql)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnCount(), 1); + assertEquals(metaData.getColumnName(1), "distribution plan"); + + StringBuilder sb = new StringBuilder(); + while (resultSet.next()) { + sb.append(resultSet.getString(1)).append(System.lineSeparator()); + } + String result = sb.toString(); + assertFalse(result.contains("CTE Query : 'cte1'")); + assertTrue(result.contains("CTE Query : 'cte2'")); + assertTrue(result.contains("Main Query")); + } } - String result = sb.toString(); - assertFalse(result.contains("CTE Query : 'cte1'")); - assertTrue(result.contains("CTE Query : 'cte2'")); - assertTrue(result.contains("Main Query")); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index 518b3da1fdd21..99188903b99a7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -44,6 +44,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.buildHeaders; import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.repeatTest; @@ -5902,19 +5903,22 @@ private void assertRateQueryUsesIntermediateAggregation(String query) { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); - try (ResultSet resultSet = statement.executeQuery("EXPLAIN (FORMAT JSON) " + query)) { - Assert.assertTrue(resultSet.next()); - String plan = resultSet.getString(1); - // The PARTIAL/FINAL split only exists when the scanned data spans multiple DataRegions. - // Under randomized partition allocation (e.g. the SHUFFLE strategy used by - // IoTDBTableAggregation2IT), all devices may occasionally land in one single region, - // where a SINGLE-step aggregation is the correct plan, so the check must be skipped. - if (countDistinctRegionIds(plan) < 2) { - return; + try (ResultSet clusterResult = statement.executeQuery("EXPLAIN (FORMAT JSON) " + query)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + Assert.assertTrue(resultSet.next()); + String plan = resultSet.getString(1); + // The PARTIAL/FINAL split only exists when the scanned data spans multiple DataRegions. + // Under randomized partition allocation (e.g. the SHUFFLE strategy used by + // IoTDBTableAggregation2IT), all devices may occasionally land in one single region, + // where a SINGLE-step aggregation is the correct plan, so the check must be skipped. + if (countDistinctRegionIds(plan) < 2) { + continue; + } + Assert.assertTrue( + "Expected a PARTIAL aggregation in plan: " + plan, plan.contains("PARTIAL")); + Assert.assertTrue( + "Expected a FINAL aggregation in plan: " + plan, plan.contains("FINAL")); } - Assert.assertTrue( - "Expected a PARTIAL aggregation in plan: " + plan, plan.contains("PARTIAL")); - Assert.assertTrue("Expected a FINAL aggregation in plan: " + plan, plan.contains("FINAL")); } } catch (Exception exception) { throw new AssertionError(exception); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java index 7697bc81649bd..a0a638dcfe0eb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java @@ -41,11 +41,14 @@ import java.io.IOException; import java.nio.file.Files; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; @@ -131,19 +134,23 @@ public void orderByTimeAscLimitOnTable() { } @Test - public void unionAllSiblingTopKOrderByTimeLimit() { - String[] expectedHeader = new String[] {"time", "device_id", "s1"}; - String[] retArray = - new String[] { - "1970-01-01T00:00:00.030Z,d1,111,", "1970-01-01T00:00:00.032Z,d3,333,", - }; - tableResultSetEqualTest( + public void unionAllSiblingTopKOrderByTimeLimit() throws SQLException { + // Each branch has an ORDER BY, but UNION ALL does not order the combined result. + String sql = "(SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd1' ORDER BY time DESC LIMIT 1)" + " UNION ALL" - + " (SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd3' ORDER BY time DESC LIMIT 1)", - expectedHeader, - retArray, - TABLE_DATABASE); + + " (SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd3' ORDER BY time DESC LIMIT 1)"; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + connection.setClientInfo("time_zone", "+00:00"); + statement.execute("USE " + TABLE_DATABASE); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertResultSetEqualUnordered( + resultSet, + "time,device_id,s1,", + Arrays.asList("1970-01-01T00:00:00.030Z,d1,111,", "1970-01-01T00:00:00.032Z,d3,333,")); + } + } } @Test From fd3d58d941d78b2ae5fa05db1172bccc030ce060 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 08:15:11 +0800 Subject: [PATCH 05/29] Fix occupied-port detection and isolate client manager test ports Read the address column rather than the socket state from lsof output and cover a real listening socket. Allow ClientManagerTest to use an explicitly configured port when local services occupy its default. --- .../apache/iotdb/it/env/cluster/EnvUtils.java | 3 +- .../it/query/recent/IoTDBTestEnvPortIT.java | 47 +++++++++++++++++++ .../commons/client/ClientManagerTest.java | 3 +- 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTestEnvPortIT.java diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java index b69e86c12bd42..e6c58e251f593 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java @@ -180,7 +180,8 @@ public static Map listPortOccupationWindows(final List p */ public static Map listPortOccupationUnix(final List ports) throws IOException { - return listPortOccupation(ports, "lsof -iTCP -sTCP:LISTEN,TIME_WAIT -P -n", 10, 9, 1); + // The last column is the socket state (e.g. LISTEN); the address precedes it. + return listPortOccupation(ports, "lsof -iTCP -sTCP:LISTEN,TIME_WAIT -P -n", 10, 8, 1); } private static Pair getClusterNodesNum(final int index) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTestEnvPortIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTestEnvPortIT.java new file mode 100644 index 0000000000000..dbb9a9d7a74c5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTestEnvPortIT.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.cluster.EnvUtils; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.net.InetAddress; +import java.net.ServerSocket; +import java.util.Collections; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTestEnvPortIT { + + @Test + public void testListeningPortIsReportedAsOccupied() throws Exception { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + Map occupiedPorts = + EnvUtils.listPortOccupation(Collections.singletonList(server.getLocalPort())); + assertEquals( + Long.valueOf(ProcessHandle.current().pid()), occupiedPorts.get(server.getLocalPort())); + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java index be7cf49239a63..82fd7cc80e224 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java @@ -55,7 +55,8 @@ public class ClientManagerTest { - private final TEndPoint endPoint = new TEndPoint("localhost", 10730); + private final TEndPoint endPoint = + new TEndPoint("localhost", Integer.getInteger("iotdb.test.client-manager.port", 10730)); private static final int CONNECTION_TIMEOUT = 5_000; From 8d36f287f2e7e6afeadf5704851ab7229b56776e Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 14:20:40 +0800 Subject: [PATCH 06/29] Limit shared exchange budgets to eligible Collect fragments Retain independent plan-node quotas for dependent operators and internal pipelines, and fall back when a split side cannot hold one TsBlock. Make memory admission atomic without speculative charges and add regression coverage for MergeSort, queue forwarding, and contention. --- ...2026-10-14-exchange-fanin-budget-design.md | 47 ++- .../exchange/SharedTsBlockQueue.java | 3 +- .../exchange/sink/ShuffleSinkHandle.java | 4 +- .../execution/exchange/sink/SinkChannel.java | 3 +- .../exchange/source/SourceHandle.java | 3 +- .../execution/memory/MemoryPool.java | 99 +++-- .../DataNodeTableOperatorGenerator.java | 4 +- .../planner/LocalExecutionPlanContext.java | 64 ++-- .../plan/planner/LocalExecutionPlanner.java | 19 +- .../plan/planner/OperatorTreeGenerator.java | 7 +- .../SharedExchangeMemoryEligibility.java | 65 ++++ .../exchange/SharedTsBlockQueueTest.java | 30 ++ .../execution/memory/MemoryPoolTest.java | 88 +++++ .../LocalExecutionPlanContextBudgetTest.java | 356 ++++++++++++++++-- 14 files changed, 652 insertions(+), 140 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java diff --git a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md index e191622940dc7..bc0b5c2557b22 100644 --- a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md +++ b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md @@ -26,21 +26,25 @@ ## 已确认范围 -实现 FI 输入输出共享账目、Collect/MappingCollect 任意就绪消费，以及由无序汇集引起的计划属性修正。 +实现仅针对无输入依赖 FI 的输入输出共享账目、Collect/MappingCollect 任意就绪消费，以及由无序汇集引起的计划属性修正。 聚合使用 HashAggregation：只清空实际输入已失去连续性的 preGroupedSymbols；有效分支内的流式聚合保留。 -MergeSort、Join、ASOF、CrossJoin 只作依赖验证，不修改其消费算法，不新增进度保障机制。 +MergeSort、Join、ASOF、CrossJoin 不修改消费算法。FI 内只要存在需要特定输入推进的算子，就整组使用原有按 plan node 限额；Collect 下方或上方出现这些算子也不例外。 TableTopK、CountMerge 的任意就绪消费留到后续；有序输入标志仍需匹配实际输入。 ## 额度与账目 -F 为配置的单 FI 交换预算。INPUT=F/2，OUTPUT=F-INPUT；同 FI 的所有 pipeline 继承相同固定值。 +共享采用 FI 级显式启用，默认保留原有配额。整个 FI 必须包含 CollectOperator 或 MappingCollectOperator，且所有多输入汇集都能消费任意就绪输入；顺序消费的 IdentitySink/CountMerge、MergeSort、Join、TopK 及未明确支持的多输入算子都禁止共享。检查在 Exchange 处停止，不跨到其他 FI。 + +在生成全部 pipeline 后再决定预算模式。内部 pipeline 队列具有转发依赖，因此存在多个 pipeline 或调度依赖时整组回退原策略。只含 Collect 的某个子树不会绕过 FI 级否决。 + +通过上述检查后，F 为配置的单 FI 交换预算，INPUT=F/2，OUTPUT=F-INPUT。两侧均须容纳配置的最大 TsBlock，否则回退原配额，避免仅有输出的 FI 因预算减半而无法打开 SinkChannel。未启用共享的 FI 恢复原有 exchangeSumNum 配额计算和按 plan node 的申请上限；不能只恢复限额数值却仍按同侧总量进行准入。 远端 SourceHandle 计入接收 FI 的 INPUT；SinkChannel 和 SharedTsBlockQueue 计入生产 FI 的 OUTPUT。 本地共享队列只计费一次，创建先后不改变所有者。 MemoryPool 维护 query→FI→planNode 明细，注册时固定 side。 -FI 两侧使用 AtomicLong；planNode 字节明细通过 compute 增减。 -申请、回滚、释放均对应修改全局池、FI 单侧与 planNode 三层账目。 -额度判断使用本次增加后的单侧总数；稳定快照中的总数等于同侧明细之和。 +FI 两侧保留 AtomicLong 统计，plan node 保留字节明细。申请和释放在 FI 的记账锁内更新；全局池采用有界原子申请。申请前先检查适用的同侧总量或 plan node 限额，失败不产生临时计费，也不需要回滚，避免并发请求分别撞到不同限额后全部挂起。 + +释放后在 FI 记账锁外完成等待 Future；保留入队后的重试和每个 Future 的完成/取消同步。稳定快照中的 FI 同侧总量等于同侧明细之和。 保留现有最高优先级查询的零计费兜底与交换组件的收发、等待、释放机制。 ## Collect 生命周期 @@ -65,7 +69,8 @@ SortElimination 在各汇集分支独立优化，并阻止跨无序汇集删除 ## 验证 -覆盖共享额度并发、INPUT/OUTPUT 隔离、所有者、奇数 F、超大申请、回滚释放取消及稳定账目一致性； +覆盖纯 Collect/MappingCollect 显式启用、混合 FI 全组回退、Exchange 边界、内部 pipeline 否决、单块预算边界，以及共享/原配额 FI 共存； +覆盖共享额度并发、INPUT/OUTPUT 隔离、所有者、奇数 F、超大申请、失败准入无计费、释放取消及稳定账目一致性； 覆盖 100/1000 child、A 阻塞 B 就绪、listener 复用、注册竞争、输出背压、空块、结束、异常取消、关闭和映射； 覆盖文档 SQL 对应的聚合、Group/Window、StreamSort、Union 与 Fill 计划及实际结果。 运行 Spotless、英文/中文全 reactor test-compile、针对性单测，以及本分支新增或修改的 IT。 @@ -117,3 +122,31 @@ Collect 后 TopK 不误用有序输入，以及有效分支保留流式聚合。 重新执行 `mvn -o install -DskipTests -pl integration-test,iotdb-core/confignode,iotdb-client/cli -am -P with-integration-tests` 后运行前述两条 verify 命令；IoTDBCollectFanInIT 的 7 项测试在单节点和三 DataNode 集群中分别全部通过。 Spotless、Checkstyle、许可证与 `git diff --check` 检查通过。 + +## FI 共享范围修正验证（2026-10-15） + +共享改为完整 FI 的显式启用：仅支持 Collect/MappingCollect 的任意就绪汇集； +同一 FI 内的多输入依赖算子、内部 pipeline 队列或调度依赖会使整组回退。 +Exchange 边界之外的上游 FI 不参与本 FI 的判断。回退同时恢复 handle 限额和按 plan node 准入。 + +107 项定向单元测试通过，包括纯 Collect/Union、混合 FI、pipeline 依赖、单块额度边界、 +共享与独立配额共存、并发准入，以及真实 TableMergeSort 配合模拟输入和 MemoryPool 的回归。 +本地双队列转发回归也通过。Spotless、Checkstyle、许可证和 `git diff --check` 检查通过。 + +```sh +mvn -o test -pl iotdb-core/datanode -Dtest=LocalExecutionPlanContextBudgetTest,MemoryPoolTest,SharedTsBlockQueueTest,SourceHandleTest,SinkChannelTest,LocalSourceHandleTest,LocalSinkChannelTest,MPPDataExchangeManagerTest,PipelineBuilderTest,CollectInputPropertyTest +mvn -o verify -DskipUTs -Dit.test=IoTDBCollectFanInIT -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -am -P TableSimpleIT -P with-integration-tests +mvn -o failsafe:integration-test@integration-test failsafe:verify@verify -DskipUTs -Dit.test=IoTDBCollectFanInIT -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -P TableClusterIT -P with-integration-tests +``` + +单节点 verify 从当前源码构建依赖和测试节点；集群运行复用同一份节点模板。 +IoTDBCollectFanInIT 的 8 项测试在两种环境中均通过，零失败、零跳过。 + +提交前再次执行完整 reactor 的中英文编译，两者均通过： + +```sh +mvn -o test-compile -DskipTests -P with-zh-locale +mvn -o test-compile -DskipTests +``` + +本次生产代码没有新增未本地化的字符串，且没有修改中英文消息常量。 diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java index cb8c681e4e9da..0d8423803618d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java @@ -86,8 +86,7 @@ public class SharedTsBlockQueue { private LocalSinkChannel sinkChannel; private long maxBytesCanReserve = - MemorySide.OUTPUT.getMaxBytes( - IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance()); + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); private final boolean isHighestPriority; private volatile Throwable abortedCause = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/ShuffleSinkHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/ShuffleSinkHandle.java index 1de3ccb897df7..e54062158a3f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/ShuffleSinkHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/ShuffleSinkHandle.java @@ -22,7 +22,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager; -import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.metric.DataExchangeCostMetricSet; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; @@ -77,8 +76,7 @@ public class ShuffleSinkHandle implements ISinkHandle { /** max bytes this ShuffleSinkHandle can reserve. */ private long maxBytesCanReserve = - MemorySide.OUTPUT.getMaxBytes( - IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance()); + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ShuffleSinkHandle.class) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java index 0dec1cbf07f26..4d592771ef23e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java @@ -132,8 +132,7 @@ private TsBlockInfo(TsBlock tsBlock, ByteBuffer serializedTsBlock, long tsBlockS /** max bytes this SinkChannel can reserve. */ private long maxBytesCanReserve = - MemorySide.OUTPUT.getMaxBytes( - IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance()); + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); private final boolean isHighestPriority; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java index b1dc2781ed658..8ee7ad601ef61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java @@ -118,8 +118,7 @@ public class SourceHandle implements ISourceHandle { /** max bytes this SourceHandle can reserve. */ private long maxBytesCanReserve = - MemorySide.INPUT.getMaxBytes( - IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance()); + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); /** * this is set to true after calling isBlocked() at least once which indicates that this diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index 07eced179d94e..84f36b52e4ed3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -59,7 +59,8 @@ public static class MemoryReservationFuture extends AbstractFuture { /** * MemoryReservationFuture is created when an exchange handle tries to reserve memory from the - * pool. This limit is shared by all plan nodes registered on the same side of that fragment. + * pool. For opted-in fragments the limit is shared by all plan nodes on the same side; + * otherwise it applies independently to the plan node. */ private final long maxBytesCanReserve; @@ -173,6 +174,8 @@ private static class FragmentMemoryReservation { private final Map planNodes = new ConcurrentHashMap<>(); private final AtomicLong inputUsedBytes = new AtomicLong(); private final AtomicLong outputUsedBytes = new AtomicLong(); + // Set by local planning before this FI starts reserving exchange memory. + private boolean sharedBudget; private AtomicLong getUsedBytes(MemorySide side) { return side == MemorySide.INPUT ? inputUsedBytes : outputUsedBytes; @@ -253,6 +256,19 @@ public void registerPlanNodeIdToQueryMemoryMap( } } + /** Opt in only after the complete FI has been checked for input and pipeline dependencies. */ + public void registerFragmentInstanceSharedBudget(String queryId, String fragmentInstanceId) { + synchronized (queryMemoryReservations) { + FragmentMemoryReservation fragment = + queryMemoryReservations + .computeIfAbsent(queryId, x -> new ConcurrentHashMap<>()) + .computeIfAbsent(fragmentInstanceId, x -> new FragmentMemoryReservation()); + synchronized (fragment) { + fragment.sharedBudget = true; + } + } + } + /** * If all fragmentInstanceIds related to one queryId have been registered, when the last fragment * instance is deregister, the queryId can be cleared. @@ -356,7 +372,6 @@ public MemoryReservationResult reserveWithPriority( if (tryReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve)) { return new MemoryReservationResult(immediateVoidFuture(), true, bytesToReserve); } else { - rollbackReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve); if (isHighestPriority) { // SHOW QUERIES: treat as success with zero bytes reserved from pool when insufficient. return new MemoryReservationResult(immediateVoidFuture(), true, 0L); @@ -398,12 +413,7 @@ public boolean tryReserveForTest( maxBytesPerFragmentInstance, bytesToReserve); - if (tryReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve)) { - return true; - } else { - rollbackReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve); - return false; - } + return tryReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve); } /** @@ -447,20 +457,19 @@ public void free(String queryId, String fragmentInstanceId, String planNodeId, l DataNodeQueryMessages .QUERY_EXCEPTION_RELATEDMEMORYRESERVED_CAN_T_BE_NULL_WHEN_FREEING_MEMORY_C80009F2); } - fragment.planNodes.compute( - planNodeId, - (key, reservation) -> { - requireReservation(reservation, queryId, fragmentInstanceId, planNodeId); - if (reservation.bytesReserved < bytes) { - throw new IllegalArgumentException( - DataNodeQueryMessages.FREE_MORE_MEMORY_THAN_HAS_BEEN_RESERVED); - } - reservation.bytesReserved -= bytes; - fragment.getUsedBytes(reservation.side).addAndGet(-bytes); - return reservation; - }); - - memoryBlock.release(bytes); + synchronized (fragment) { + PlanNodeReservation reservation = + requireReservation( + fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); + if (reservation.bytesReserved < bytes) { + throw new IllegalArgumentException( + DataNodeQueryMessages.FREE_MORE_MEMORY_THAN_HAS_BEEN_RESERVED); + } + reservation.bytesReserved -= bytes; + fragment.getUsedBytes(reservation.side).addAndGet(-bytes); + memoryBlock.release(bytes); + } + // Complete futures outside the accounting lock: listeners can reserve or free other inputs. if (memoryReservationFutures.isEmpty()) { return; @@ -492,7 +501,6 @@ private boolean tryCompleteReservation(MemoryReservationFuture future) { future.set(null); return true; } - rollbackReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve); return false; } } @@ -538,33 +546,24 @@ public boolean tryReserve( long maxBytesCanReserve) { FragmentMemoryReservation fragment = getFragmentReservation(queryId, fragmentInstanceId, planNodeId); - PlanNodeReservation reservation = - requireReservation( - fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); - long tryUsedBytes = memoryBlock.forceAllocateWithoutLimitation(bytesToReserve); - long sideUsedBytes = fragment.getUsedBytes(reservation.side).addAndGet(bytesToReserve); - fragment.planNodes.compute( - planNodeId, - (key, value) -> { - value.bytesReserved += bytesToReserve; - return value; - }); - return tryUsedBytes <= memoryBlock.getTotalMemorySizeInBytes() - && sideUsedBytes <= maxBytesCanReserve; - } - - private void rollbackReserve( - String queryId, String fragmentInstanceId, String planNodeId, long bytesToReserve) { - FragmentMemoryReservation fragment = - getFragmentReservation(queryId, fragmentInstanceId, planNodeId); - fragment.planNodes.compute( - planNodeId, - (key, reservation) -> { - reservation.bytesReserved -= bytesToReserve; - fragment.getUsedBytes(reservation.side).addAndGet(-bytesToReserve); - return reservation; - }); - memoryBlock.release(bytesToReserve); + synchronized (fragment) { + PlanNodeReservation reservation = + requireReservation( + fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); + long usedBytes = + fragment.sharedBudget + ? fragment.getUsedBytes(reservation.side).get() + : reservation.bytesReserved; + // Do not publish speculative charges. Two competing requests must not reject one another + // against different counters and then both wait after their rollbacks restore capacity. + if (bytesToReserve > maxBytesCanReserve - usedBytes + || !memoryBlock.allocate(bytesToReserve)) { + return false; + } + fragment.getUsedBytes(reservation.side).addAndGet(bytesToReserve); + reservation.bytesReserved += bytesToReserve; + return true; + } } private FragmentMemoryReservation getFragmentReservation( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index fd539be7467e2..2fae9aa537081 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -343,8 +343,7 @@ public Operator visitIdentitySink(IdentitySinkNode node, LocalExecutionPlanConte Operator child = node.getChildren().get(0).accept(this, context); List children = new ArrayList<>(1); children.add(child); - sinkHandle.setMaxBytesCanReserve(context.getOutputMaxBytes()); - context.getDriverContext().setSink(sinkHandle); + context.setISink(sinkHandle); return new IdentitySinkOperator( operatorContext, children, downStreamChannelIndex, sinkHandle); } else { @@ -385,7 +384,6 @@ public Operator visitTableExchange(ExchangeNode node, LocalExecutionPlanContext if (!isSameNode) { context.addExchangeSumNum(1); } - sourceHandle.setMaxBytesCanReserve(context.getInputMaxBytes()); ExchangeOperator exchangeOperator = new ExchangeOperator(operatorContext, sourceHandle, node.getUpstreamPlanNodeId()); context.addExchangeOperator(exchangeOperator); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java index bfed28ccc4ca4..82c58cad65594 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java @@ -32,9 +32,10 @@ import org.apache.iotdb.db.queryengine.execution.driver.DriverContext; import org.apache.iotdb.db.queryengine.execution.driver.SchemaDriverContext; import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISink; -import org.apache.iotdb.db.queryengine.execution.exchange.source.PipelineSourceHandle; +import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISinkHandle; import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; import org.apache.iotdb.db.queryengine.plan.analyze.TemplatedInfo; @@ -43,6 +44,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; @@ -55,6 +57,7 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -82,8 +85,8 @@ public class LocalExecutionPlanContext implements ITableOperatorGeneratorContext private List pipelineDriverFactories; private List exchangeOperatorList = new ArrayList<>(); private int exchangeSumNum = 0; - private final long inputMaxBytes; - private final long outputMaxBytes; + private final long fragmentExchangeMaxBytes; + private Map legacySinkLimits = new IdentityHashMap<>(); private List cachedDataTypes; @@ -111,10 +114,8 @@ public LocalExecutionPlanContext( TypeProvider typeProvider, FragmentInstanceContext instanceContext, DataNodeQueryContext dataNodeQueryContext) { - long fragmentExchangeMaxBytes = + this.fragmentExchangeMaxBytes = IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); - this.inputMaxBytes = MemorySide.INPUT.getMaxBytes(fragmentExchangeMaxBytes); - this.outputMaxBytes = MemorySide.OUTPUT.getMaxBytes(fragmentExchangeMaxBytes); this.typeProvider = typeProvider; this.allSensorsMap = new ConcurrentHashMap<>(); this.nextOperatorId = new AtomicInteger(0); @@ -126,8 +127,8 @@ public LocalExecutionPlanContext( // For creating subContext, differ from parent context mainly in driver context public LocalExecutionPlanContext(LocalExecutionPlanContext parentContext) { - this.inputMaxBytes = parentContext.inputMaxBytes; - this.outputMaxBytes = parentContext.outputMaxBytes; + this.fragmentExchangeMaxBytes = parentContext.fragmentExchangeMaxBytes; + this.legacySinkLimits = parentContext.legacySinkLimits; this.nextOperatorId = parentContext.nextOperatorId; this.typeProvider = parentContext.typeProvider; this.allSensorsMap = parentContext.allSensorsMap; @@ -149,10 +150,8 @@ public LocalExecutionPlanContext( final TypeProvider typeProvider, final FragmentInstanceContext instanceContext, final ISchemaRegion schemaRegion) { - long fragmentExchangeMaxBytes = + this.fragmentExchangeMaxBytes = IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); - this.inputMaxBytes = MemorySide.INPUT.getMaxBytes(fragmentExchangeMaxBytes); - this.outputMaxBytes = MemorySide.OUTPUT.getMaxBytes(fragmentExchangeMaxBytes); this.allSensorsMap = new ConcurrentHashMap<>(); this.typeProvider = typeProvider; this.nextOperatorId = new AtomicInteger(0); @@ -258,12 +257,10 @@ public void setExchangeSumNum(int exchangeSumNum) { this.exchangeSumNum = exchangeSumNum; } - public long getInputMaxBytes() { - return inputMaxBytes; - } - - public long getOutputMaxBytes() { - return outputMaxBytes; + public long getMaxBytesOneHandleCanReserve() { + return exchangeSumNum == 0 + ? fragmentExchangeMaxBytes + : fragmentExchangeMaxBytes / exchangeSumNum; } public void addExchangeSumNum(int addValue) { @@ -274,16 +271,27 @@ public void addExchangeOperator(ExchangeOperator exchangeOperator) { this.exchangeOperatorList.add(exchangeOperator); } - public void setExchangeMemoryLimits() { + public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { + long inputMaxBytes = MemorySide.INPUT.getMaxBytes(fragmentExchangeMaxBytes); + long outputMaxBytes = MemorySide.OUTPUT.getMaxBytes(fragmentExchangeMaxBytes); + long maxBlockBytes = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); + boolean sharedBudget = + SharedExchangeMemoryEligibility.isEligible(plan) + // Internal pipeline queues and their downstream sinks must not compete for OUTPUT. + && pipelineDriverFactories.size() == 1 + && pipelineDriverFactories.get(0).getDependencyPipelineIndex() == -1 + && inputMaxBytes >= maxBlockBytes + && outputMaxBytes >= maxBlockBytes; + if (sharedBudget) { + FragmentInstanceId id = getFragmentInstanceId(); + memoryPool.registerFragmentInstanceSharedBudget(id.getQueryId().getId(), id.getFullId()); + } + legacySinkLimits.forEach( + (sink, legacyLimit) -> + sink.setMaxBytesCanReserve(sharedBudget ? outputMaxBytes : legacyLimit)); + long sourceLimit = sharedBudget ? inputMaxBytes : getMaxBytesOneHandleCanReserve(); exchangeOperatorList.forEach( - exchangeOperator -> { - exchangeOperator - .getSourceHandle() - .setMaxBytesCanReserve( - exchangeOperator.getSourceHandle() instanceof PipelineSourceHandle - ? outputMaxBytes - : inputMaxBytes); - }); + exchangeOperator -> exchangeOperator.getSourceHandle().setMaxBytesCanReserve(sourceLimit)); } public Set getAllSensors(IDeviceID deviceId, String sensorId) { @@ -316,7 +324,9 @@ public void setISink(ISink sink) { checkArgument( driverContext.getSink() == null, DataNodeQueryMessages.EXCEPTION_THERE_MUST_BE_AT_MOST_ONE_SINKNODE_A965AFE7); - sink.setMaxBytesCanReserve(outputMaxBytes); + if (sink instanceof ISinkHandle) { + legacySinkLimits.put(sink, getMaxBytesOneHandleCanReserve()); + } driverContext.setSink(sink); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java index 1a85cb148c4f4..8e020cf736453 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.driver.DataDriverContext; +import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeService; import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine; @@ -132,8 +133,13 @@ public List plan( context.getTimePartitions().ifPresent(instanceContext::setTimePartitions); - // set maxBytes one SourceHandle can reserve after visiting the whole tree - context.setExchangeMemoryLimits(); + // Choose the accounting policy only after all FI operators and pipelines are known. + context.setExchangeMemoryLimits( + plan, + MPPDataExchangeService.getInstance() + .getMPPDataExchangeManager() + .getLocalMemoryManager() + .getQueryPool()); return context.getPipelineDriverFactories(); } @@ -165,8 +171,13 @@ public List plan( context.addPipelineDriverFactory(root, context.getDriverContext(), 0); - // set maxBytes one SourceHandle can reserve after visiting the whole tree - context.setExchangeMemoryLimits(); + // Choose the accounting policy only after all FI operators and pipelines are known. + context.setExchangeMemoryLimits( + plan, + MPPDataExchangeService.getInstance() + .getMPPDataExchangeManager() + .getLocalMemoryManager() + .getQueryPool()); return context.getPipelineDriverFactories(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index d643ec03752e1..a46877ac1d5f6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -2577,7 +2577,6 @@ public Operator visitExchange(ExchangeNode node, LocalExecutionPlanContext conte if (!isSameNode) { context.addExchangeSumNum(1); } - sourceHandle.setMaxBytesCanReserve(context.getInputMaxBytes()); ExchangeOperator exchangeOperator = new ExchangeOperator(operatorContext, sourceHandle, node.getUpstreamPlanNodeId()); context.addExchangeOperator(exchangeOperator); @@ -2616,8 +2615,7 @@ public Operator visitIdentitySink(IdentitySinkNode node, LocalExecutionPlanConte node.getPlanNodeId().getId(), context.getInstanceContext()); List children = dealWithConsumeChildrenOneByOneNode(node, context); - sinkHandle.setMaxBytesCanReserve(context.getOutputMaxBytes()); - context.getDriverContext().setSink(sinkHandle); + context.setISink(sinkHandle); return new IdentitySinkOperator(operatorContext, children, downStreamChannelIndex, sinkHandle); } @@ -2658,8 +2656,7 @@ public Operator visitShuffleSink(ShuffleSinkNode node, LocalExecutionPlanContext localInstanceId.toThrift(), node.getPlanNodeId().getId(), context.getInstanceContext()); - sinkHandle.setMaxBytesCanReserve(context.getOutputMaxBytes()); - context.getDriverContext().setSink(sinkHandle); + context.setISink(sinkHandle); return new ShuffleHelperOperator(operatorContext, children, downStreamChannelIndex, sinkHandle); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java new file mode 100644 index 0000000000000..f5a716a72b8f7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.queryengine.plan.planner; + +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +/** Select shared budgets only for FIs whose fan-in can consume any ready input. */ +final class SharedExchangeMemoryEligibility { + private SharedExchangeMemoryEligibility() {} + + static boolean isEligible(PlanNode root) { + boolean hasCollect = false; + Deque nodes = new ArrayDeque<>(); + nodes.push(root); + while (!nodes.isEmpty()) { + PlanNode node = nodes.pop(); + if (node instanceof ExchangeNode + || node + instanceof + org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode) { + // An upstream plan, if still attached, belongs to a different FI. + continue; + } + // Core relational nodes need not implement the serialization-only PlanNode.getType(). + boolean anyReady = + node instanceof CollectNode + || node instanceof UnionNode + || node + instanceof + org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.CollectNode; + hasCollect |= anyReady; + List children = node.getChildren(); + // Unknown multi-input operators are not implicitly safe. In particular, sequential sinks + // and joins can wait for an input while another input holds the entire shared budget. + if (!anyReady && children.size() > 1) { + return false; + } + nodes.addAll(children); + } + return hasCollect; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java index a92eaa5d0e283..8b97de5f3a259 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java @@ -254,4 +254,34 @@ public void run() { } } } + + @Test + public void testDependentQueuesKeepIndependentBudgetsByDefault() { + MemoryPool pool = new MemoryPool("pipeline-chain", new MemoryManager(1024), 512); + LocalMemoryManager manager = Mockito.mock(LocalMemoryManager.class); + Mockito.when(manager.getQueryPool()).thenReturn(pool); + ExecutorService executor = newDirectExecutorService(); + TFragmentInstanceId id = new TFragmentInstanceId("pipeline-chain", 0, "0"); + SharedTsBlockQueue upstream = new SharedTsBlockQueue(id, "a", manager, executor); + SharedTsBlockQueue downstream = new SharedTsBlockQueue(id, "b", manager, executor); + upstream.setMaxBytesCanReserve(256); + downstream.setMaxBytesCanReserve(256); + TsBlock block = Utils.createMockTsBlock(64); + try { + for (int i = 0; i < 4; i++) Assert.assertTrue(upstream.add(block).isDone()); + ListenableFuture fifth = upstream.add(block); + Assert.assertFalse(fifth.isDone()); + TsBlock forwarded = upstream.remove(); + Assert.assertTrue(fifth.isDone()); + Assert.assertTrue(downstream.add(forwarded).isDone()); + Assert.assertEquals(4, upstream.getNumOfBufferedTsBlocks()); + Assert.assertEquals(1, downstream.getNumOfBufferedTsBlocks()); + Assert.assertEquals(320, pool.getReservedBytes()); + } finally { + downstream.close(); + upstream.close(); + executor.shutdownNow(); + } + Assert.assertEquals(0, pool.getReservedBytes()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index 0254fa1ac33c3..c590e5a870dae 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -57,6 +57,7 @@ public class MemoryPoolTest { public void before() { MemoryManager memoryManager = Mockito.spy(new MemoryManager(1024L)); pool = new MemoryPool("test", memoryManager, 512L); + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, MemorySide.INPUT); } @@ -767,4 +768,91 @@ public void testConcurrentPendingReservationsCompleteAndReleaseAllAccounting() t Assert.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); } } + + @Test + public void testDefaultFragmentKeepsIndependentPlanNodeQuotas() { + MemoryPool legacy = new MemoryPool("legacy", new MemoryManager(1024), 512); + legacy.registerPlanNodeIdToQueryMemoryMap("q", "f", "a", MemorySide.INPUT); + legacy.registerPlanNodeIdToQueryMemoryMap("q", "f", "b", MemorySide.INPUT); + Assert.assertTrue( + legacy.reserveWithPriority("q", "f", "a", 256, 256, false).isReserveSuccess()); + Assert.assertTrue( + legacy.reserveWithPriority("q", "f", "b", 256, 256, false).isReserveSuccess()); + Assert.assertEquals(512, legacy.getFragmentMemoryReservedBytes("q", "f", MemorySide.INPUT)); + legacy.free("q", "f", "a", 256); + legacy.free("q", "f", "b", 256); + Assert.assertEquals(0, legacy.getReservedBytes()); + } + + @Test + public void testSharedAndLegacyFragmentsDoNotChangeEachOthersPolicy() { + pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "legacy", "a", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "legacy", "b", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, "b", MemorySide.INPUT); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256, 256, false) + .isReserveSuccess()); + MemoryReservationResult blocked = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, "b", 64, 256, false); + Assert.assertFalse(blocked.getFuture().isDone()); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, "legacy", "a", 256, 256, false).isReserveSuccess()); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, "legacy", "b", 256, 256, false).isReserveSuccess()); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256); + Assert.assertTrue(blocked.getFuture().isDone()); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, "b", 64); + pool.free(QUERY_ID, "legacy", "a", 256); + pool.free(QUERY_ID, "legacy", "b", 256); + Assert.assertEquals(0, pool.getReservedBytes()); + } + + @Test + public void testConcurrentAdmissionAlwaysGrantsOneRequestWhenOneFits() throws Exception { + for (int i = 0; i < 3; i++) { + String fi = "other" + i; + pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, fi, PLAN_NODE_ID, MemorySide.OUTPUT); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, fi, PLAN_NODE_ID, 256, 256, false).isReserveSuccess()); + } + pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, "b", MemorySide.INPUT); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + for (int i = 0; i < 100; i++) { + CountDownLatch start = new CountDownLatch(1); + Future first = + executor.submit( + () -> { + start.await(); + return pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256, 256, false); + }); + Future second = + executor.submit( + () -> { + start.await(); + return pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, "b", 256, 256, false); + }); + start.countDown(); + MemoryReservationResult a = first.get(10, TimeUnit.SECONDS); + MemoryReservationResult b = second.get(10, TimeUnit.SECONDS); + Assert.assertTrue(a.getFuture().isDone() ^ b.getFuture().isDone()); + String granted = a.getFuture().isDone() ? PLAN_NODE_ID : "b"; + String waiting = a.getFuture().isDone() ? "b" : PLAN_NODE_ID; + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, granted, 256); + Assert.assertTrue(a.getFuture().isDone()); + Assert.assertTrue(b.getFuture().isDone()); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, waiting, 256); + Assert.assertEquals(256, pool.getRemainingBytes()); + Assert.assertEquals( + 0, + pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.INPUT)); + } + } finally { + executor.shutdownNow(); + } + for (int i = 0; i < 3; i++) pool.free(QUERY_ID, "other" + i, PLAN_NODE_ID, 256); + Assert.assertEquals(0, pool.getReservedBytes()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java index 68bff035fe2c3..2d911fcd01897 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -19,60 +19,346 @@ package org.apache.iotdb.db.queryengine.plan.planner; +import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; +import org.apache.iotdb.calc.execution.operator.Operator; +import org.apache.iotdb.calc.execution.operator.process.TableMergeSortOperator; +import org.apache.iotdb.commons.memory.MemoryManager; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.common.PlanFragmentId; import org.apache.iotdb.db.queryengine.common.QueryId; -import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISink; +import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISinkHandle; import org.apache.iotdb.db.queryengine.execution.exchange.source.ISourceHandle; import org.apache.iotdb.db.queryengine.execution.exchange.source.PipelineSourceHandle; import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; +import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; +import com.google.common.collect.ArrayListMultimap; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.junit.After; +import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class LocalExecutionPlanContextBudgetTest { - @Test - public void testFixedBudgetsInheritedByPipelinesAndWiredBySide() { - long previous = + private long previousBudget; + private int previousBlockSize; + private MemoryPool pool; + private LocalExecutionPlanContext context; + + @Before + public void setUp() { + previousBudget = IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); - try { - IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(513); - FragmentInstanceContext instance = mock(FragmentInstanceContext.class); - when(instance.getId()) - .thenReturn(new FragmentInstanceId(new PlanFragmentId(new QueryId("budget"), 0), "0")); - LocalExecutionPlanContext context = - new LocalExecutionPlanContext(new TypeProvider(), instance, (DataNodeQueryContext) null); - context.setExchangeSumNum(500); - assertEquals(256, context.getInputMaxBytes()); - assertEquals(257, context.getOutputMaxBytes()); - IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(1024); - LocalExecutionPlanContext child = context.createSubContext(); - assertEquals(256, child.getInputMaxBytes()); - assertEquals(257, child.getOutputMaxBytes()); - ISink sink = mock(ISink.class); - child.setISink(sink); - verify(sink).setMaxBytesCanReserve(257); - ISourceHandle remote = mock(ISourceHandle.class); - PipelineSourceHandle pipeline = mock(PipelineSourceHandle.class); - ExchangeOperator remoteOperator = mock(ExchangeOperator.class); - ExchangeOperator pipelineOperator = mock(ExchangeOperator.class); - when(remoteOperator.getSourceHandle()).thenReturn(remote); - when(pipelineOperator.getSourceHandle()).thenReturn(pipeline); - context.addExchangeOperator(remoteOperator); - child.addExchangeOperator(pipelineOperator); - context.setExchangeMemoryLimits(); - verify(remote).setMaxBytesCanReserve(256); - verify(pipeline).setMaxBytesCanReserve(257); - } finally { - IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(previous); + previousBlockSize = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); + IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(1024); + TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(64); + pool = new MemoryPool("budget-test", new MemoryManager(4096), 1024); + context = newContext(); + } + + @After + public void tearDown() { + IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(previousBudget); + TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(previousBlockSize); + } + + private LocalExecutionPlanContext newContext() { + FragmentInstanceContext instance = mock(FragmentInstanceContext.class); + when(instance.getId()) + .thenReturn(new FragmentInstanceId(new PlanFragmentId(new QueryId("budget"), 0), "0")); + LocalExecutionPlanContext result = + new LocalExecutionPlanContext(new TypeProvider(), instance, (DataNodeQueryContext) null); + result.addPipelineDriverFactory(mock(Operator.class), result.getDriverContext(), 0); + result.setExchangeSumNum(4); + return result; + } + + private static PlanNode node(PlanNodeType type, PlanNode... children) { + Class nodeClass; + switch (type) { + case TABLE_EXCHANGE_NODE: + nodeClass = org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode.class; + break; + case EXCHANGE: + nodeClass = + org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode.class; + break; + case TABLE_COLLECT_NODE: + nodeClass = CollectNode.class; + break; + case TREE_COLLECT: + nodeClass = + org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.CollectNode.class; + break; + default: + nodeClass = PlanNode.class; + break; + } + PlanNode node = mock(nodeClass); + when(node.getType()).thenReturn(type); + when(node.getChildren()).thenReturn(Arrays.asList(children)); + return node; + } + + private static PlanNode collect() { + return new CollectNode( + new PlanNodeId("collect"), + Arrays.asList( + node(PlanNodeType.TABLE_EXCHANGE_NODE), node(PlanNodeType.TABLE_EXCHANGE_NODE)), + Collections.emptyList()); + } + + private ISourceHandle addSource(LocalExecutionPlanContext target) { + ISourceHandle handle = mock(ISourceHandle.class); + ExchangeOperator operator = mock(ExchangeOperator.class); + when(operator.getSourceHandle()).thenReturn(handle); + target.addExchangeOperator(operator); + return handle; + } + + private void assertAdmissionIsShared(boolean shared) { + String fi = context.getFragmentInstanceId().getFullId(); + pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "a", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "b", MemorySide.INPUT); + assertTrue(pool.reserveWithPriority("budget", fi, "a", 256, 256, false).isReserveSuccess()); + MemoryPool.MemoryReservationResult second = + pool.reserveWithPriority("budget", fi, "b", 256, 256, false); + if (shared) { + assertFalse(second.getFuture().isDone()); + pool.free("budget", fi, "a", 256); + assertTrue(second.getFuture().isDone()); + } else { + assertTrue(second.isReserveSuccess()); + pool.free("budget", fi, "a", 256); + } + pool.free("budget", fi, "b", 256); + } + + @Test + public void testPureCollectUsesSharedSideBudgets() { + ISourceHandle source = addSource(context); + ISinkHandle sink = mock(ISinkHandle.class); + context.setISink(sink); + context.setExchangeMemoryLimits(node(PlanNodeType.IDENTITY_SINK, collect()), pool); + verify(source).setMaxBytesCanReserve(512); + verify(sink).setMaxBytesCanReserve(512); + assertAdmissionIsShared(true); + } + + @Test + public void testMappingCollectUsesSharedBudget() { + PlanNode union = + new UnionNode( + new PlanNodeId("union"), + Arrays.asList( + node(PlanNodeType.TABLE_EXCHANGE_NODE), node(PlanNodeType.TABLE_EXCHANGE_NODE)), + ArrayListMultimap.create(), + Collections.emptyList()); + context.setExchangeMemoryLimits(union, pool); + assertAdmissionIsShared(true); + } + + @Test + public void testTreeCollectUsesSharedBudget() { + context.setExchangeMemoryLimits( + node(PlanNodeType.TREE_COLLECT, node(PlanNodeType.EXCHANGE), node(PlanNodeType.EXCHANGE)), + pool); + assertAdmissionIsShared(true); + } + + @Test + public void testNestedDependencyDisablesSharingForEntireFragment() { + for (PlanNodeType dependency : + new PlanNodeType[] { + PlanNodeType.TABLE_MERGESORT_NODE, PlanNodeType.MERGE_SORT, PlanNodeType.AGG_MERGE_SORT, + PlanNodeType.TABLE_JOIN_NODE, PlanNodeType.TABLE_SEMI_JOIN_NODE, + PlanNodeType.TABLE_TOPK_NODE, + PlanNodeType.COUNT_MERGE, PlanNodeType.DEVICE_VIEW + }) { + PlanNode dependent = + node( + dependency, + node(PlanNodeType.TABLE_EXCHANGE_NODE), + node(PlanNodeType.TABLE_EXCHANGE_NODE)); + assertFalse( + SharedExchangeMemoryEligibility.isEligible( + node(PlanNodeType.TABLE_COLLECT_NODE, dependent, collect()))); + assertFalse( + SharedExchangeMemoryEligibility.isEligible(node(dependency, collect(), collect()))); + } + ISourceHandle source = addSource(context); + ISinkHandle sink = mock(ISinkHandle.class); + context.setExchangeSumNum(2); + context.setISink(sink); + context.setExchangeSumNum(4); + context.setExchangeMemoryLimits( + node( + PlanNodeType.TABLE_COLLECT_NODE, + collect(), + node( + PlanNodeType.TABLE_MERGESORT_NODE, + node(PlanNodeType.TABLE_EXCHANGE_NODE), + node(PlanNodeType.TABLE_EXCHANGE_NODE))), + pool); + verify(source).setMaxBytesCanReserve(256); + // The legacy sink quota is captured at its original point in operator generation. + verify(sink).setMaxBytesCanReserve(512); + assertAdmissionIsShared(false); + } + + @Test + public void testSequentialAndUnknownMultiInputOperatorsDisableSharing() { + assertFalse( + SharedExchangeMemoryEligibility.isEligible( + node(PlanNodeType.IDENTITY_SINK, collect(), collect()))); + assertFalse( + SharedExchangeMemoryEligibility.isEligible( + node(PlanNodeType.SCHEMA_QUERY_MERGE, collect(), collect()))); + } + + @Test + public void testExchangeBoundaryDoesNotInspectUpstreamFragment() { + PlanNode remote = node(PlanNodeType.TABLE_MERGESORT_NODE, collect(), collect()); + assertTrue( + SharedExchangeMemoryEligibility.isEligible( + node( + PlanNodeType.TABLE_COLLECT_NODE, + node(PlanNodeType.TABLE_EXCHANGE_NODE, remote), + node(PlanNodeType.TABLE_EXCHANGE_NODE)))); + } + + @Test + public void testNoCollectKeepsLegacyBudget() { + context.setExchangeMemoryLimits( + node( + PlanNodeType.TABLE_MERGESORT_NODE, + node(PlanNodeType.TABLE_EXCHANGE_NODE), + node(PlanNodeType.TABLE_EXCHANGE_NODE)), + pool); + assertAdmissionIsShared(false); + } + + @Test + public void testInternalPipelineDisablesSharingForEntireFragment() { + LocalExecutionPlanContext child = context.createSubContext(); + child.addPipelineDriverFactory(mock(Operator.class), child.getDriverContext(), 0); + ISourceHandle source = addSource(child); + PipelineSourceHandle pipelineSource = mock(PipelineSourceHandle.class); + ExchangeOperator exchange = mock(ExchangeOperator.class); + when(exchange.getSourceHandle()).thenReturn(pipelineSource); + context.addExchangeOperator(exchange); + context.setExchangeMemoryLimits(collect(), pool); + verify(source).setMaxBytesCanReserve(256); + verify(pipelineSource).setMaxBytesCanReserve(256); + assertAdmissionIsShared(false); + } + + @Test + public void testPipelineSchedulingDependencyDisablesSharing() { + context.getCurrentPipelineDriverFactory().setDependencyPipeline(0); + context.setExchangeMemoryLimits(collect(), pool); + assertAdmissionIsShared(false); + } + + @Test + public void testSideSmallerThanOneBlockKeepsLegacyBudget() { + TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(1024); + context.setExchangeMemoryLimits(collect(), pool); + assertAdmissionIsShared(false); + } + + @Test + public void testSinkOnlyFragmentCanStillReserveOneFullBlock() { + IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(64); + context = newContext(); + context.setExchangeSumNum(1); + ISinkHandle sink = mock(ISinkHandle.class); + context.setISink(sink); + context.setExchangeMemoryLimits( + node(PlanNodeType.IDENTITY_SINK, node(PlanNodeType.SERIES_SCAN)), pool); + verify(sink).setMaxBytesCanReserve(64); + } + + @Test + public void testMergeSortGetsBothHeadsDespitePrefetchOnOneInput() throws Exception { + context.setExchangeMemoryLimits( + node(PlanNodeType.TABLE_MERGESORT_NODE, collect(), node(PlanNodeType.TABLE_EXCHANGE_NODE)), + pool); + String fi = context.getFragmentInstanceId().getFullId(); + pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "a", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "b", MemorySide.INPUT); + for (int i = 0; i < 4; i++) { + assertTrue(pool.reserveWithPriority("budget", fi, "a", 64, 256, false).isReserveSuccess()); } + MemoryPool.MemoryReservationResult prefetch = + pool.reserveWithPriority("budget", fi, "a", 64, 256, false); + assertFalse(prefetch.getFuture().isDone()); + MemoryPool.MemoryReservationResult headB = + pool.reserveWithPriority("budget", fi, "b", 64, 256, false); + assertTrue(headB.isReserveSuccess()); + Operator a = mock(Operator.class); + Operator b = mock(Operator.class); + doReturn(Operator.NOT_BLOCKED).when(a).isBlocked(); + doReturn(headB.getFuture()).when(b).isBlocked(); + when(a.hasNextWithTimer()).thenReturn(true); + when(b.hasNextWithTimer()).thenReturn(true); + TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.INT32)); + builder.getTimeColumnBuilder().writeLong(1); + builder.getColumnBuilder(0).writeInt(1); + builder.declarePosition(); + TsBlock first = builder.build(); + builder.reset(); + builder.getTimeColumnBuilder().writeLong(2); + builder.getColumnBuilder(0).writeInt(2); + builder.declarePosition(); + TsBlock second = builder.build(); + when(a.nextWithTimer()) + .thenAnswer( + call -> { + pool.free("budget", fi, "a", 64); + return first; + }); + when(b.nextWithTimer()) + .thenAnswer( + call -> { + pool.free("budget", fi, "b", 64); + return second; + }); + TableMergeSortOperator merge = + new TableMergeSortOperator( + mock(CommonOperatorContext.class), + new ArrayList<>(Arrays.asList(a, b)), + Collections.singletonList(TSDataType.INT32), + Comparator.comparingLong(key -> key.tsBlock.getTimeByIndex(key.rowIndex))); + assertTrue(merge.isBlocked().isDone()); + TsBlock result = merge.next(); + assertTrue(result != null && !result.isEmpty()); + assertTrue(prefetch.getFuture().isDone()); + pool.free("budget", fi, "a", 256); + merge.close(); } } From 1238bf20460f8060c774c9c27d516b4723a413b8 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 14:39:22 +0800 Subject: [PATCH 07/29] Preserve full exchange budgets for single-direction fragments Determine charged directions from remote sources and fragment sinks after planning. Split shared budgets only for bidirectional fragments and validate the maximum TsBlock size only for active directions. Add full-budget and local-input ownership regressions. --- ...2026-10-14-exchange-fanin-budget-design.md | 17 +++- .../execution/memory/MemorySide.java | 9 ++- .../planner/LocalExecutionPlanContext.java | 20 +++-- .../execution/memory/MemoryPoolTest.java | 29 ++++++- .../LocalExecutionPlanContextBudgetTest.java | 81 ++++++++++++++++++- 5 files changed, 144 insertions(+), 12 deletions(-) diff --git a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md index bc0b5c2557b22..1c84700602d88 100644 --- a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md +++ b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md @@ -37,7 +37,7 @@ TableTopK、CountMerge 的任意就绪消费留到后续；有序输入标志仍 在生成全部 pipeline 后再决定预算模式。内部 pipeline 队列具有转发依赖，因此存在多个 pipeline 或调度依赖时整组回退原策略。只含 Collect 的某个子树不会绕过 FI 级否决。 -通过上述检查后，F 为配置的单 FI 交换预算，INPUT=F/2，OUTPUT=F-INPUT。两侧均须容纳配置的最大 TsBlock，否则回退原配额，避免仅有输出的 FI 因预算减半而无法打开 SinkChannel。未启用共享的 FI 恢复原有 exchangeSumNum 配额计算和按 plan node 的申请上限；不能只恢复限额数值却仍按同侧总量进行准入。 +通过上述检查后，F 为配置的单 FI 交换预算。按照实际由本 FI 计费的方向分配：只有 INPUT 时 INPUT=F；只有 OUTPUT 时 OUTPUT=F；同时存在两侧时 INPUT=F/2、OUTPUT=F-INPUT；未使用的方向预算为 0。方向在完整本地计划生成后确定，远端 SourceHandle 表示本 FI 的 INPUT，fragment sink 表示 OUTPUT；LocalSourceHandle 的队列由生产 FI 计费，不重复占用接收 FI 的 INPUT。仅实际启用的方向须容纳配置的最大 TsBlock，否则回退原配额。未启用共享的 FI 恢复原有 exchangeSumNum 配额计算和按 plan node 的申请上限；不能只恢复限额数值却仍按同侧总量进行准入。 远端 SourceHandle 计入接收 FI 的 INPUT；SinkChannel 和 SharedTsBlockQueue 计入生产 FI 的 OUTPUT。 本地共享队列只计费一次，创建先后不改变所有者。 @@ -150,3 +150,18 @@ mvn -o test-compile -DskipTests ``` 本次生产代码没有新增未本地化的字符串，且没有修改中英文消息常量。 + +## 单方向预算修正验证（2026-10-15） + +移除默认按两侧均分的单参数接口。`MemorySide.getMaxBytes(F, hasInput, hasOutput)` +显式接收本 FI 实际计费的方向：单方向使用完整 F，双方向才拆分，未使用方向为 0。 +最大 TsBlock 校验只作用于启用的方向；原有 Collect 资格和依赖回退规则保持不变。 + +新增输入单向、输出单向、B=F、本地输入和本地/远端混合输入用例； +同时验证奇数 F、未使用方向及每个单方向一次申请完整 F 的边界。 +113 项定向单测全部通过。首轮中单块边界测试的默认测试池上限为 512、申请为 513， +将该测试池配置为目标 F=513 后重跑 MemoryPoolTest，38 项全部通过，其余 75 项首轮通过。 + +完整 reactor 中英文 test-compile、Spotless、Checkstyle 和许可证检查通过。 +重新从当前代码打包节点后，IoTDBCollectFanInIT 在 TableSimpleIT 和三 DataNode 的 +TableClusterIT 中各 8 项全部通过；集群复用同一份节点模板，节点包哈希与本次构建一致。 diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java index a59ef20d26848..d4c1908e2cc84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java @@ -24,7 +24,14 @@ public enum MemorySide { INPUT, OUTPUT; - public long getMaxBytes(long fragmentExchangeMaxBytes) { + /** Split only when both directions charge this FI; an unused direction has no budget. */ + public long getMaxBytes(long fragmentExchangeMaxBytes, boolean hasInput, boolean hasOutput) { + if (!(this == INPUT ? hasInput : hasOutput)) { + return 0; + } + if (!hasInput || !hasOutput) { + return fragmentExchangeMaxBytes; + } long inputMaxBytes = fragmentExchangeMaxBytes / 2; return this == INPUT ? inputMaxBytes : fragmentExchangeMaxBytes - inputMaxBytes; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java index 82c58cad65594..4c0a516270c46 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.execution.driver.SchemaDriverContext; import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISink; import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISinkHandle; +import org.apache.iotdb.db.queryengine.execution.exchange.source.SourceHandle; import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; @@ -272,16 +273,25 @@ public void addExchangeOperator(ExchangeOperator exchangeOperator) { } public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { - long inputMaxBytes = MemorySide.INPUT.getMaxBytes(fragmentExchangeMaxBytes); - long outputMaxBytes = MemorySide.OUTPUT.getMaxBytes(fragmentExchangeMaxBytes); + // Only remote sources charge this FI's INPUT. Local inputs are owned by the producing FI; + // internal pipeline queues cannot enable sharing because of the pipeline check below. + boolean hasInput = + exchangeOperatorList.stream() + .anyMatch(exchange -> exchange.getSourceHandle() instanceof SourceHandle); + boolean hasOutput = !legacySinkLimits.isEmpty(); + long inputMaxBytes = + MemorySide.INPUT.getMaxBytes(fragmentExchangeMaxBytes, hasInput, hasOutput); + long outputMaxBytes = + MemorySide.OUTPUT.getMaxBytes(fragmentExchangeMaxBytes, hasInput, hasOutput); long maxBlockBytes = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); boolean sharedBudget = - SharedExchangeMemoryEligibility.isEligible(plan) + (hasInput || hasOutput) + && SharedExchangeMemoryEligibility.isEligible(plan) // Internal pipeline queues and their downstream sinks must not compete for OUTPUT. && pipelineDriverFactories.size() == 1 && pipelineDriverFactories.get(0).getDependencyPipelineIndex() == -1 - && inputMaxBytes >= maxBlockBytes - && outputMaxBytes >= maxBlockBytes; + && (!hasInput || inputMaxBytes >= maxBlockBytes) + && (!hasOutput || outputMaxBytes >= maxBlockBytes); if (sharedBudget) { FragmentInstanceId id = getFragmentInstanceId(); memoryPool.registerFragmentInstanceSharedBudget(id.getQueryId().getId(), id.getFullId()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index c590e5a870dae..bf9b497ede88c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -577,8 +577,8 @@ public void testSideRegistrationCannotChange() { @Test public void testOddBudgetAndSingleRequestBoundary() { - Assert.assertEquals(256, MemorySide.INPUT.getMaxBytes(513)); - Assert.assertEquals(257, MemorySide.OUTPUT.getMaxBytes(513)); + Assert.assertEquals(256, MemorySide.INPUT.getMaxBytes(513, true, true)); + Assert.assertEquals(257, MemorySide.OUTPUT.getMaxBytes(513, true, true)); try { pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 257, 256, false); Assert.fail("request exceeds the input budget"); @@ -588,6 +588,31 @@ public void testOddBudgetAndSingleRequestBoundary() { } } + @Test + public void testSingleDirectionKeepsFullBudgetAndSingleBlockLimit() { + Assert.assertEquals(513, MemorySide.INPUT.getMaxBytes(513, true, false)); + Assert.assertEquals(0, MemorySide.OUTPUT.getMaxBytes(513, true, false)); + Assert.assertEquals(0, MemorySide.INPUT.getMaxBytes(513, false, true)); + Assert.assertEquals(513, MemorySide.OUTPUT.getMaxBytes(513, false, true)); + Assert.assertEquals(0, MemorySide.INPUT.getMaxBytes(513, false, false)); + Assert.assertEquals(0, MemorySide.OUTPUT.getMaxBytes(513, false, false)); + MemoryPool singleDirectionPool = + new MemoryPool("single-direction", new MemoryManager(1024), 513); + singleDirectionPool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); + for (MemorySide side : MemorySide.values()) { + String node = side.name(); + singleDirectionPool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, node, side); + long limit = side.getMaxBytes(513, side == MemorySide.INPUT, side == MemorySide.OUTPUT); + Assert.assertTrue( + singleDirectionPool + .reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, node, 513, limit, false) + .isReserveSuccess()); + singleDirectionPool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, node, 513); + Assert.assertEquals(0, singleDirectionPool.getReservedBytes()); + } + } + @Test public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(16); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java index 2d911fcd01897..2f1035245bfce 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -34,7 +34,9 @@ import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISinkHandle; import org.apache.iotdb.db.queryengine.execution.exchange.source.ISourceHandle; +import org.apache.iotdb.db.queryengine.execution.exchange.source.LocalSourceHandle; import org.apache.iotdb.db.queryengine.execution.exchange.source.PipelineSourceHandle; +import org.apache.iotdb.db.queryengine.execution.exchange.source.SourceHandle; import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; @@ -133,7 +135,7 @@ private static PlanNode collect() { } private ISourceHandle addSource(LocalExecutionPlanContext target) { - ISourceHandle handle = mock(ISourceHandle.class); + ISourceHandle handle = mock(SourceHandle.class); ExchangeOperator operator = mock(ExchangeOperator.class); when(operator.getSourceHandle()).thenReturn(handle); target.addExchangeOperator(operator); @@ -141,9 +143,13 @@ private ISourceHandle addSource(LocalExecutionPlanContext target) { } private void assertAdmissionIsShared(boolean shared) { + assertAdmissionIsShared(shared, MemorySide.INPUT); + } + + private void assertAdmissionIsShared(boolean shared, MemorySide side) { String fi = context.getFragmentInstanceId().getFullId(); - pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "a", MemorySide.INPUT); - pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "b", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "a", side); + pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "b", side); assertTrue(pool.reserveWithPriority("budget", fi, "a", 256, 256, false).isReserveSuccess()); MemoryPool.MemoryReservationResult second = pool.reserveWithPriority("budget", fi, "b", 256, 256, false); @@ -169,8 +175,74 @@ public void testPureCollectUsesSharedSideBudgets() { assertAdmissionIsShared(true); } + @Test + public void testInputOnlyCollectUsesFullBudgetEvenForOneFullBlock() { + TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(1024); + context.setExchangeSumNum(1); + ISourceHandle source = addSource(context); + context.setExchangeMemoryLimits( + node(PlanNodeType.TABLE_COLLECT_NODE, node(PlanNodeType.TABLE_EXCHANGE_NODE)), pool); + verify(source).setMaxBytesCanReserve(1024); + assertAdmissionIsShared(true); + } + + @Test + public void testOutputOnlyCollectUsesFullBudgetEvenForOneFullBlock() { + TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(1024); + context.setExchangeSumNum(1); + ISinkHandle sink = mock(ISinkHandle.class); + context.setISink(sink); + context.setExchangeMemoryLimits( + node( + PlanNodeType.IDENTITY_SINK, + node( + PlanNodeType.TABLE_COLLECT_NODE, + node(PlanNodeType.SERIES_SCAN), + node(PlanNodeType.SERIES_SCAN))), + pool); + verify(sink).setMaxBytesCanReserve(1024); + assertAdmissionIsShared(true, MemorySide.OUTPUT); + } + + private void addLocalInput() { + ExchangeOperator exchange = mock(ExchangeOperator.class); + when(exchange.getSourceHandle()).thenReturn(mock(LocalSourceHandle.class)); + context.addExchangeOperator(exchange); + } + + @Test + public void testLocalInputsDoNotSplitOutputBudget() { + TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(1024); + addLocalInput(); + ISinkHandle sink = mock(ISinkHandle.class); + context.setISink(sink); + context.setExchangeMemoryLimits(node(PlanNodeType.IDENTITY_SINK, collect()), pool); + verify(sink).setMaxBytesCanReserve(1024); + assertAdmissionIsShared(true, MemorySide.OUTPUT); + } + + @Test + public void testMixedLocalAndRemoteInputsStillSplitBidirectionalBudget() { + addLocalInput(); + ISourceHandle source = addSource(context); + ISinkHandle sink = mock(ISinkHandle.class); + context.setISink(sink); + context.setExchangeMemoryLimits(node(PlanNodeType.IDENTITY_SINK, collect()), pool); + verify(source).setMaxBytesCanReserve(512); + verify(sink).setMaxBytesCanReserve(512); + assertAdmissionIsShared(true); + } + + @Test + public void testLocalInputsWithoutSinkDoNotRegisterUnusedSharedBudget() { + addLocalInput(); + context.setExchangeMemoryLimits(collect(), pool); + assertAdmissionIsShared(false); + } + @Test public void testMappingCollectUsesSharedBudget() { + addSource(context); PlanNode union = new UnionNode( new PlanNodeId("union"), @@ -184,6 +256,7 @@ public void testMappingCollectUsesSharedBudget() { @Test public void testTreeCollectUsesSharedBudget() { + addSource(context); context.setExchangeMemoryLimits( node(PlanNodeType.TREE_COLLECT, node(PlanNodeType.EXCHANGE), node(PlanNodeType.EXCHANGE)), pool); @@ -286,6 +359,8 @@ public void testPipelineSchedulingDependencyDisablesSharing() { @Test public void testSideSmallerThanOneBlockKeepsLegacyBudget() { + addSource(context); + context.setISink(mock(ISinkHandle.class)); TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(1024); context.setExchangeMemoryLimits(collect(), pool); assertAdmissionIsShared(false); From 428d00729025b8f02cdc4cdc96112aa3f4fadf59 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 15:16:31 +0800 Subject: [PATCH 08/29] Initialize shared exchange limits in fragment memory records --- ...2026-10-14-exchange-fanin-budget-design.md | 20 ++++- .../execution/memory/MemoryPool.java | 31 +++++++- .../execution/memory/MemorySide.java | 14 +--- .../planner/LocalExecutionPlanContext.java | 24 +++--- .../execution/memory/MemoryPoolTest.java | 76 ++++++++++++++----- .../LocalExecutionPlanContextBudgetTest.java | 40 +++++++--- 6 files changed, 148 insertions(+), 57 deletions(-) diff --git a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md index 1c84700602d88..87dcfa5fb8c90 100644 --- a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md +++ b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md @@ -42,6 +42,7 @@ TableTopK、CountMerge 的任意就绪消费留到后续；有序输入标志仍 本地共享队列只计费一次，创建先后不改变所有者。 MemoryPool 维护 query→FI→planNode 明细，注册时固定 side。 +本地计划生成全部算子和 pipeline 后，一次性检查共享资格并计算两个方向的预算，写入 FI 记录的 `inputMaxBytes`、`outputMaxBytes` 字段。`MemorySide` 仅表示方向，不计算预算，也不接受 `hasInput`/`hasOutput` 参数。共享模式的申请、单块上限检查和等待唤醒统一读取 FI 保存的预算；handle 传入的限额仅用于原有独立配额模式。 FI 两侧保留 AtomicLong 统计，plan node 保留字节明细。申请和释放在 FI 的记账锁内更新；全局池采用有界原子申请。申请前先检查适用的同侧总量或 plan node 限额，失败不产生临时计费，也不需要回滚，避免并发请求分别撞到不同限额后全部挂起。 释放后在 FI 记账锁外完成等待 Future；保留入队后的重试和每个 Future 的完成/取消同步。稳定快照中的 FI 同侧总量等于同侧明细之和。 @@ -153,8 +154,9 @@ mvn -o test-compile -DskipTests ## 单方向预算修正验证（2026-10-15） -移除默认按两侧均分的单参数接口。`MemorySide.getMaxBytes(F, hasInput, hasOutput)` -显式接收本 FI 实际计费的方向：单方向使用完整 F，双方向才拆分，未使用方向为 0。 +该轮修正取消默认按两侧均分，改为根据本 FI 实际计费方向分配： +单方向使用完整 F，双方向才拆分，未使用方向为 0。本次已将计算统一收回本地计划初始化， +不再保留该轮引入的 `MemorySide.getMaxBytes(F, hasInput, hasOutput)` 接口。 最大 TsBlock 校验只作用于启用的方向；原有 Collect 资格和依赖回退规则保持不变。 新增输入单向、输出单向、B=F、本地输入和本地/远端混合输入用例； @@ -165,3 +167,17 @@ mvn -o test-compile -DskipTests 完整 reactor 中英文 test-compile、Spotless、Checkstyle 和许可证检查通过。 重新从当前代码打包节点后，IoTDBCollectFanInIT 在 TableSimpleIT 和三 DataNode 的 TableClusterIT 中各 8 项全部通过；集群复用同一份节点模板，节点包哈希与本次构建一致。 + +## FI 预算字段初始化验证（2026-10-15） + +`FragmentMemoryReservation` 保存 `inputMaxBytes`、`outputMaxBytes`； +完整本地计划生成后按输入 exchange、fragment sink 和依赖检查结果一次性初始化。 +共享申请、单块上限检查与等待重试均读取 FI 字段，独立配额模式仍读取 handle 限额。 +`MemorySide` 只保留方向枚举，移除布尔参数预算接口。 + +115 项定向单测通过，覆盖 handle 参数不能增减共享上限、超额请求不进入等待队列、 +未使用方向预算为零、奇数预算拆分和原有独立配额回退。 +完整 reactor 中英文编译、Spotless、Checkstyle 和许可证检查通过；生产代码没有新增字符串字面量。 + +重新打包当前源码后，`IoTDBCollectFanInIT` 在 TableSimpleIT 和三 DataNode 的 TableClusterIT +中各 8 项全部通过，无失败或跳过。集群复用同一份节点模板，其服务端 jar 哈希与本次构建一致。 diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index 84f36b52e4ed3..98108b324758e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -59,8 +59,8 @@ public static class MemoryReservationFuture extends AbstractFuture { /** * MemoryReservationFuture is created when an exchange handle tries to reserve memory from the - * pool. For opted-in fragments the limit is shared by all plan nodes on the same side; - * otherwise it applies independently to the plan node. + * pool. This is the per-plan-node limit for legacy fragments. Shared fragments always use their + * initialized FI-side limit, including when retrying a pending reservation. */ private final long maxBytesCanReserve; @@ -176,6 +176,14 @@ private static class FragmentMemoryReservation { private final AtomicLong outputUsedBytes = new AtomicLong(); // Set by local planning before this FI starts reserving exchange memory. private boolean sharedBudget; + private long inputMaxBytes; + private long outputMaxBytes; + + private long getMaxBytesCanReserve(PlanNodeReservation reservation, long legacyLimit) { + return sharedBudget + ? (reservation.side == MemorySide.INPUT ? inputMaxBytes : outputMaxBytes) + : legacyLimit; + } private AtomicLong getUsedBytes(MemorySide side) { return side == MemorySide.INPUT ? inputUsedBytes : outputUsedBytes; @@ -256,14 +264,20 @@ public void registerPlanNodeIdToQueryMemoryMap( } } - /** Opt in only after the complete FI has been checked for input and pipeline dependencies. */ - public void registerFragmentInstanceSharedBudget(String queryId, String fragmentInstanceId) { + /** + * Initialize once after the complete FI has been checked for input and pipeline dependencies, + * before any driver starts reserving exchange memory. An unused direction has a zero budget. + */ + public void registerFragmentInstanceSharedBudget( + String queryId, String fragmentInstanceId, long inputMaxBytes, long outputMaxBytes) { synchronized (queryMemoryReservations) { FragmentMemoryReservation fragment = queryMemoryReservations .computeIfAbsent(queryId, x -> new ConcurrentHashMap<>()) .computeIfAbsent(fragmentInstanceId, x -> new FragmentMemoryReservation()); synchronized (fragment) { + fragment.inputMaxBytes = inputMaxBytes; + fragment.outputMaxBytes = outputMaxBytes; fragment.sharedBudget = true; } } @@ -354,6 +368,14 @@ public MemoryReservationResult reserveWithPriority( .EXCEPTION_BYTESTORESERVE_SHOULD_BE_IN_LEFT_PAREN_0_COMMA_MAXBYTESPERFI_RIGHT_BRACKET_DOT_M_0753BB69, maxBytesPerFragmentInstance, bytesToReserve); + FragmentMemoryReservation fragment = + getFragmentReservation(queryId, fragmentInstanceId, planNodeId); + synchronized (fragment) { + PlanNodeReservation reservation = + requireReservation( + fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); + maxBytesCanReserve = fragment.getMaxBytesCanReserve(reservation, maxBytesCanReserve); + } if (bytesToReserve > maxBytesCanReserve) { LOGGER.warn( DataNodeQueryMessages @@ -550,6 +572,7 @@ public boolean tryReserve( PlanNodeReservation reservation = requireReservation( fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); + maxBytesCanReserve = fragment.getMaxBytesCanReserve(reservation, maxBytesCanReserve); long usedBytes = fragment.sharedBudget ? fragment.getUsedBytes(reservation.side).get() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java index d4c1908e2cc84..7149960538559 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemorySide.java @@ -22,17 +22,5 @@ /** The exchange budget charged by a registered plan node. */ public enum MemorySide { INPUT, - OUTPUT; - - /** Split only when both directions charge this FI; an unused direction has no budget. */ - public long getMaxBytes(long fragmentExchangeMaxBytes, boolean hasInput, boolean hasOutput) { - if (!(this == INPUT ? hasInput : hasOutput)) { - return 0; - } - if (!hasInput || !hasOutput) { - return fragmentExchangeMaxBytes; - } - long inputMaxBytes = fragmentExchangeMaxBytes / 2; - return this == INPUT ? inputMaxBytes : fragmentExchangeMaxBytes - inputMaxBytes; - } + OUTPUT } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java index 4c0a516270c46..e78240ec0bce1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java @@ -37,7 +37,6 @@ import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; -import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; import org.apache.iotdb.db.queryengine.plan.analyze.TemplatedInfo; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; @@ -275,26 +274,29 @@ public void addExchangeOperator(ExchangeOperator exchangeOperator) { public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { // Only remote sources charge this FI's INPUT. Local inputs are owned by the producing FI; // internal pipeline queues cannot enable sharing because of the pipeline check below. - boolean hasInput = + long inputExchangeCount = exchangeOperatorList.stream() - .anyMatch(exchange -> exchange.getSourceHandle() instanceof SourceHandle); - boolean hasOutput = !legacySinkLimits.isEmpty(); + .filter(exchange -> exchange.getSourceHandle() instanceof SourceHandle) + .count(); + int outputSinkCount = legacySinkLimits.size(); long inputMaxBytes = - MemorySide.INPUT.getMaxBytes(fragmentExchangeMaxBytes, hasInput, hasOutput); - long outputMaxBytes = - MemorySide.OUTPUT.getMaxBytes(fragmentExchangeMaxBytes, hasInput, hasOutput); + inputExchangeCount == 0 + ? 0 + : (outputSinkCount == 0 ? fragmentExchangeMaxBytes : fragmentExchangeMaxBytes / 2); + long outputMaxBytes = outputSinkCount == 0 ? 0 : fragmentExchangeMaxBytes - inputMaxBytes; long maxBlockBytes = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); boolean sharedBudget = - (hasInput || hasOutput) + (inputMaxBytes > 0 || outputMaxBytes > 0) && SharedExchangeMemoryEligibility.isEligible(plan) // Internal pipeline queues and their downstream sinks must not compete for OUTPUT. && pipelineDriverFactories.size() == 1 && pipelineDriverFactories.get(0).getDependencyPipelineIndex() == -1 - && (!hasInput || inputMaxBytes >= maxBlockBytes) - && (!hasOutput || outputMaxBytes >= maxBlockBytes); + && (inputExchangeCount == 0 || inputMaxBytes >= maxBlockBytes) + && (outputSinkCount == 0 || outputMaxBytes >= maxBlockBytes); if (sharedBudget) { FragmentInstanceId id = getFragmentInstanceId(); - memoryPool.registerFragmentInstanceSharedBudget(id.getQueryId().getId(), id.getFullId()); + memoryPool.registerFragmentInstanceSharedBudget( + id.getQueryId().getId(), id.getFullId(), inputMaxBytes, outputMaxBytes); } legacySinkLimits.forEach( (sink, legacyLimit) -> diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index bf9b497ede88c..136e317e665bb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -57,7 +57,6 @@ public class MemoryPoolTest { public void before() { MemoryManager memoryManager = Mockito.spy(new MemoryManager(1024L)); pool = new MemoryPool("test", memoryManager, 512L); - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, MemorySide.INPUT); } @@ -530,6 +529,7 @@ public void testReserveWithPriorityShowQueriesWithinAvailable() { @Test public void testSharedInputBudgetAndIndependentOutput() { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap( @@ -576,45 +576,80 @@ public void testSideRegistrationCannotChange() { } @Test - public void testOddBudgetAndSingleRequestBoundary() { - Assert.assertEquals(256, MemorySide.INPUT.getMaxBytes(513, true, true)); - Assert.assertEquals(257, MemorySide.OUTPUT.getMaxBytes(513, true, true)); + public void testSharedRequestCannotExceedInitializedInputBudget() { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); try { - pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 257, 256, false); - Assert.fail("request exceeds the input budget"); + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 257, Long.MAX_VALUE, false); + Assert.fail("request exceeds the initialized input budget"); } catch (IllegalArgumentException expected) { Assert.assertEquals(0, pool.getReservedBytes()); Assert.assertEquals(0, pool.getQueryMemoryReservedBytes(QUERY_ID)); + Assert.assertEquals(0, pool.getMemoryReservationSize()); } } + @Test + public void testHandleLimitsCannotChangeInitializedSharedBudgets() throws Exception { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, "output", MemorySide.OUTPUT); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 200, 1, false) + .isReserveSuccess()); + MemoryReservationResult pending = + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 100, Long.MAX_VALUE, false); + Assert.assertFalse(pending.getFuture().isDone()); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, "output", 256, 1, false) + .isReserveSuccess()); + // Waking the input request must still respect the FI's saved limit. + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, "output", 256); + Assert.assertFalse(pending.getFuture().isDone()); + Assert.assertFalse( + pool.tryReserveForTest(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 100, Long.MAX_VALUE)); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 200); + pending.getFuture().get(10, TimeUnit.SECONDS); + Assert.assertEquals( + 100, pool.getFragmentMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.INPUT)); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 100); + Assert.assertEquals(0, pool.getReservedBytes()); + Assert.assertEquals(0, pool.getMemoryReservationSize()); + } + @Test public void testSingleDirectionKeepsFullBudgetAndSingleBlockLimit() { - Assert.assertEquals(513, MemorySide.INPUT.getMaxBytes(513, true, false)); - Assert.assertEquals(0, MemorySide.OUTPUT.getMaxBytes(513, true, false)); - Assert.assertEquals(0, MemorySide.INPUT.getMaxBytes(513, false, true)); - Assert.assertEquals(513, MemorySide.OUTPUT.getMaxBytes(513, false, true)); - Assert.assertEquals(0, MemorySide.INPUT.getMaxBytes(513, false, false)); - Assert.assertEquals(0, MemorySide.OUTPUT.getMaxBytes(513, false, false)); MemoryPool singleDirectionPool = new MemoryPool("single-direction", new MemoryManager(1024), 513); - singleDirectionPool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); for (MemorySide side : MemorySide.values()) { - String node = side.name(); + String fi = side.name(); + singleDirectionPool.registerFragmentInstanceSharedBudget( + QUERY_ID, fi, side == MemorySide.INPUT ? 513 : 0, side == MemorySide.OUTPUT ? 513 : 0); + singleDirectionPool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, fi, "active", side); singleDirectionPool.registerPlanNodeIdToQueryMemoryMap( - QUERY_ID, FRAGMENT_INSTANCE_ID, node, side); - long limit = side.getMaxBytes(513, side == MemorySide.INPUT, side == MemorySide.OUTPUT); + QUERY_ID, fi, "unused", side == MemorySide.INPUT ? MemorySide.OUTPUT : MemorySide.INPUT); Assert.assertTrue( singleDirectionPool - .reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, node, 513, limit, false) + .reserveWithPriority(QUERY_ID, fi, "active", 513, 1, false) .isReserveSuccess()); - singleDirectionPool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, node, 513); + try { + singleDirectionPool.reserveWithPriority(QUERY_ID, fi, "unused", 1, Long.MAX_VALUE, false); + Assert.fail("an unused direction must not acquire a budget from a handle"); + } catch (IllegalArgumentException expected) { + Assert.assertEquals(0, singleDirectionPool.getMemoryReservationSize()); + Assert.assertEquals(513, singleDirectionPool.getReservedBytes()); + } + singleDirectionPool.free(QUERY_ID, fi, "active", 513); Assert.assertEquals(0, singleDirectionPool.getReservedBytes()); } } @Test public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 128, 128); ExecutorService executor = Executors.newFixedThreadPool(16); CountDownLatch start = new CountDownLatch(1); CountDownLatch attempted = new CountDownLatch(16); @@ -662,6 +697,7 @@ public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { @Test public void testReleaseBeforePendingReservationIsPublished() throws Exception { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); pool = Mockito.spy(pool); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); @@ -714,6 +750,7 @@ public void testReleaseBeforePendingReservationIsPublished() throws Exception { @Test public void testCancelledInputDoesNotConsumeReleasedSharedBudget() throws Exception { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "cancelled", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap( @@ -751,6 +788,7 @@ public void testCancelledInputDoesNotConsumeReleasedSharedBudget() throws Except @Test public void testConcurrentPendingReservationsCompleteAndReleaseAllAccounting() throws Exception { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 128, 128); int count = 16; ExecutorService executor = Executors.newFixedThreadPool(count); CountDownLatch attempted = new CountDownLatch(count); @@ -811,6 +849,7 @@ public void testDefaultFragmentKeepsIndependentPlanNodeQuotas() { @Test public void testSharedAndLegacyFragmentsDoNotChangeEachOthersPolicy() { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "legacy", "a", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "legacy", "b", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, "b", MemorySide.INPUT); @@ -834,6 +873,7 @@ public void testSharedAndLegacyFragmentsDoNotChangeEachOthersPolicy() { @Test public void testConcurrentAdmissionAlwaysGrantsOneRequestWhenOneFits() throws Exception { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); for (int i = 0; i < 3; i++) { String fi = "other" + i; pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, fi, PLAN_NODE_ID, MemorySide.OUTPUT); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java index 2f1035245bfce..7d2e23e9d554e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -147,21 +147,26 @@ private void assertAdmissionIsShared(boolean shared) { } private void assertAdmissionIsShared(boolean shared, MemorySide side) { + assertAdmissionIsShared(shared, side, shared ? 1024 : 256); + } + + private void assertAdmissionIsShared(boolean shared, MemorySide side, long limit) { String fi = context.getFragmentInstanceId().getFullId(); - pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "a", side); - pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "b", side); - assertTrue(pool.reserveWithPriority("budget", fi, "a", 256, 256, false).isReserveSuccess()); + pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "a" + side, side); + pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "b" + side, side); + assertTrue( + pool.reserveWithPriority("budget", fi, "a" + side, limit, limit, false).isReserveSuccess()); MemoryPool.MemoryReservationResult second = - pool.reserveWithPriority("budget", fi, "b", 256, 256, false); + pool.reserveWithPriority("budget", fi, "b" + side, limit, limit, false); if (shared) { assertFalse(second.getFuture().isDone()); - pool.free("budget", fi, "a", 256); + pool.free("budget", fi, "a" + side, limit); assertTrue(second.getFuture().isDone()); } else { assertTrue(second.isReserveSuccess()); - pool.free("budget", fi, "a", 256); + pool.free("budget", fi, "a" + side, limit); } - pool.free("budget", fi, "b", 256); + pool.free("budget", fi, "b" + side, limit); } @Test @@ -172,7 +177,23 @@ public void testPureCollectUsesSharedSideBudgets() { context.setExchangeMemoryLimits(node(PlanNodeType.IDENTITY_SINK, collect()), pool); verify(source).setMaxBytesCanReserve(512); verify(sink).setMaxBytesCanReserve(512); - assertAdmissionIsShared(true); + assertAdmissionIsShared(true, MemorySide.INPUT, 512); + assertAdmissionIsShared(true, MemorySide.OUTPUT, 512); + } + + @Test + public void testOddBudgetIsInitializedAfterBothDirectionsAreKnown() { + IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(1025); + pool = new MemoryPool("odd-budget", new MemoryManager(4096), 1025); + context = newContext(); + ISourceHandle source = addSource(context); + ISinkHandle sink = mock(ISinkHandle.class); + context.setISink(sink); + context.setExchangeMemoryLimits(node(PlanNodeType.IDENTITY_SINK, collect()), pool); + verify(source).setMaxBytesCanReserve(512); + verify(sink).setMaxBytesCanReserve(513); + assertAdmissionIsShared(true, MemorySide.INPUT, 512); + assertAdmissionIsShared(true, MemorySide.OUTPUT, 513); } @Test @@ -230,7 +251,8 @@ public void testMixedLocalAndRemoteInputsStillSplitBidirectionalBudget() { context.setExchangeMemoryLimits(node(PlanNodeType.IDENTITY_SINK, collect()), pool); verify(source).setMaxBytesCanReserve(512); verify(sink).setMaxBytesCanReserve(512); - assertAdmissionIsShared(true); + assertAdmissionIsShared(true, MemorySide.INPUT, 512); + assertAdmissionIsShared(true, MemorySide.OUTPUT, 512); } @Test From 5bac76c7adfed30ec919c3315976697722acb09f Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 15:42:07 +0800 Subject: [PATCH 09/29] Move exchange budget configuration into fragment instance context --- ...2026-10-14-exchange-fanin-budget-design.md | 17 ++++++- .../fragment/FragmentInstanceContext.java | 18 ++++++++ .../execution/memory/MemoryPool.java | 33 +++++++------- .../planner/LocalExecutionPlanContext.java | 7 +-- .../execution/memory/MemoryPoolTest.java | 44 ++++++++++++++----- .../LocalExecutionPlanContextBudgetTest.java | 23 ++++++++-- 6 files changed, 108 insertions(+), 34 deletions(-) diff --git a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md index 87dcfa5fb8c90..3eb5cbc906259 100644 --- a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md +++ b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md @@ -42,7 +42,7 @@ TableTopK、CountMerge 的任意就绪消费留到后续；有序输入标志仍 本地共享队列只计费一次，创建先后不改变所有者。 MemoryPool 维护 query→FI→planNode 明细，注册时固定 side。 -本地计划生成全部算子和 pipeline 后，一次性检查共享资格并计算两个方向的预算，写入 FI 记录的 `inputMaxBytes`、`outputMaxBytes` 字段。`MemorySide` 仅表示方向，不计算预算，也不接受 `hasInput`/`hasOutput` 参数。共享模式的申请、单块上限检查和等待唤醒统一读取 FI 保存的预算；handle 传入的限额仅用于原有独立配额模式。 +本地计划生成全部算子和 pipeline 后，一次性检查共享资格并计算两个方向的预算，写入 `FragmentInstanceContext` 的 `inputMaxBytes`、`outputMaxBytes` 字段。`MemorySide` 仅表示方向，不计算预算，也不接受 `hasInput`/`hasOutput` 参数。MemoryPool 的 FI 账目仅保留该上下文的引用，不复制预算字段。共享模式的申请、单块上限检查和等待唤醒统一读取 FI 保存的预算；handle 传入的限额仅用于原有独立配额模式。回退独立配额时上下文的两个共享预算字段为零。 FI 两侧保留 AtomicLong 统计，plan node 保留字节明细。申请和释放在 FI 的记账锁内更新；全局池采用有界原子申请。申请前先检查适用的同侧总量或 plan node 限额，失败不产生临时计费，也不需要回滚，避免并发请求分别撞到不同限额后全部挂起。 释放后在 FI 记账锁外完成等待 Future；保留入队后的重试和每个 Future 的完成/取消同步。稳定快照中的 FI 同侧总量等于同侧明细之和。 @@ -181,3 +181,18 @@ TableClusterIT 中各 8 项全部通过；集群复用同一份节点模板， 重新打包当前源码后，`IoTDBCollectFanInIT` 在 TableSimpleIT 和三 DataNode 的 TableClusterIT 中各 8 项全部通过，无失败或跳过。集群复用同一份节点模板，其服务端 jar 哈希与本次构建一致。 + +## FI 预算配置归属修正验证（2026-10-15） + +预算字段移至 `FragmentInstanceContext`，由完整本地计划生成后的初始化流程赋值。 +MemoryPool 的 FI 账目保存上下文引用，通过 getter 读取限额，不再复制预算或保存共享布尔字段。 +该引用随 FI 记账记录注销；预算仍在 driver 启动前初始化。 +独立配额回退时上下文的两个共享预算字段为零，不注册共享预算。 + +123 项定向单测通过，包括真实 FI 上下文的初始化字段断言、申请上限和等待唤醒回归， +以及已有的 FI 上下文生命周期测试。生产代码没有新增字符串字面量。 + +本次单测在前述 `-Dtest` 列表中加入 `FragmentInstanceContextTest`。完整 reactor 的中英文 +`test-compile -DskipTests`、Spotless、Checkstyle、许可证与 `git diff --check` 检查均通过。 +重新打包节点后，`IoTDBCollectFanInIT` 在 TableSimpleIT 和三 DataNode 的 TableClusterIT 中各 +8 项全部通过，无失败或跳过；测试模板与当前服务端 jar 的哈希一致。 diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 5b8ee6ae1c1ae..16fecf6bc4983 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -120,6 +120,11 @@ public class FragmentInstanceContext extends QueryContext { private final MemoryReservationManager memoryReservationManager; + // Initialized after the complete local plan is generated, before drivers start. Both are zero + // when this FI uses independent per-plan-node exchange quotas. + private long inputMaxBytes; + private long outputMaxBytes; + protected IDataRegionForQuery dataRegion; private Filter globalTimeFilter; private List globalTimeFilterTimeRanges; @@ -551,6 +556,19 @@ public FragmentInstanceId getId() { return id; } + public void initializeExchangeMemoryBudget(long inputMaxBytes, long outputMaxBytes) { + this.inputMaxBytes = inputMaxBytes; + this.outputMaxBytes = outputMaxBytes; + } + + public long getInputMaxBytes() { + return inputMaxBytes; + } + + public long getOutputMaxBytes() { + return outputMaxBytes; + } + public void failed(Throwable cause) { stateMachine.failed(cause); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index 98108b324758e..8c3ebcbbc9fb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.exception.runtime.MemoryLeakException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import com.google.common.util.concurrent.AbstractFuture; import com.google.common.util.concurrent.ListenableFuture; @@ -174,14 +175,18 @@ private static class FragmentMemoryReservation { private final Map planNodes = new ConcurrentHashMap<>(); private final AtomicLong inputUsedBytes = new AtomicLong(); private final AtomicLong outputUsedBytes = new AtomicLong(); - // Set by local planning before this FI starts reserving exchange memory. - private boolean sharedBudget; - private long inputMaxBytes; - private long outputMaxBytes; + // Registered after local planning; the FI context owns the initialized shared limits. + private FragmentInstanceContext instanceContext; + + private boolean isSharedBudget() { + return instanceContext != null; + } private long getMaxBytesCanReserve(PlanNodeReservation reservation, long legacyLimit) { - return sharedBudget - ? (reservation.side == MemorySide.INPUT ? inputMaxBytes : outputMaxBytes) + return isSharedBudget() + ? (reservation.side == MemorySide.INPUT + ? instanceContext.getInputMaxBytes() + : instanceContext.getOutputMaxBytes()) : legacyLimit; } @@ -264,21 +269,17 @@ public void registerPlanNodeIdToQueryMemoryMap( } } - /** - * Initialize once after the complete FI has been checked for input and pipeline dependencies, - * before any driver starts reserving exchange memory. An unused direction has a zero budget. - */ - public void registerFragmentInstanceSharedBudget( - String queryId, String fragmentInstanceId, long inputMaxBytes, long outputMaxBytes) { + /** Register the FI-owned shared limits after local planning and before drivers start. */ + public void registerFragmentInstanceSharedBudget(FragmentInstanceContext instanceContext) { + String queryId = instanceContext.getId().getQueryId().getId(); + String fragmentInstanceId = instanceContext.getId().getFullId(); synchronized (queryMemoryReservations) { FragmentMemoryReservation fragment = queryMemoryReservations .computeIfAbsent(queryId, x -> new ConcurrentHashMap<>()) .computeIfAbsent(fragmentInstanceId, x -> new FragmentMemoryReservation()); synchronized (fragment) { - fragment.inputMaxBytes = inputMaxBytes; - fragment.outputMaxBytes = outputMaxBytes; - fragment.sharedBudget = true; + fragment.instanceContext = instanceContext; } } } @@ -574,7 +575,7 @@ public boolean tryReserve( fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); maxBytesCanReserve = fragment.getMaxBytesCanReserve(reservation, maxBytesCanReserve); long usedBytes = - fragment.sharedBudget + fragment.isSharedBudget() ? fragment.getUsedBytes(reservation.side).get() : reservation.bytesReserved; // Do not publish speculative charges. Two competing requests must not reject one another diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java index e78240ec0bce1..d7e89cbd6ff32 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java @@ -293,10 +293,11 @@ public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { && pipelineDriverFactories.get(0).getDependencyPipelineIndex() == -1 && (inputExchangeCount == 0 || inputMaxBytes >= maxBlockBytes) && (outputSinkCount == 0 || outputMaxBytes >= maxBlockBytes); + FragmentInstanceContext instanceContext = driverContext.getFragmentInstanceContext(); + instanceContext.initializeExchangeMemoryBudget( + sharedBudget ? inputMaxBytes : 0, sharedBudget ? outputMaxBytes : 0); if (sharedBudget) { - FragmentInstanceId id = getFragmentInstanceId(); - memoryPool.registerFragmentInstanceSharedBudget( - id.getQueryId().getId(), id.getFullId(), inputMaxBytes, outputMaxBytes); + memoryPool.registerFragmentInstanceSharedBudget(instanceContext); } legacySinkLimits.forEach( (sink, legacyLimit) -> diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index 136e317e665bb..1744216249f80 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -19,8 +19,13 @@ package org.apache.iotdb.db.queryengine.execution.memory; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.commons.memory.MemoryManager; import org.apache.iotdb.db.exception.runtime.MemoryLeakException; +import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; import com.google.common.util.concurrent.ListenableFuture; @@ -61,6 +66,20 @@ public void before() { QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, MemorySide.INPUT); } + private void registerSharedBudget( + MemoryPool target, String fragmentInstanceId, long inputMaxBytes, long outputMaxBytes) { + FragmentInstanceId id = Mockito.mock(FragmentInstanceId.class); + Mockito.when(id.getQueryId()).thenReturn(new QueryId(QUERY_ID)); + Mockito.when(id.getFullId()).thenReturn(fragmentInstanceId); + FragmentInstanceContext instanceContext = + FragmentInstanceContext.createFragmentInstanceContext( + id, + Mockito.mock(FragmentInstanceStateMachine.class), + Mockito.mock(MemoryReservationManager.class)); + instanceContext.initializeExchangeMemoryBudget(inputMaxBytes, outputMaxBytes); + target.registerFragmentInstanceSharedBudget(instanceContext); + } + @Test public void testDeregisterDoesNotRemoveReregisteredQueryMemory() throws Exception { Field reservationsField = MemoryPool.class.getDeclaredField("queryMemoryReservations"); @@ -529,7 +548,7 @@ public void testReserveWithPriorityShowQueriesWithinAvailable() { @Test public void testSharedInputBudgetAndIndependentOutput() { - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); + registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap( @@ -577,7 +596,7 @@ public void testSideRegistrationCannotChange() { @Test public void testSharedRequestCannotExceedInitializedInputBudget() { - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); + registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); try { pool.reserveWithPriority( QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 257, Long.MAX_VALUE, false); @@ -591,7 +610,7 @@ public void testSharedRequestCannotExceedInitializedInputBudget() { @Test public void testHandleLimitsCannotChangeInitializedSharedBudgets() throws Exception { - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); + registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap( @@ -626,8 +645,11 @@ public void testSingleDirectionKeepsFullBudgetAndSingleBlockLimit() { new MemoryPool("single-direction", new MemoryManager(1024), 513); for (MemorySide side : MemorySide.values()) { String fi = side.name(); - singleDirectionPool.registerFragmentInstanceSharedBudget( - QUERY_ID, fi, side == MemorySide.INPUT ? 513 : 0, side == MemorySide.OUTPUT ? 513 : 0); + registerSharedBudget( + singleDirectionPool, + fi, + side == MemorySide.INPUT ? 513 : 0, + side == MemorySide.OUTPUT ? 513 : 0); singleDirectionPool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, fi, "active", side); singleDirectionPool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, fi, "unused", side == MemorySide.INPUT ? MemorySide.OUTPUT : MemorySide.INPUT); @@ -649,7 +671,7 @@ public void testSingleDirectionKeepsFullBudgetAndSingleBlockLimit() { @Test public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 128, 128); + registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 128, 128); ExecutorService executor = Executors.newFixedThreadPool(16); CountDownLatch start = new CountDownLatch(1); CountDownLatch attempted = new CountDownLatch(16); @@ -697,7 +719,7 @@ public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { @Test public void testReleaseBeforePendingReservationIsPublished() throws Exception { - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); + registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); pool = Mockito.spy(pool); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); @@ -750,7 +772,7 @@ public void testReleaseBeforePendingReservationIsPublished() throws Exception { @Test public void testCancelledInputDoesNotConsumeReleasedSharedBudget() throws Exception { - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); + registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "cancelled", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap( @@ -788,7 +810,7 @@ public void testCancelledInputDoesNotConsumeReleasedSharedBudget() throws Except @Test public void testConcurrentPendingReservationsCompleteAndReleaseAllAccounting() throws Exception { - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 128, 128); + registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 128, 128); int count = 16; ExecutorService executor = Executors.newFixedThreadPool(count); CountDownLatch attempted = new CountDownLatch(count); @@ -849,7 +871,7 @@ public void testDefaultFragmentKeepsIndependentPlanNodeQuotas() { @Test public void testSharedAndLegacyFragmentsDoNotChangeEachOthersPolicy() { - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); + registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "legacy", "a", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "legacy", "b", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, "b", MemorySide.INPUT); @@ -873,7 +895,7 @@ public void testSharedAndLegacyFragmentsDoNotChangeEachOthersPolicy() { @Test public void testConcurrentAdmissionAlwaysGrantsOneRequestWhenOneFits() throws Exception { - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID, 256, 256); + registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); for (int i = 0; i < 3; i++) { String fi = "other" + i; pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, fi, PLAN_NODE_ID, MemorySide.OUTPUT); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java index 7d2e23e9d554e..21dafba0432e6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; import org.apache.iotdb.calc.execution.operator.Operator; import org.apache.iotdb.calc.execution.operator.process.TableMergeSortOperator; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.commons.memory.MemoryManager; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; @@ -39,6 +40,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.source.SourceHandle; import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; @@ -58,6 +60,7 @@ import java.util.Collections; import java.util.Comparator; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; @@ -89,9 +92,11 @@ public void tearDown() { } private LocalExecutionPlanContext newContext() { - FragmentInstanceContext instance = mock(FragmentInstanceContext.class); - when(instance.getId()) - .thenReturn(new FragmentInstanceId(new PlanFragmentId(new QueryId("budget"), 0), "0")); + FragmentInstanceContext instance = + FragmentInstanceContext.createFragmentInstanceContext( + new FragmentInstanceId(new PlanFragmentId(new QueryId("budget"), 0), "0"), + mock(FragmentInstanceStateMachine.class), + mock(MemoryReservationManager.class)); LocalExecutionPlanContext result = new LocalExecutionPlanContext(new TypeProvider(), instance, (DataNodeQueryContext) null); result.addPipelineDriverFactory(mock(Operator.class), result.getDriverContext(), 0); @@ -151,6 +156,15 @@ private void assertAdmissionIsShared(boolean shared, MemorySide side) { } private void assertAdmissionIsShared(boolean shared, MemorySide side, long limit) { + FragmentInstanceContext instance = context.getDriverContext().getFragmentInstanceContext(); + if (shared) { + assertEquals( + limit, + side == MemorySide.INPUT ? instance.getInputMaxBytes() : instance.getOutputMaxBytes()); + } else { + assertEquals(0, instance.getInputMaxBytes()); + assertEquals(0, instance.getOutputMaxBytes()); + } String fi = context.getFragmentInstanceId().getFullId(); pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "a" + side, side); pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "b" + side, side); @@ -204,6 +218,7 @@ public void testInputOnlyCollectUsesFullBudgetEvenForOneFullBlock() { context.setExchangeMemoryLimits( node(PlanNodeType.TABLE_COLLECT_NODE, node(PlanNodeType.TABLE_EXCHANGE_NODE)), pool); verify(source).setMaxBytesCanReserve(1024); + assertEquals(0, context.getDriverContext().getFragmentInstanceContext().getOutputMaxBytes()); assertAdmissionIsShared(true); } @@ -222,6 +237,7 @@ public void testOutputOnlyCollectUsesFullBudgetEvenForOneFullBlock() { node(PlanNodeType.SERIES_SCAN))), pool); verify(sink).setMaxBytesCanReserve(1024); + assertEquals(0, context.getDriverContext().getFragmentInstanceContext().getInputMaxBytes()); assertAdmissionIsShared(true, MemorySide.OUTPUT); } @@ -239,6 +255,7 @@ public void testLocalInputsDoNotSplitOutputBudget() { context.setISink(sink); context.setExchangeMemoryLimits(node(PlanNodeType.IDENTITY_SINK, collect()), pool); verify(sink).setMaxBytesCanReserve(1024); + assertEquals(0, context.getDriverContext().getFragmentInstanceContext().getInputMaxBytes()); assertAdmissionIsShared(true, MemorySide.OUTPUT); } From e66e08b71f2be540f99ed1bf97ea66afdec64c29 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 17:29:36 +0800 Subject: [PATCH 10/29] Simplify FI exchange limits and isolate SQL assertions --- ...2026-10-14-exchange-fanin-budget-design.md | 198 ------------------ .../apache/iotdb/it/env/cluster/EnvUtils.java | 3 +- .../db/it/utils/CollectQueryAssertions.java | 120 +++++++++++ .../apache/iotdb/db/it/utils/TestUtils.java | 46 ++-- .../it/triple/IoTDBShowReceiversTripleIT.java | 84 +++----- .../it/db/it/IoTDBMultiDeviceTableIT.java | 29 ++- .../udf/IoTDBUserDefinedTableFunctionIT.java | 18 +- .../it/insertquery/IoTDBInsertQueryIT.java | 21 +- .../query/recent/IoTDBClusterResultSetIT.java | 91 +++++--- .../it/query/recent/IoTDBCollectFanInIT.java | 12 +- .../it/query/recent/IoTDBCteIT.java | 8 +- .../query/recent/IoTDBTableAggregationIT.java | 5 +- .../it/query/recent/IoTDBTestEnvPortIT.java | 47 ----- .../IoTDBTopKRuntimeFilterScanPathIT.java | 30 ++- .../operator/process/CollectOperator.java | 1 - .../execution/memory/MemoryPool.java | 58 ++--- .../planner/LocalExecutionPlanContext.java | 9 +- .../execution/memory/MemoryPoolTest.java | 92 ++++---- .../LocalExecutionPlanContextBudgetTest.java | 2 + .../commons/client/ClientManagerTest.java | 3 +- 20 files changed, 358 insertions(+), 519 deletions(-) delete mode 100644 docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java delete mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTestEnvPortIT.java diff --git a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md b/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md deleted file mode 100644 index 3eb5cbc906259..0000000000000 --- a/docs/superpowers/specs/2026-10-14-exchange-fanin-budget-design.md +++ /dev/null @@ -1,198 +0,0 @@ - - -# Exchange 无序扇入与输入输出预算 - -基线：Apache master `aec1ad6139cb4dd1450725f53418cf40c26efb4e`。 -依据：[技术设计文档](https://timechor.feishu.cn/docx/EpDFdmS94ovMCWxluEOc3WGDn8d)，修订 861。 - -## 已确认范围 - -实现仅针对无输入依赖 FI 的输入输出共享账目、Collect/MappingCollect 任意就绪消费，以及由无序汇集引起的计划属性修正。 -聚合使用 HashAggregation：只清空实际输入已失去连续性的 preGroupedSymbols；有效分支内的流式聚合保留。 -MergeSort、Join、ASOF、CrossJoin 不修改消费算法。FI 内只要存在需要特定输入推进的算子，就整组使用原有按 plan node 限额；Collect 下方或上方出现这些算子也不例外。 -TableTopK、CountMerge 的任意就绪消费留到后续；有序输入标志仍需匹配实际输入。 - -## 额度与账目 - -共享采用 FI 级显式启用，默认保留原有配额。整个 FI 必须包含 CollectOperator 或 MappingCollectOperator，且所有多输入汇集都能消费任意就绪输入；顺序消费的 IdentitySink/CountMerge、MergeSort、Join、TopK 及未明确支持的多输入算子都禁止共享。检查在 Exchange 处停止，不跨到其他 FI。 - -在生成全部 pipeline 后再决定预算模式。内部 pipeline 队列具有转发依赖，因此存在多个 pipeline 或调度依赖时整组回退原策略。只含 Collect 的某个子树不会绕过 FI 级否决。 - -通过上述检查后，F 为配置的单 FI 交换预算。按照实际由本 FI 计费的方向分配：只有 INPUT 时 INPUT=F；只有 OUTPUT 时 OUTPUT=F；同时存在两侧时 INPUT=F/2、OUTPUT=F-INPUT；未使用的方向预算为 0。方向在完整本地计划生成后确定，远端 SourceHandle 表示本 FI 的 INPUT，fragment sink 表示 OUTPUT；LocalSourceHandle 的队列由生产 FI 计费，不重复占用接收 FI 的 INPUT。仅实际启用的方向须容纳配置的最大 TsBlock，否则回退原配额。未启用共享的 FI 恢复原有 exchangeSumNum 配额计算和按 plan node 的申请上限；不能只恢复限额数值却仍按同侧总量进行准入。 -远端 SourceHandle 计入接收 FI 的 INPUT；SinkChannel 和 SharedTsBlockQueue 计入生产 FI 的 OUTPUT。 -本地共享队列只计费一次，创建先后不改变所有者。 - -MemoryPool 维护 query→FI→planNode 明细，注册时固定 side。 -本地计划生成全部算子和 pipeline 后，一次性检查共享资格并计算两个方向的预算，写入 `FragmentInstanceContext` 的 `inputMaxBytes`、`outputMaxBytes` 字段。`MemorySide` 仅表示方向，不计算预算，也不接受 `hasInput`/`hasOutput` 参数。MemoryPool 的 FI 账目仅保留该上下文的引用，不复制预算字段。共享模式的申请、单块上限检查和等待唤醒统一读取 FI 保存的预算；handle 传入的限额仅用于原有独立配额模式。回退独立配额时上下文的两个共享预算字段为零。 -FI 两侧保留 AtomicLong 统计，plan node 保留字节明细。申请和释放在 FI 的记账锁内更新；全局池采用有界原子申请。申请前先检查适用的同侧总量或 plan node 限额，失败不产生临时计费，也不需要回滚，避免并发请求分别撞到不同限额后全部挂起。 - -释放后在 FI 记账锁外完成等待 Future；保留入队后的重试和每个 Future 的完成/取消同步。稳定快照中的 FI 同侧总量等于同侧明细之和。 -保留现有最高优先级查询的零计费兜底与交换组件的收发、等待、释放机制。 - -## Collect 生命周期 - -AnyChildBlocked 缓存直接 child.isBlocked 的 Future，每个 pending 周期只有一个 listener。 -新轮次重定向现有 listener；发布后检查完成状态，覆盖注册与完成交错。 -回调只记录就绪或错误，在锁外完成选择 Future；所有 child 方法由 Driver 线程调用。 -重复 isBlocked 与输出背压保留选择；取消对外等待不向 child 传播。 - -一次 next 消耗一次选择。null/空块刷新等待，不表示结束。 -hasNext=false 后检查 isFinished，确认结束才关闭 child 并移除记录。 -MappingCollect 按实际 childId 映射。关闭唤醒待完成选择，忽略迟到回调，逐一尝试关闭全部输入。 -保留内存 R 为各存活 child 保留内存之和；峰值为 max(R,max(R-r_i+p_i))；返回大小为 max(b_i)。 - -## 计划属性 - -多路 Collect/Union 打断分支连续性和排序；单路可透传。 -Filter 与符号透传 Project 不恢复属性；有效 Sort/MergeSort 可以重新建立属性。 -在 SINGLE/PARTIAL/FINAL 实际 child 组装后校验聚合，包括 DISTINCT/mask、普通分阶段聚合及 Union 下推。 -Group 在分区连续性失效时改为完整 Sort。复用 master 的 StreamSort 前缀校验和 Window/Fill 修复。 -SortElimination 在各汇集分支独立优化，并阻止跨无序汇集删除恢复排序。 - -## 验证 - -覆盖纯 Collect/MappingCollect 显式启用、混合 FI 全组回退、Exchange 边界、内部 pipeline 否决、单块预算边界，以及共享/原配额 FI 共存； -覆盖共享额度并发、INPUT/OUTPUT 隔离、所有者、奇数 F、超大申请、失败准入无计费、释放取消及稳定账目一致性； -覆盖 100/1000 child、A 阻塞 B 就绪、listener 复用、注册竞争、输出背压、空块、结束、异常取消、关闭和映射； -覆盖文档 SQL 对应的聚合、Group/Window、StreamSort、Union 与 Fill 计划及实际结果。 -运行 Spotless、英文/中文全 reactor test-compile、针对性单测，以及本分支新增或修改的 IT。 - -## 实现验证记录（2026-10-14） - -上述约定范围的实现已完成。单元测试 86 项通过；IoTDBCollectFanInIT 的 6 项测试在单节点和三 DataNode 集群中分别通过。 -集成验证包含普通与 DISTINCT 聚合、Union 分阶段聚合、嵌套窗口、Fill、排序/TopK、Join 依赖、树模型以及树表视图。 - -构建先执行以下干净构建，生成当前分支的节点包并安装本地依赖： - -```sh -mvn -o clean install -DskipTests -pl integration-test,iotdb-core/confignode,iotdb-client/cli -am -P with-integration-tests -``` - -全 reactor 中文编译与英文编译/回归均通过；英文 test 阶段包含完整 test-compile： - -```sh -mvn -o test-compile -DskipTests -P with-zh-locale -mvn -o test -Dtest=CollectOperatorTest,MemoryPoolTest,LocalExecutionPlanContextBudgetTest,CollectInputPropertyTest,SharedTsBlockQueueTest,SourceHandleTest,SinkChannelTest,LocalSourceHandleTest,LocalSinkChannelTest,MPPDataExchangeManagerTest,ParallelizeGroupingTest,TableFunctionSortTest -Dsurefire.failIfNoSpecifiedTests=false -``` - -两个集成环境均只运行本分支新增的测试类： - -```sh -mvn -o verify -DskipUTs -Dit.test=IoTDBCollectFanInIT -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -am -P TableSimpleIT -P with-integration-tests -mvn -o verify -DskipUTs -Dit.test=IoTDBCollectFanInIT -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -am -P TableClusterIT -P with-integration-tests -``` - -使用离线模式是为了避免远端 SNAPSHOT ZIP 下载停滞；节点执行的是当前分支重新打包的代码。 -Spotless、Checkstyle、新增文件许可证及中英文消息键名/占位符一致性检查通过。 - -## 自查与补测（2026-10-14） - -自查复现了 MemoryPool 的等待申请漏唤醒：一次申请额度不足后，旧申请先释放全部额度， -新申请才进入等待队列；释放线程看不到新申请，导致空闲额度下仍无限等待。 -回归测试用 latch 固定这一时序，在修复前稳定失败。 -修复在申请入队后再次检查额度，并与 free/tryCancel 共用该 future 的锁，避免重复计费。 - -新增 8 项单元测试：等待申请发布竞态、取消不占用共享额度、16 路阻塞申请反复获得额度后账目归零； -空 Collect、全部阻塞时异常唤醒、并发就绪后每路恰好消费一次； -Collect 后 TopK 不误用有序输入，以及有效分支保留流式聚合。 -增强关闭异常测试，检查其他输入仍被关闭且后续异常被保留。 -新增一项集成测试，验证全局窗口后按字段倒序的 TopK 跨设备结果及完整窗口计数。 - -补测后的 118 项定向单测通过（14 个测试类，零失败、零跳过）；全 reactor 中英文编译通过。 -在前述英文回归命令中加入 `OrderingPropertyPropagationTest,TopKRuntimeFilterDistributedPlanTest`， -同时验证有效排序优化与 TopK runtime filter 不受影响。 -重新执行 `mvn -o install -DskipTests -pl integration-test,iotdb-core/confignode,iotdb-client/cli -am -P with-integration-tests` -后运行前述两条 verify 命令；IoTDBCollectFanInIT 的 7 项测试在单节点和三 DataNode 集群中分别全部通过。 -Spotless、Checkstyle、许可证与 `git diff --check` 检查通过。 - -## FI 共享范围修正验证（2026-10-15） - -共享改为完整 FI 的显式启用：仅支持 Collect/MappingCollect 的任意就绪汇集； -同一 FI 内的多输入依赖算子、内部 pipeline 队列或调度依赖会使整组回退。 -Exchange 边界之外的上游 FI 不参与本 FI 的判断。回退同时恢复 handle 限额和按 plan node 准入。 - -107 项定向单元测试通过，包括纯 Collect/Union、混合 FI、pipeline 依赖、单块额度边界、 -共享与独立配额共存、并发准入，以及真实 TableMergeSort 配合模拟输入和 MemoryPool 的回归。 -本地双队列转发回归也通过。Spotless、Checkstyle、许可证和 `git diff --check` 检查通过。 - -```sh -mvn -o test -pl iotdb-core/datanode -Dtest=LocalExecutionPlanContextBudgetTest,MemoryPoolTest,SharedTsBlockQueueTest,SourceHandleTest,SinkChannelTest,LocalSourceHandleTest,LocalSinkChannelTest,MPPDataExchangeManagerTest,PipelineBuilderTest,CollectInputPropertyTest -mvn -o verify -DskipUTs -Dit.test=IoTDBCollectFanInIT -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -am -P TableSimpleIT -P with-integration-tests -mvn -o failsafe:integration-test@integration-test failsafe:verify@verify -DskipUTs -Dit.test=IoTDBCollectFanInIT -DfailIfNoTests=false -Dfailsafe.failIfNoSpecifiedTests=false -pl integration-test -P TableClusterIT -P with-integration-tests -``` - -单节点 verify 从当前源码构建依赖和测试节点；集群运行复用同一份节点模板。 -IoTDBCollectFanInIT 的 8 项测试在两种环境中均通过，零失败、零跳过。 - -提交前再次执行完整 reactor 的中英文编译，两者均通过： - -```sh -mvn -o test-compile -DskipTests -P with-zh-locale -mvn -o test-compile -DskipTests -``` - -本次生产代码没有新增未本地化的字符串，且没有修改中英文消息常量。 - -## 单方向预算修正验证（2026-10-15） - -该轮修正取消默认按两侧均分，改为根据本 FI 实际计费方向分配： -单方向使用完整 F，双方向才拆分，未使用方向为 0。本次已将计算统一收回本地计划初始化， -不再保留该轮引入的 `MemorySide.getMaxBytes(F, hasInput, hasOutput)` 接口。 -最大 TsBlock 校验只作用于启用的方向；原有 Collect 资格和依赖回退规则保持不变。 - -新增输入单向、输出单向、B=F、本地输入和本地/远端混合输入用例； -同时验证奇数 F、未使用方向及每个单方向一次申请完整 F 的边界。 -113 项定向单测全部通过。首轮中单块边界测试的默认测试池上限为 512、申请为 513， -将该测试池配置为目标 F=513 后重跑 MemoryPoolTest，38 项全部通过，其余 75 项首轮通过。 - -完整 reactor 中英文 test-compile、Spotless、Checkstyle 和许可证检查通过。 -重新从当前代码打包节点后，IoTDBCollectFanInIT 在 TableSimpleIT 和三 DataNode 的 -TableClusterIT 中各 8 项全部通过；集群复用同一份节点模板，节点包哈希与本次构建一致。 - -## FI 预算字段初始化验证（2026-10-15） - -`FragmentMemoryReservation` 保存 `inputMaxBytes`、`outputMaxBytes`； -完整本地计划生成后按输入 exchange、fragment sink 和依赖检查结果一次性初始化。 -共享申请、单块上限检查与等待重试均读取 FI 字段，独立配额模式仍读取 handle 限额。 -`MemorySide` 只保留方向枚举，移除布尔参数预算接口。 - -115 项定向单测通过，覆盖 handle 参数不能增减共享上限、超额请求不进入等待队列、 -未使用方向预算为零、奇数预算拆分和原有独立配额回退。 -完整 reactor 中英文编译、Spotless、Checkstyle 和许可证检查通过；生产代码没有新增字符串字面量。 - -重新打包当前源码后，`IoTDBCollectFanInIT` 在 TableSimpleIT 和三 DataNode 的 TableClusterIT -中各 8 项全部通过，无失败或跳过。集群复用同一份节点模板，其服务端 jar 哈希与本次构建一致。 - -## FI 预算配置归属修正验证（2026-10-15） - -预算字段移至 `FragmentInstanceContext`，由完整本地计划生成后的初始化流程赋值。 -MemoryPool 的 FI 账目保存上下文引用，通过 getter 读取限额，不再复制预算或保存共享布尔字段。 -该引用随 FI 记账记录注销；预算仍在 driver 启动前初始化。 -独立配额回退时上下文的两个共享预算字段为零，不注册共享预算。 - -123 项定向单测通过，包括真实 FI 上下文的初始化字段断言、申请上限和等待唤醒回归， -以及已有的 FI 上下文生命周期测试。生产代码没有新增字符串字面量。 - -本次单测在前述 `-Dtest` 列表中加入 `FragmentInstanceContextTest`。完整 reactor 的中英文 -`test-compile -DskipTests`、Spotless、Checkstyle、许可证与 `git diff --check` 检查均通过。 -重新打包节点后，`IoTDBCollectFanInIT` 在 TableSimpleIT 和三 DataNode 的 TableClusterIT 中各 -8 项全部通过，无失败或跳过；测试模板与当前服务端 jar 的哈希一致。 diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java index e6c58e251f593..b69e86c12bd42 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java @@ -180,8 +180,7 @@ public static Map listPortOccupationWindows(final List p */ public static Map listPortOccupationUnix(final List ports) throws IOException { - // The last column is the socket state (e.g. LISTEN); the address precedes it. - return listPortOccupation(ports, "lsof -iTCP -sTCP:LISTEN,TIME_WAIT -P -n", 10, 8, 1); + return listPortOccupation(ports, "lsof -iTCP -sTCP:LISTEN,TIME_WAIT -P -n", 10, 9, 1); } private static Pair getClusterNodesNum(final int index) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java new file mode 100644 index 0000000000000..e53ad150ca431 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.utils; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Query; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QueryBody; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QuerySpecification; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.TableSubquery; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.itbase.runtime.ClusterTestResultSet; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** Explicit SQL-aware assertions for queries affected by ready-input Collect consumption. */ +public final class CollectQueryAssertions { + private CollectQueryAssertions() {} + + /** Only queries without final ordering or an arbitrary top-level slice may use this path. */ + public static List getUnorderedResultSets(String sql, ResultSet resultSet) { + org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement statement = + new SqlParser().createStatement(sql, ZoneOffset.UTC, null); + assertTrue("Unordered assertions require a SELECT query: " + sql, statement instanceof Query); + Query query = (Query) statement; + while (true) { + assertFalse( + "Final ORDER BY requires ordered assertions: " + sql, query.getOrderBy().isPresent()); + assertFalse( + "LIMIT requires deterministic result selection: " + sql, query.getLimit().isPresent()); + assertFalse( + "OFFSET requires deterministic result selection: " + sql, query.getOffset().isPresent()); + QueryBody body = query.getQueryBody(); + if (body instanceof QuerySpecification) { + QuerySpecification specification = (QuerySpecification) body; + assertFalse( + "Final ORDER BY requires ordered assertions: " + sql, + specification.getOrderBy().isPresent()); + assertFalse( + "LIMIT requires deterministic result selection: " + sql, + specification.getLimit().isPresent()); + assertFalse( + "OFFSET requires deterministic result selection: " + sql, + specification.getOffset().isPresent()); + } + if (!(body instanceof TableSubquery)) { + break; + } + // Parentheses around the whole query retain its ordering; nested FROM/UNION branches do not. + query = ((TableSubquery) body).getQuery(); + } + return nodeResults(resultSet); + } + + /** EXPLAIN text may contain node-specific details; each node must satisfy the plan assertions. */ + public static List getExplainResultSets(String sql, ResultSet resultSet) { + assertTrue( + "Plan assertions require EXPLAIN: " + sql, + new SqlParser().createStatement(sql, ZoneOffset.UTC, null) instanceof Explain); + return nodeResults(resultSet); + } + + private static List nodeResults(ResultSet resultSet) { + return resultSet instanceof ClusterTestResultSet + ? ClusterTestResultSet.getNodeResultSets(resultSet) + : Collections.singletonList(resultSet); + } + + /** Compare every node independently while retaining duplicate multiplicity. */ + public static void assertUnorderedQueryResult( + String sql, ResultSet resultSet, String expectedHeader, List expectedRows) + throws SQLException { + List sortedExpected = new ArrayList<>(expectedRows); + Collections.sort(sortedExpected); + for (ResultSet nodeResult : getUnorderedResultSets(sql, resultSet)) { + ResultSetMetaData metadata = nodeResult.getMetaData(); + StringBuilder header = new StringBuilder(); + for (int i = 1; i <= metadata.getColumnCount(); i++) { + header.append(metadata.getColumnName(i)).append(","); + } + assertEquals(expectedHeader, header.toString()); + List actualRows = new ArrayList<>(); + while (nodeResult.next()) { + StringBuilder row = new StringBuilder(); + for (int i = 1; i <= metadata.getColumnCount(); i++) { + row.append(nodeResult.getString(i)).append(","); + } + actualRows.add(row.toString()); + } + Collections.sort(actualRows); + assertEquals(sortedExpected, actualRows); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java index 04499fedf367a..79a2f1c984769 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java @@ -65,7 +65,6 @@ import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT; import static org.apache.iotdb.itbase.env.BaseEnv.TREE_SQL_DIALECT; -import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -808,15 +807,6 @@ public static void assertResultSetEqual( final String expectedHeader, final Collection expectedResult, final boolean printResult) { - if (expectedResult instanceof Set) { - List nodeResults = getNodeResultSets(actualResultSet); - if (nodeResults.size() != 1 || nodeResults.get(0) != actualResultSet) { - for (ResultSet nodeResult : nodeResults) { - assertResultSetEqual(nodeResult, expectedHeader, expectedResult, printResult); - } - return; - } - } try { final ResultSetMetaData resultSetMetaData = actualResultSet.getMetaData(); final StringBuilder header = new StringBuilder(); @@ -878,31 +868,27 @@ public static void assertResultSetEqual( String expectedHeader, Set expectedRetSet, Consumer consumer) { - assertResultSetEqual(actualResultSet, expectedHeader, expectedRetSet, false); - } - - /** Checks unordered rows on every node, preserving duplicate multiplicity. */ - public static void assertResultSetEqualUnordered( - ResultSet resultSet, String expectedHeader, List expectedRows) throws SQLException { - List sortedExpected = new ArrayList<>(expectedRows); - Collections.sort(sortedExpected); - for (ResultSet nodeResult : getNodeResultSets(resultSet)) { - ResultSetMetaData metadata = nodeResult.getMetaData(); + try { + ResultSetMetaData resultSetMetaData = actualResultSet.getMetaData(); StringBuilder header = new StringBuilder(); - for (int i = 1; i <= metadata.getColumnCount(); i++) { - header.append(metadata.getColumnName(i)).append(","); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + header.append(resultSetMetaData.getColumnName(i)).append(","); } assertEquals(expectedHeader, header.toString()); - List actualRows = new ArrayList<>(); - while (nodeResult.next()) { - StringBuilder row = new StringBuilder(); - for (int i = 1; i <= metadata.getColumnCount(); i++) { - row.append(nodeResult.getString(i)).append(","); + + Set actualRetSet = new HashSet<>(); + + while (actualResultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + builder.append(actualResultSet.getString(i)).append(","); } - actualRows.add(row.toString()); + actualRetSet.add(builder.toString()); } - Collections.sort(actualRows); - assertEquals(sortedExpected, actualRows); + assertStringSetEqual(expectedRetSet, actualRetSet); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(String.valueOf(e)); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java index fda9935e31918..0712387775b9d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java @@ -44,7 +44,6 @@ import java.sql.Connection; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; @@ -55,8 +54,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; -import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; - @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT3.class}) public class IoTDBShowReceiversTripleIT { @@ -217,44 +214,39 @@ private boolean hasReceiversOnDifferentDataNodes( final String secondPipeName, final int secondReceiverDataNodeId) throws SQLException { + boolean hasFirstPipe = false; + boolean hasSecondPipe = false; + final Set receiverDataNodeIds = new HashSet<>(); try (final Connection connection = receiverEnv.getConnection( SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect); final Statement statement = connection.createStatement(); - final ResultSet clusterResult = statement.executeQuery(sql)) { - for (ResultSet resultSet : getNodeResultSets(clusterResult)) { - boolean hasFirstPipe = false; - boolean hasSecondPipe = false; - final Set receiverDataNodeIds = new HashSet<>(); - while (resultSet.next()) { - final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids"); - if (!"DataNode".equals(getString(resultSet, "ReceiverNodeType", "receiver_node_type")) - || !"thrift".equals(getString(resultSet, "Protocol", "protocol")) - || pipeIds == null) { - continue; - } - final int receiverDataNodeId = getInt(resultSet, "ReceiverNodeId", "receiver_node_id"); - if (receiverDataNodeId == firstReceiverDataNodeId - && pipeIds.contains(firstPipeName + "@") - && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 - && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { - hasFirstPipe = true; - receiverDataNodeIds.add(receiverDataNodeId); - } - if (receiverDataNodeId == secondReceiverDataNodeId - && pipeIds.contains(secondPipeName + "@") - && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 - && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { - hasSecondPipe = true; - receiverDataNodeIds.add(receiverDataNodeId); - } + final ResultSet resultSet = statement.executeQuery(sql)) { + while (resultSet.next()) { + final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids"); + if (!"DataNode".equals(getString(resultSet, "ReceiverNodeType", "receiver_node_type")) + || !"thrift".equals(getString(resultSet, "Protocol", "protocol")) + || pipeIds == null) { + continue; + } + final int receiverDataNodeId = getInt(resultSet, "ReceiverNodeId", "receiver_node_id"); + if (receiverDataNodeId == firstReceiverDataNodeId + && pipeIds.contains(firstPipeName + "@") + && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 + && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { + hasFirstPipe = true; + receiverDataNodeIds.add(receiverDataNodeId); } - if (!hasFirstPipe || !hasSecondPipe || receiverDataNodeIds.size() < 2) { - return false; + if (receiverDataNodeId == secondReceiverDataNodeId + && pipeIds.contains(secondPipeName + "@") + && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 + && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { + hasSecondPipe = true; + receiverDataNodeIds.add(receiverDataNodeId); } } } - return true; + return hasFirstPipe && hasSecondPipe && receiverDataNodeIds.size() >= 2; } private int getDataNodeId(final DataNodeWrapper targetDataNode) throws Exception { @@ -276,28 +268,20 @@ private int getDataNodeId(final DataNodeWrapper targetDataNode) throws Exception private static String getString( final ResultSet resultSet, final String treeColumnName, final String tableColumnName) throws SQLException { - return resultSet.getString(getColumnIndex(resultSet, treeColumnName, tableColumnName)); + try { + return resultSet.getString(treeColumnName); + } catch (final SQLException ignored) { + return resultSet.getString(tableColumnName); + } } private static int getInt( final ResultSet resultSet, final String treeColumnName, final String tableColumnName) throws SQLException { - return resultSet.getInt(getColumnIndex(resultSet, treeColumnName, tableColumnName)); - } - - private static int getColumnIndex( - final ResultSet resultSet, final String treeColumnName, final String tableColumnName) - throws SQLException { - // SHOW RECEIVERS and information_schema.receivers expose different column names. - // Resolve from metadata because JDBC getters do not all use SQLException for unknown names. - final ResultSetMetaData metadata = resultSet.getMetaData(); - for (int column = 1; column <= metadata.getColumnCount(); column++) { - final String name = metadata.getColumnName(column); - if (treeColumnName.equals(name) || tableColumnName.equals(name)) { - return column; - } + try { + return resultSet.getInt(treeColumnName); + } catch (final SQLException ignored) { + return resultSet.getInt(tableColumnName); } - throw new AssertionError( - "Missing receiver column: " + treeColumnName + " / " + tableColumnName); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java index 096c1383c10d6..999f16e4b0806 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java @@ -37,7 +37,6 @@ import java.util.HashMap; import java.util.Map; -import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -265,27 +264,25 @@ public void testMultiDeviceQueryAndDelete() { } private void testSelectAll() { - String selectSql = "select * from t"; + String selectSql = "select * from t ORDER BY tag1, tag2, time"; try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("use \"test\""); - try (ResultSet clusterResult = statement.executeQuery(selectSql)) { - for (ResultSet resultSet : getNodeResultSets(clusterResult)) { - Map lastTimeMap = new HashMap<>(); - int cnt = 0; - while (resultSet.next()) { - String tag = resultSet.getString("tag1") + "_" + resultSet.getString("tag2"); - long before = lastTimeMap.getOrDefault(tag, -1L); - long cur = resultSet.getTimestamp("time").getTime(); - if (cur <= before) { - fail("time order wrong!"); - } - lastTimeMap.put(tag, cur); - cnt++; + Map lastTimeMap = new HashMap<>(); + try (ResultSet resultSet = statement.executeQuery(selectSql)) { + int cnt = 0; + while (resultSet.next()) { + String tag = resultSet.getString("tag1") + "_" + resultSet.getString("tag2"); + long before = lastTimeMap.getOrDefault(tag, -1L); + long cur = resultSet.getTimestamp("time").getTime(); + if (cur <= before) { + fail("time order wrong!"); } - assertEquals(16030, cnt); + lastTimeMap.put(tag, cur); + cnt++; } + assertEquals(16030, cnt); } } catch (Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java index 25967b8caeb83..25c7176ca9fc8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java @@ -39,9 +39,9 @@ import java.util.HashSet; import java.util.Set; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getUnorderedResultSets; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; -import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -204,14 +204,14 @@ public void testLargeResultIsSplitWithoutDataLoss() throws Exception { try (Connection connection = EnvFactory.getEnv().getTableConnection(); Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); - try (ResultSet clusterResult = - statement.executeQuery( - "SELECT * FROM large_result(vehicle, " - + LARGE_RESULT_REPEAT_COUNT - + ", " - + LARGE_RESULT_PAYLOAD_SIZE - + ")")) { - for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + String sql = + "SELECT * FROM large_result(vehicle, " + + LARGE_RESULT_REPEAT_COUNT + + ", " + + LARGE_RESULT_PAYLOAD_SIZE + + ")"; + try (ResultSet clusterResult = statement.executeQuery(sql)) { + for (ResultSet resultSet : getUnorderedResultSets(sql, clusterResult)) { Set returnedRows = new HashSet<>(); while (resultSet.next()) { int repeatIndex = resultSet.getInt("repeat_index"); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java index 0b43dc0e4152a..dc32124cc64d3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java @@ -49,7 +49,6 @@ import java.util.List; import java.util.Locale; -import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -281,17 +280,15 @@ public void testPartialColumn() throws SQLException { statement.execute("DELETE FROM vehicle1"); statement.execute( "INSERT INTO vehicle1(time, s2, manufacturer) SELECT time, s2, manufacturer FROM vehicle0"); - try (ResultSet clusterResult = - statement.executeQuery("SELECT deviceId, manufacturer, s1, s2 FROM vehicle1")) { - for (ResultSet nodeResult : getNodeResultSets(clusterResult)) { - while (nodeResult.next()) { - for (int i = 1; i <= 4; i++) { - if (i % 2 == 1) { - assertNull(nodeResult.getString(i)); - } else { - assertNotNull(nodeResult.getString(i)); - } - } + resultSet = + statement.executeQuery( + "SELECT deviceId, manufacturer, s1, s2 FROM vehicle1 ORDER BY manufacturer, s2"); + while (resultSet.next()) { + for (int i = 1; i <= 4; i++) { + if (i % 2 == 1) { + assertNull(resultSet.getString(i)); + } else { + assertNotNull(resultSet.getString(i)); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java index a29c9793b72d2..332c69477bdad 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java @@ -38,9 +38,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Set; -import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getExplainResultSets; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getUnorderedResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; @@ -49,48 +50,49 @@ public class IoTDBClusterResultSetIT { @Test - public void testUnorderedSetChecksEveryNode() throws SQLException { + public void testUnorderedRowsCheckEveryNode() throws SQLException { try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { - TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,")); + assertUnorderedQueryResult( + "SELECT value FROM t", result, "value,", Arrays.asList("a,", "b,")); } try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "c"})) { - assertThrows( - AssertionError.class, - () -> TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,"))); - } - } - - @Test - public void testUnorderedSetConsumerOverloadChecksEveryNode() throws SQLException { - try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { - TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,"), message -> {}); - } - try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"a"})) { assertThrows( AssertionError.class, () -> - TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,"), message -> {})); + assertUnorderedQueryResult( + "SELECT value FROM t", result, "value,", Arrays.asList("a,", "b,"))); } } @Test public void testUnorderedRowsPreserveDuplicateMultiplicity() throws SQLException { try (ResultSet result = cluster(new String[] {"a", "a", "b"}, new String[] {"b", "a", "a"})) { - TestUtils.assertResultSetEqualUnordered(result, "value,", Arrays.asList("a,", "a,", "b,")); + assertUnorderedQueryResult( + "SELECT value FROM t", result, "value,", Arrays.asList("a,", "a,", "b,")); } - for (String[] rows : new String[][] {{"a", "b"}, {"a", "a", "a", "b"}}) { - try (ResultSet result = cluster(new String[] {"a", "a", "b"}, rows)) { + for (String[] values : new String[][] {{"a", "b"}, {"a", "a", "a", "b"}}) { + try (ResultSet result = cluster(new String[] {"a", "a", "b"}, values)) { assertThrows( AssertionError.class, () -> - TestUtils.assertResultSetEqualUnordered( - result, "value,", Arrays.asList("a,", "a,", "b,"))); + assertUnorderedQueryResult( + "SELECT value FROM t", result, "value,", Arrays.asList("a,", "a,", "b,"))); } } } @Test - public void testOrderedRowsStillRejectDifferentNodeOrder() throws SQLException { + public void testFinalOrderingRejectsUnorderedAssertions() throws SQLException { + for (String sql : + new String[] { + "SELECT value FROM t ORDER BY value", + "(SELECT value FROM t ORDER BY value)", + "SELECT value FROM t UNION ALL SELECT value FROM t ORDER BY value" + }) { + try (ResultSet result = rows("a")) { + assertThrows(AssertionError.class, () -> getUnorderedResultSets(sql, result)); + } + } try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { assertThrows( AssertionError.class, @@ -98,15 +100,50 @@ public void testOrderedRowsStillRejectDifferentNodeOrder() throws SQLException { } } + @Test + public void testNestedOrderingDoesNotOrderFinalResult() throws SQLException { + for (String sql : + new String[] { + "(SELECT value FROM t ORDER BY value LIMIT 1) UNION ALL (SELECT value FROM t ORDER BY value LIMIT 1)", + "SELECT * FROM (SELECT value FROM t ORDER BY value)", + "SELECT row_number() OVER (ORDER BY value) FROM t", + "SELECT 'order by' FROM t /* ORDER BY value */" + }) { + try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { + assertUnorderedQueryResult(sql, result, "value,", Arrays.asList("a,", "b,")); + } + } + } + + @Test + public void testArbitrarySlicesNeedDeterministicSelection() throws SQLException { + for (String sql : + new String[] {"SELECT value FROM t LIMIT 1", "SELECT value FROM t OFFSET 1"}) { + try (ResultSet result = rows("a")) { + assertThrows(AssertionError.class, () -> getUnorderedResultSets(sql, result)); + } + } + } + @Test public void testSingleNodeAndNullValues() throws SQLException { try (ResultSet result = rows("a", null, "a")) { - assertEquals(1, getNodeResultSets(result).size()); - TestUtils.assertResultSetEqualUnordered(result, "value,", Arrays.asList("null,", "a,", "a,")); + assertEquals(1, getUnorderedResultSets("SELECT value FROM t", result).size()); + assertUnorderedQueryResult( + "SELECT value FROM t", result, "value,", Arrays.asList("null,", "a,", "a,")); } try (ResultSet result = cluster(new String[] {"a"})) { - assertEquals(1, getNodeResultSets(result).size()); - TestUtils.assertResultSetEqual(result, "value,", Set.of("a,")); + assertEquals(1, getUnorderedResultSets("SELECT value FROM t", result).size()); + assertUnorderedQueryResult("SELECT value FROM t", result, "value,", Arrays.asList("a,")); + } + } + + @Test + public void testExplainChecksAreExplicit() throws SQLException { + try (ResultSet result = cluster(new String[] {"plan-a"}, new String[] {"plan-b"})) { + assertThrows(AssertionError.class, () -> getExplainResultSets("SELECT value FROM t", result)); + assertEquals( + 2, getExplainResultSets("EXPLAIN SELECT value FROM t ORDER BY value", result).size()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java index f9de8a0664fb4..943a2150d7f6b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java @@ -36,7 +36,7 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -99,11 +99,11 @@ public void testUnorderedUnionPreservesEveryDuplicateOnEachNode() throws Excepti try (Connection connection = EnvFactory.getEnv().getTableConnection(); Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE); - try (ResultSet result = - statement.executeQuery( - "SELECT device, zone, value FROM readings1 UNION ALL " - + "SELECT device, zone, value FROM readings1")) { - assertResultSetEqualUnordered(result, "device,zone,value,", expectedRows); + String sql = + "SELECT device, zone, value FROM readings1 UNION ALL " + + "SELECT device, zone, value FROM readings1"; + try (ResultSet result = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, result, "device,zone,value,", expectedRows); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java index e8089db62f13b..3d2966b1872c4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java @@ -58,9 +58,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getExplainResultSets; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; -import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -363,7 +363,7 @@ public void testNestExplain1() throws SQLException { // explain try (ResultSet clusterResult = statement.executeQuery(sql)) { - for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + for (ResultSet resultSet : getExplainResultSets(sql, clusterResult)) { ResultSetMetaData metaData = resultSet.getMetaData(); assertEquals(metaData.getColumnCount(), 1); assertEquals(metaData.getColumnName(1), "distribution plan"); @@ -393,7 +393,7 @@ public void testNestExplain2() throws SQLException { // explain try (ResultSet clusterResult = statement.executeQuery(sql)) { - for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + for (ResultSet resultSet : getExplainResultSets(sql, clusterResult)) { ResultSetMetaData metaData = resultSet.getMetaData(); assertEquals(metaData.getColumnCount(), 1); assertEquals(metaData.getColumnName(1), "distribution plan"); @@ -423,7 +423,7 @@ public void testExplainJoin() throws SQLException { // explain try (ResultSet clusterResult = statement.executeQuery(sql)) { - for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + for (ResultSet resultSet : getExplainResultSets(sql, clusterResult)) { ResultSetMetaData metaData = resultSet.getMetaData(); assertEquals(metaData.getColumnCount(), 1); assertEquals(metaData.getColumnName(1), "distribution plan"); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index 99188903b99a7..d707cd65bdc51 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -41,10 +41,10 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getExplainResultSets; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; -import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.buildHeaders; import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.repeatTest; @@ -5904,7 +5904,8 @@ private void assertRateQueryUsesIntermediateAggregation(String query) { Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); try (ResultSet clusterResult = statement.executeQuery("EXPLAIN (FORMAT JSON) " + query)) { - for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + for (ResultSet resultSet : + getExplainResultSets("EXPLAIN (FORMAT JSON) " + query, clusterResult)) { Assert.assertTrue(resultSet.next()); String plan = resultSet.getString(1); // The PARTIAL/FINAL split only exists when the scanned data spans multiple DataRegions. diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTestEnvPortIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTestEnvPortIT.java deleted file mode 100644 index dbb9a9d7a74c5..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTestEnvPortIT.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.relational.it.query.recent; - -import org.apache.iotdb.it.env.cluster.EnvUtils; -import org.apache.iotdb.itbase.category.TableClusterIT; -import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; - -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import java.net.InetAddress; -import java.net.ServerSocket; -import java.util.Collections; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) -public class IoTDBTestEnvPortIT { - - @Test - public void testListeningPortIsReportedAsOccupied() throws Exception { - try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { - Map occupiedPorts = - EnvUtils.listPortOccupation(Collections.singletonList(server.getLocalPort())); - assertEquals( - Long.valueOf(ProcessHandle.current().pid()), occupiedPorts.get(server.getLocalPort())); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java index a0a638dcfe0eb..59499aa5db0de 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java @@ -41,14 +41,11 @@ import java.io.IOException; import java.nio.file.Files; import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; @@ -134,23 +131,20 @@ public void orderByTimeAscLimitOnTable() { } @Test - public void unionAllSiblingTopKOrderByTimeLimit() throws SQLException { - // Each branch has an ORDER BY, but UNION ALL does not order the combined result. - String sql = + public void unionAllSiblingTopKOrderByTimeLimit() { + String[] expectedHeader = new String[] {"time", "device_id", "s1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.030Z,d1,111,", "1970-01-01T00:00:00.032Z,d3,333,", + }; + tableResultSetEqualTest( "(SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd1' ORDER BY time DESC LIMIT 1)" + " UNION ALL" - + " (SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd3' ORDER BY time DESC LIMIT 1)"; - try (Connection connection = EnvFactory.getEnv().getTableConnection(); - Statement statement = connection.createStatement()) { - connection.setClientInfo("time_zone", "+00:00"); - statement.execute("USE " + TABLE_DATABASE); - try (ResultSet resultSet = statement.executeQuery(sql)) { - assertResultSetEqualUnordered( - resultSet, - "time,device_id,s1,", - Arrays.asList("1970-01-01T00:00:00.030Z,d1,111,", "1970-01-01T00:00:00.032Z,d3,333,")); - } - } + + " (SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd3' ORDER BY time DESC LIMIT 1)" + + " ORDER BY time, device_id, s1", + expectedHeader, + retArray, + TABLE_DATABASE); } @Test diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java index 2cc5deec59ea2..bd7dec028b137 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java @@ -208,7 +208,6 @@ public long ramBytesUsed() { return INSTANCE_SIZE + anyChildBlocked.ramBytesUsed() + children.stream() - .filter(java.util.Objects::nonNull) .mapToLong(MemoryEstimationHelper::getEstimatedSizeOfAccountableObject) .sum() + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index 8c3ebcbbc9fb4..219372c9bd34b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -25,7 +25,6 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.exception.runtime.MemoryLeakException; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; -import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import com.google.common.util.concurrent.AbstractFuture; import com.google.common.util.concurrent.ListenableFuture; @@ -60,8 +59,8 @@ public static class MemoryReservationFuture extends AbstractFuture { /** * MemoryReservationFuture is created when an exchange handle tries to reserve memory from the - * pool. This is the per-plan-node limit for legacy fragments. Shared fragments always use their - * initialized FI-side limit, including when retrying a pending reservation. + * pool. Local planning gives handles their FI-side limit in shared mode or their independent + * plan-node limit in legacy mode. Retain the supplied limit when retrying this request. */ private final long maxBytesCanReserve; @@ -155,7 +154,6 @@ public long getReservedBytes() { private final String id; private final IMemoryBlock memoryBlock; - private final long maxBytesPerFragmentInstance; private static class PlanNodeReservation { private final MemorySide side; @@ -175,20 +173,8 @@ private static class FragmentMemoryReservation { private final Map planNodes = new ConcurrentHashMap<>(); private final AtomicLong inputUsedBytes = new AtomicLong(); private final AtomicLong outputUsedBytes = new AtomicLong(); - // Registered after local planning; the FI context owns the initialized shared limits. - private FragmentInstanceContext instanceContext; - - private boolean isSharedBudget() { - return instanceContext != null; - } - - private long getMaxBytesCanReserve(PlanNodeReservation reservation, long legacyLimit) { - return isSharedBudget() - ? (reservation.side == MemorySide.INPUT - ? instanceContext.getInputMaxBytes() - : instanceContext.getOutputMaxBytes()) - : legacyLimit; - } + // Enabled by local planning before drivers start; request limits are supplied by handles. + private boolean sharedBudget; private AtomicLong getUsedBytes(MemorySide side) { return side == MemorySide.INPUT ? inputUsedBytes : outputUsedBytes; @@ -217,7 +203,6 @@ public MemoryPool(String id, MemoryManager memoryManager, long maxBytesPerFragme .EXCEPTION_MAX_BYTES_PER_FI_SHOULD_BE_IN_LEFT_PAREN_0_COMMA_MAXBYTES_RIGHT_BRACKET_DOT_MAXB_4D37C457, maxBytesPerFragmentInstance, this.memoryBlock.getTotalMemorySizeInBytes()); - this.maxBytesPerFragmentInstance = maxBytesPerFragmentInstance; } public String getId() { @@ -269,17 +254,15 @@ public void registerPlanNodeIdToQueryMemoryMap( } } - /** Register the FI-owned shared limits after local planning and before drivers start. */ - public void registerFragmentInstanceSharedBudget(FragmentInstanceContext instanceContext) { - String queryId = instanceContext.getId().getQueryId().getId(); - String fragmentInstanceId = instanceContext.getId().getFullId(); + /** Enable shared accounting after the complete FI has been checked by local planning. */ + public void registerFragmentInstanceSharedBudget(String queryId, String fragmentInstanceId) { synchronized (queryMemoryReservations) { FragmentMemoryReservation fragment = queryMemoryReservations .computeIfAbsent(queryId, x -> new ConcurrentHashMap<>()) .computeIfAbsent(fragmentInstanceId, x -> new FragmentMemoryReservation()); synchronized (fragment) { - fragment.instanceContext = instanceContext; + fragment.sharedBudget = true; } } } @@ -364,19 +347,10 @@ public MemoryReservationResult reserveWithPriority( Validate.notNull( planNodeId, DataNodeQueryMessages.EXCEPTION_PLANNODEID_CAN_NOT_BE_NULL_DOT_44027620); Validate.isTrue( - bytesToReserve > 0L && bytesToReserve <= maxBytesPerFragmentInstance, - DataNodeQueryMessages - .EXCEPTION_BYTESTORESERVE_SHOULD_BE_IN_LEFT_PAREN_0_COMMA_MAXBYTESPERFI_RIGHT_BRACKET_DOT_M_0753BB69, - maxBytesPerFragmentInstance, - bytesToReserve); - FragmentMemoryReservation fragment = - getFragmentReservation(queryId, fragmentInstanceId, planNodeId); - synchronized (fragment) { - PlanNodeReservation reservation = - requireReservation( - fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); - maxBytesCanReserve = fragment.getMaxBytesCanReserve(reservation, maxBytesCanReserve); - } + bytesToReserve > 0L, + DataNodeQueryMessages.EXCEPTION_BYTESTORESERVE_SHOULD_BE_GREATER_THAN_ZERO_DOT_56D15DE0); + // A request must fit both the caller's FI/handle limit and the physical global pool. + maxBytesCanReserve = Math.min(maxBytesCanReserve, getMaxBytes()); if (bytesToReserve > maxBytesCanReserve) { LOGGER.warn( DataNodeQueryMessages @@ -430,11 +404,8 @@ public boolean tryReserveForTest( Validate.notNull( planNodeId, DataNodeQueryMessages.EXCEPTION_PLANNODEID_CAN_NOT_BE_NULL_DOT_44027620); Validate.isTrue( - bytesToReserve > 0L && bytesToReserve <= maxBytesPerFragmentInstance, - DataNodeQueryMessages - .EXCEPTION_BYTESTORESERVE_SHOULD_BE_IN_LEFT_PAREN_0_COMMA_MAXBYTESPERFI_RIGHT_BRACKET_DOT_M_0753BB69, - maxBytesPerFragmentInstance, - bytesToReserve); + bytesToReserve > 0L, + DataNodeQueryMessages.EXCEPTION_BYTESTORESERVE_SHOULD_BE_GREATER_THAN_ZERO_DOT_56D15DE0); return tryReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve); } @@ -573,9 +544,8 @@ public boolean tryReserve( PlanNodeReservation reservation = requireReservation( fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); - maxBytesCanReserve = fragment.getMaxBytesCanReserve(reservation, maxBytesCanReserve); long usedBytes = - fragment.isSharedBudget() + fragment.sharedBudget ? fragment.getUsedBytes(reservation.side).get() : reservation.bytesReserved; // Do not publish speculative charges. Two competing requests must not reject one another diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java index d7e89cbd6ff32..f5ecc27fe45a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java @@ -297,12 +297,15 @@ public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { instanceContext.initializeExchangeMemoryBudget( sharedBudget ? inputMaxBytes : 0, sharedBudget ? outputMaxBytes : 0); if (sharedBudget) { - memoryPool.registerFragmentInstanceSharedBudget(instanceContext); + memoryPool.registerFragmentInstanceSharedBudget( + instanceContext.getId().getQueryId().getId(), instanceContext.getId().getFullId()); } legacySinkLimits.forEach( (sink, legacyLimit) -> - sink.setMaxBytesCanReserve(sharedBudget ? outputMaxBytes : legacyLimit)); - long sourceLimit = sharedBudget ? inputMaxBytes : getMaxBytesOneHandleCanReserve(); + sink.setMaxBytesCanReserve( + sharedBudget ? instanceContext.getOutputMaxBytes() : legacyLimit)); + long sourceLimit = + sharedBudget ? instanceContext.getInputMaxBytes() : getMaxBytesOneHandleCanReserve(); exchangeOperatorList.forEach( exchangeOperator -> exchangeOperator.getSourceHandle().setMaxBytesCanReserve(sourceLimit)); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index 1744216249f80..ed19981b400d7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -19,13 +19,8 @@ package org.apache.iotdb.db.queryengine.execution.memory; -import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.commons.memory.MemoryManager; import org.apache.iotdb.db.exception.runtime.MemoryLeakException; -import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; -import org.apache.iotdb.db.queryengine.common.QueryId; -import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; -import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; import com.google.common.util.concurrent.ListenableFuture; @@ -66,20 +61,6 @@ public void before() { QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, MemorySide.INPUT); } - private void registerSharedBudget( - MemoryPool target, String fragmentInstanceId, long inputMaxBytes, long outputMaxBytes) { - FragmentInstanceId id = Mockito.mock(FragmentInstanceId.class); - Mockito.when(id.getQueryId()).thenReturn(new QueryId(QUERY_ID)); - Mockito.when(id.getFullId()).thenReturn(fragmentInstanceId); - FragmentInstanceContext instanceContext = - FragmentInstanceContext.createFragmentInstanceContext( - id, - Mockito.mock(FragmentInstanceStateMachine.class), - Mockito.mock(MemoryReservationManager.class)); - instanceContext.initializeExchangeMemoryBudget(inputMaxBytes, outputMaxBytes); - target.registerFragmentInstanceSharedBudget(instanceContext); - } - @Test public void testDeregisterDoesNotRemoveReregisteredQueryMemory() throws Exception { Field reservationsField = MemoryPool.class.getDeclaredField("queryMemoryReservations"); @@ -548,7 +529,7 @@ public void testReserveWithPriorityShowQueriesWithinAvailable() { @Test public void testSharedInputBudgetAndIndependentOutput() { - registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap( @@ -595,12 +576,33 @@ public void testSideRegistrationCannotChange() { } @Test - public void testSharedRequestCannotExceedInitializedInputBudget() { - registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); + public void testRequestsUseCurrentSuppliedLimitInsteadOfPoolCreationBudget() { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); + Assert.assertTrue( + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 768, 768, false) + .isReserveSuccess()); + Assert.assertEquals(768, pool.getReservedBytes()); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 768); + Assert.assertEquals(0, pool.getReservedBytes()); + } + + @Test + public void testRequestLargerThanGlobalCapacityDoesNotWait() { try { - pool.reserveWithPriority( - QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 257, Long.MAX_VALUE, false); - Assert.fail("request exceeds the initialized input budget"); + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 1025, 2048, false); + Assert.fail("a request larger than the global pool cannot become ready"); + } catch (IllegalArgumentException expected) { + Assert.assertEquals(0, pool.getReservedBytes()); + Assert.assertEquals(0, pool.getMemoryReservationSize()); + } + } + + @Test + public void testSharedRequestCannotExceedSuppliedInputBudget() { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); + try { + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 257, 256, false); + Assert.fail("request exceeds the supplied input budget"); } catch (IllegalArgumentException expected) { Assert.assertEquals(0, pool.getReservedBytes()); Assert.assertEquals(0, pool.getQueryMemoryReservedBytes(QUERY_ID)); @@ -609,27 +611,25 @@ public void testSharedRequestCannotExceedInitializedInputBudget() { } @Test - public void testHandleLimitsCannotChangeInitializedSharedBudgets() throws Exception { - registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); + public void testSharedReservationsAndWakeupsUseSuppliedSideLimits() throws Exception { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "output", MemorySide.OUTPUT); Assert.assertTrue( - pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 200, 1, false) + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 200, 256, false) .isReserveSuccess()); MemoryReservationResult pending = - pool.reserveWithPriority( - QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 100, Long.MAX_VALUE, false); + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 100, 256, false); Assert.assertFalse(pending.getFuture().isDone()); Assert.assertTrue( - pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, "output", 256, 1, false) + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, "output", 256, 256, false) .isReserveSuccess()); - // Waking the input request must still respect the FI's saved limit. + // An OUTPUT release cannot grant an INPUT request whose shared side remains full. pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, "output", 256); Assert.assertFalse(pending.getFuture().isDone()); - Assert.assertFalse( - pool.tryReserveForTest(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 100, Long.MAX_VALUE)); + Assert.assertFalse(pool.tryReserveForTest(QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", 100, 256)); pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 200); pending.getFuture().get(10, TimeUnit.SECONDS); Assert.assertEquals( @@ -645,21 +645,17 @@ public void testSingleDirectionKeepsFullBudgetAndSingleBlockLimit() { new MemoryPool("single-direction", new MemoryManager(1024), 513); for (MemorySide side : MemorySide.values()) { String fi = side.name(); - registerSharedBudget( - singleDirectionPool, - fi, - side == MemorySide.INPUT ? 513 : 0, - side == MemorySide.OUTPUT ? 513 : 0); + singleDirectionPool.registerFragmentInstanceSharedBudget(QUERY_ID, fi); singleDirectionPool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, fi, "active", side); singleDirectionPool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, fi, "unused", side == MemorySide.INPUT ? MemorySide.OUTPUT : MemorySide.INPUT); Assert.assertTrue( singleDirectionPool - .reserveWithPriority(QUERY_ID, fi, "active", 513, 1, false) + .reserveWithPriority(QUERY_ID, fi, "active", 513, 513, false) .isReserveSuccess()); try { - singleDirectionPool.reserveWithPriority(QUERY_ID, fi, "unused", 1, Long.MAX_VALUE, false); - Assert.fail("an unused direction must not acquire a budget from a handle"); + singleDirectionPool.reserveWithPriority(QUERY_ID, fi, "unused", 1, 0, false); + Assert.fail("an unused direction supplies a zero limit"); } catch (IllegalArgumentException expected) { Assert.assertEquals(0, singleDirectionPool.getMemoryReservationSize()); Assert.assertEquals(513, singleDirectionPool.getReservedBytes()); @@ -671,7 +667,7 @@ public void testSingleDirectionKeepsFullBudgetAndSingleBlockLimit() { @Test public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { - registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 128, 128); + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); ExecutorService executor = Executors.newFixedThreadPool(16); CountDownLatch start = new CountDownLatch(1); CountDownLatch attempted = new CountDownLatch(16); @@ -719,7 +715,7 @@ public void testConcurrentPlanNodesShareOneSideCounter() throws Exception { @Test public void testReleaseBeforePendingReservationIsPublished() throws Exception { - registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); pool = Mockito.spy(pool); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "input2", MemorySide.INPUT); @@ -772,7 +768,7 @@ public void testReleaseBeforePendingReservationIsPublished() throws Exception { @Test public void testCancelledInputDoesNotConsumeReleasedSharedBudget() throws Exception { - registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); pool.registerPlanNodeIdToQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, "cancelled", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap( @@ -810,7 +806,7 @@ public void testCancelledInputDoesNotConsumeReleasedSharedBudget() throws Except @Test public void testConcurrentPendingReservationsCompleteAndReleaseAllAccounting() throws Exception { - registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 128, 128); + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); int count = 16; ExecutorService executor = Executors.newFixedThreadPool(count); CountDownLatch attempted = new CountDownLatch(count); @@ -871,7 +867,7 @@ public void testDefaultFragmentKeepsIndependentPlanNodeQuotas() { @Test public void testSharedAndLegacyFragmentsDoNotChangeEachOthersPolicy() { - registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "legacy", "a", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "legacy", "b", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, "b", MemorySide.INPUT); @@ -895,7 +891,7 @@ public void testSharedAndLegacyFragmentsDoNotChangeEachOthersPolicy() { @Test public void testConcurrentAdmissionAlwaysGrantsOneRequestWhenOneFits() throws Exception { - registerSharedBudget(pool, FRAGMENT_INSTANCE_ID, 256, 256); + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); for (int i = 0; i < 3; i++) { String fi = "other" + i; pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, fi, PLAN_NODE_ID, MemorySide.OUTPUT); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java index 21dafba0432e6..ebcc275717f6f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -186,10 +186,12 @@ private void assertAdmissionIsShared(boolean shared, MemorySide side, long limit @Test public void testPureCollectUsesSharedSideBudgets() { ISourceHandle source = addSource(context); + ISourceHandle secondSource = addSource(context); ISinkHandle sink = mock(ISinkHandle.class); context.setISink(sink); context.setExchangeMemoryLimits(node(PlanNodeType.IDENTITY_SINK, collect()), pool); verify(source).setMaxBytesCanReserve(512); + verify(secondSource).setMaxBytesCanReserve(512); verify(sink).setMaxBytesCanReserve(512); assertAdmissionIsShared(true, MemorySide.INPUT, 512); assertAdmissionIsShared(true, MemorySide.OUTPUT, 512); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java index 82fd7cc80e224..be7cf49239a63 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java @@ -55,8 +55,7 @@ public class ClientManagerTest { - private final TEndPoint endPoint = - new TEndPoint("localhost", Integer.getInteger("iotdb.test.client-manager.port", 10730)); + private final TEndPoint endPoint = new TEndPoint("localhost", 10730); private static final int CONNECTION_TIMEOUT = 5_000; From 82e56ad7ce8676f2b7ef54f919bedccef1d9aebb Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 18:44:40 +0800 Subject: [PATCH 11/29] Avoid per-block Collect input scans with a ready queue --- .../operator/process/AnyChildBlocked.java | 180 +++++++++++------- .../operator/process/CollectOperatorTest.java | 163 +++++++++++++++- 2 files changed, 270 insertions(+), 73 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java index c0397bf1ff5a4..c1ef2edb1b619 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java @@ -50,10 +50,13 @@ final class AnyChildBlocked implements Accountable { private final List children; private final Object lock = new Object(); private final ChildFutureListener[] listeners; + private ChildFutureListener readyHead; + private ChildFutureListener readyTail; private WaitRound currentRound; private Throwable firstFailure; private boolean closed; - private int nextChild; + private boolean initialized; + private int childToObserve = -1; AnyChildBlocked(List children) { this.children = children; @@ -62,65 +65,100 @@ final class AnyChildBlocked implements Accountable { ListenableFuture getAnyChild() { WaitRound round; + WaitRound selection; + boolean initialize; + int invalidatedChild; synchronized (lock) { if (currentRound != null) { return currentRound.selected; } round = new WaitRound(); currentRound = round; - for (ChildFutureListener listener : listeners) { - if (listener != null) { - listener.target = round; - } + initialize = !initialized; + initialized = true; + invalidatedChild = childToObserve; + childToObserve = -1; + selection = selectReadyChild(); + } + completeSelection(selection); + + if (initialize) { + // Activate every direct input once, even if an earlier input is already ready. Pending + // listeners enqueue their completion without being retargeted or polled on every block. + for (int childId = 0; childId < children.size(); childId++) { + observeChild(childId); } + } else if (invalidatedChild >= 0) { + // Only advancing an input invalidates its readiness. Requeue it behind inputs that are + // already ready so that a continuously ready input cannot starve its siblings. + observeChild(invalidatedChild); } + return round.selected; + } - // Activate every live direct child, even if an earlier one has already become ready. Only - // invalidated records call child.isBlocked(); a long-pending future keeps its one listener. - for (int offset = 0; offset < children.size(); offset++) { - int childId = (nextChild + offset) % children.size(); - Operator child; - ChildFutureListener listener; - synchronized (lock) { - if (closed) { - break; - } - child = children.get(childId); - listener = listeners[childId]; + private void observeChild(int childId) { + Operator child; + synchronized (lock) { + if (closed) { + return; } - if (child == null) { - continue; + child = children.get(childId); + } + if (child == null) { + return; + } + ListenableFuture observedFuture = child.isBlocked(); + ChildFutureListener listener = new ChildFutureListener(childId, observedFuture); + synchronized (lock) { + if (closed) { + return; } - if (listener == null) { - ListenableFuture observedFuture = child.isBlocked(); - synchronized (lock) { - if (closed) { - break; - } - listener = new ChildFutureListener(childId, observedFuture, round); - listeners[childId] = listener; + listeners[childId] = listener; + } + // Publish the record before registration, including when completion runs inline. Callbacks + // only update readiness; operator methods remain confined to the driver thread. + observedFuture.addListener(listener, directExecutor()); + } + + /** Called under lock; claiming a selection and removing its ready record are atomic. */ + private WaitRound selectReadyChild() { + if (currentRound == null || currentRound.claimed) { + return null; + } + if (closed || children.isEmpty()) { + currentRound.childId = -1; + } else if (firstFailure != null) { + currentRound.failure = firstFailure; + } else { + ChildFutureListener listener; + do { + listener = readyHead; + if (listener == null) { + return null; } - // The target is published before registration. addListener also handles a future that - // finishes between isBlocked and registration, including inline completion. - observedFuture.addListener(listener, directExecutor()); - } else if (listener.observedFuture.isDone()) { - // A completed record may have notified a previous round. Recheck it for this round. - listener.run(); - } + readyHead = listener.nextReady; + listener.nextReady = null; + if (readyHead == null) { + readyTail = null; + } + // Retired records are discarded once, rather than scanning closed input slots per block. + } while (listeners[listener.childId] != listener); + currentRound.childId = listener.childId; } + currentRound.claimed = true; + return currentRound; + } - Throwable failure; - boolean isClosed; - synchronized (lock) { - failure = firstFailure; - isClosed = closed; + private static void completeSelection(WaitRound selection) { + if (selection == null) { + return; } - if (isClosed || children.isEmpty()) { - round.selected.set(-1); - } else if (failure != null) { - round.selected.setException(failure); + // Completing the future can invoke downstream listeners synchronously. Do not hold lock. + if (selection.failure == null) { + selection.selected.set(selection.childId); + } else { + selection.selected.setException(selection.failure); } - return round.selected; } void checkFailure() throws Exception { @@ -142,11 +180,8 @@ void checkFailure() throws Exception { void invalidateChild(int childId) { synchronized (lock) { listeners[childId] = null; - if (currentRound != null) { - currentRound.valid = false; - currentRound = null; - } - nextChild = (childId + 1) % children.size(); + currentRound = null; + childToObserve = childId; } } @@ -160,14 +195,13 @@ void close() { closed = true; round = currentRound; currentRound = null; - if (round != null) { - round.valid = false; - } - for (ChildFutureListener listener : listeners) { - if (listener != null) { - listener.target = null; - } + childToObserve = -1; + while (readyHead != null) { + ChildFutureListener next = readyHead.nextReady; + readyHead.nextReady = null; + readyHead = next; } + readyTail = null; Arrays.fill(listeners, null); } if (round != null) { @@ -193,18 +227,20 @@ public long ramBytesUsed() { private static final class WaitRound { private final SettableFuture selected = SettableFuture.create(); - private boolean valid = true; + private boolean claimed; + private int childId; + private Throwable failure; } private final class ChildFutureListener implements Runnable { private final int childId; private final ListenableFuture observedFuture; - private WaitRound target; + private ChildFutureListener nextReady; + private boolean completed; - private ChildFutureListener(int childId, ListenableFuture observedFuture, WaitRound target) { + private ChildFutureListener(int childId, ListenableFuture observedFuture) { this.childId = childId; this.observedFuture = observedFuture; - this.target = target; } @Override @@ -217,26 +253,26 @@ public void run() { } catch (CancellationException e) { failure = e; } - WaitRound round; + WaitRound selection; synchronized (lock) { - if (closed || listeners[childId] != this) { + if (closed || listeners[childId] != this || completed) { return; } + completed = true; if (failure != null && firstFailure == null) { firstFailure = failure; } - round = target; - if (round == null || !round.valid || round != currentRound) { - return; + if (failure == null) { + if (readyTail == null) { + readyHead = this; + } else { + readyTail.nextReady = this; + } + readyTail = this; } - failure = firstFailure; - } - // Completing a selection can invoke downstream listeners synchronously. Do not hold lock. - if (failure == null) { - round.selected.set(childId); - } else { - round.selected.setException(failure); + selection = selectReadyChild(); } + completeSelection(selection); } } } diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java index b446f9c999b95..6de8128fe70a3 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java @@ -33,6 +33,7 @@ import org.junit.Test; import java.io.IOException; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -47,6 +48,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; @@ -120,7 +122,98 @@ public void testThousandChildrenAndOneListenerPerPendingPeriod() throws Exceptio } @Test - public void testPendingListenerRetargetedAndStaleCompletionIgnored() throws Exception { + public void testAllReadyChildrenHaveLinearSelectionCost() throws Exception { + int childCount = 1000; + int rounds = 5; + List children = new ArrayList<>(); + for (int i = 0; i < childCount; i++) { + Child child = new Child(block(i)); + child.ready(); + children.add(child); + } + CountingList countedChildren = new CountingList(children); + CollectOperator collect = new CollectOperator(new TestContext(), countedChildren); + try { + for (int round = 0; round < rounds; round++) { + assertTrue(collect.isBlocked().isDone()); + // A completed future may notify more than once. It must occupy only one ready slot. + for (Child child : children) { + child.wait.listener.run(); + child.wait.listener.run(); + } + Set consumed = new HashSet<>(); + for (int i = 0; i < childCount; i++) { + assertTrue(collect.isBlocked().isDone()); + TsBlock result = collect.next(); + assertTrue( + "every ready child must be selected once per round", + consumed.add(result.getColumn(0).getInt(0))); + } + assertEquals(childCount, consumed.size()); + } + for (Child child : children) { + assertEquals(rounds, child.advances); + } + long probes = countedChildren.getCalls + countFutureProbes(children); + long linearLimit = 20L * (childCount + childCount * rounds); + assertTrue( + "selection must use O(children + blocks) probes, observed " + probes, + probes <= linearLimit); + } finally { + collect.close(); + } + } + + @Test + public void testClosedChildrenDoNotIncreaseRemainingChildSelectionCost() throws Exception { + int childCount = 1000; + int remainingBlocks = 3000; + List children = new ArrayList<>(); + for (int i = 0; i < childCount; i++) { + Child child = new Child(block(i)); + child.hasNext = i == childCount - 1; + child.finished = !child.hasNext; + child.ready(); + children.add(child); + } + CountingList countedChildren = new CountingList(children); + CollectOperator collect = new CollectOperator(new TestContext(), countedChildren); + try { + // Retire the first 999 children and consume one block from the remaining child. + for (int i = 0; i < childCount; i++) { + assertTrue(collect.isBlocked().isDone()); + collect.next(); + } + for (int i = 0; i < childCount - 1; i++) { + assertEquals(1, children.get(i).closes); + } + Child remaining = children.get(childCount - 1); + assertEquals(1, remaining.advances); + long initialProbes = countedChildren.getCalls + countFutureProbes(children); + for (int i = 0; i < remainingBlocks; i++) { + assertTrue(collect.isBlocked().isDone()); + assertSame(remaining.output, collect.next()); + } + assertEquals(remainingBlocks + 1, remaining.advances); + long probes = countedChildren.getCalls + countFutureProbes(children) - initialProbes; + assertTrue( + "retired children must not be visited for each remaining block, observed " + probes, + probes <= 20L * remainingBlocks); + } finally { + collect.close(); + } + } + + private static long countFutureProbes(List children) { + long probes = 0; + for (Child child : children) { + probes += child.wait.probes.get(); + } + return probes; + } + + @Test + public void testPendingListenerReusedAndStaleCompletionIgnored() throws Exception { Child a = new Child(block(1)); Child b = new Child(block(2)); CollectOperator collect = collect(a, b); @@ -141,6 +234,29 @@ public void testPendingListenerRetargetedAndStaleCompletionIgnored() throws Exce collect.close(); } + @Test + public void testCompletionBetweenSelectionsIsNotLost() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + b.ready(); + CollectOperator collect = collect(a, b); + try { + assertTrue(collect.isBlocked().isDone()); + assertSame(b.output, collect.next()); + b.reset(); + // No selection is waiting yet: retain this completion until the next driver observation. + a.ready(); + assertTrue(collect.isBlocked().isDone()); + assertSame(a.output, collect.next()); + a.reset(); + assertFalse(collect.isBlocked().isDone()); + assertEquals(1, a.advances); + assertEquals(1, b.advances); + } finally { + collect.close(); + } + } + @Test public void testNullEmptyAndUnfinishedChildRefreshReadiness() throws Exception { Child child = new Child(null); @@ -401,10 +517,36 @@ public void testConcurrentCompletionsConsumeEveryChildExactlyOnce() throws Excep } } + private static final class CountingList extends AbstractList { + private final List delegate; + private long getCalls; + + private CountingList(List children) { + delegate = new ArrayList<>(children); + } + + @Override + public Operator get(int index) { + getCalls++; + return delegate.get(index); + } + + @Override + public Operator set(int index, Operator child) { + return delegate.set(index, child); + } + + @Override + public int size() { + return delegate.size(); + } + } + private static final class CountingFuture extends ForwardingListenableFuture.SimpleForwardingListenableFuture { private final SettableFuture delegate; private final AtomicInteger registrations = new AtomicInteger(); + private final AtomicInteger probes = new AtomicInteger(); private Runnable beforeRegistration; private Runnable listener; @@ -417,6 +559,25 @@ private CountingFuture(SettableFuture delegate) { this.delegate = delegate; } + @Override + public boolean isDone() { + probes.incrementAndGet(); + return super.isDone(); + } + + @Override + public Void get() throws InterruptedException, ExecutionException { + probes.incrementAndGet(); + return super.get(); + } + + @Override + public Void get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + probes.incrementAndGet(); + return super.get(timeout, unit); + } + @Override public void addListener(Runnable listener, Executor executor) { this.listener = listener; From ab4aa3cc650b0f0226e0acedf4003bc43f43f3d0 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 15 Oct 2026 20:40:53 +0800 Subject: [PATCH 12/29] Limit shared exchange budgets and ready Collect to table fragments --- .../it/query/recent/IoTDBCollectFanInIT.java | 21 +++- .../operator/process/CollectOperator.java | 61 ++++++++++- .../operator/process/CollectOperatorTest.java | 51 ++++++++- .../iotdb/db/i18n/DataNodeQueryMessages.java | 1 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 1 + .../DataNodeTableOperatorGenerator.java | 17 +++ .../planner/LocalExecutionPlanContext.java | 44 +++++++- .../plan/planner/LocalExecutionPlanner.java | 38 ++++--- .../plan/planner/OperatorTreeGenerator.java | 7 +- .../SharedExchangeMemoryEligibility.java | 19 ++-- .../TableDistributedPlanGenerator.java | 25 +---- .../LocalExecutionPlanContextBudgetTest.java | 103 +++++++++++++++++- .../planner/CollectInputPropertyTest.java | 19 ---- 13 files changed, 322 insertions(+), 85 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java index 943a2150d7f6b..356908b5affc2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java @@ -34,7 +34,9 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.junit.Assert.assertEquals; @@ -109,7 +111,7 @@ public void testUnorderedUnionPreservesEveryDuplicateOnEachNode() throws Excepti } @Test - public void testSharedExchangePathsForTreeAndTreeToTable() throws Exception { + public void testLegacyTreeAndSharedTreeToTablePaths() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { statement.execute("CREATE DATABASE root.collect_tree"); @@ -124,10 +126,25 @@ public void testSharedExchangePathsForTreeAndTreeToTable() throws Exception { try (ResultSet result = statement.executeQuery("SELECT value FROM root.collect_tree.*.* ALIGN BY DEVICE")) { int rows = 0; + Set seenDevices = new HashSet<>(); + String currentDevice = null; + int deviceRows = 0; while (result.next()) { + String nextDevice = result.getString("Device"); + if (!nextDevice.equals(currentDevice)) { + if (currentDevice != null) { + assertEquals(ROWS, deviceRows); + } + assertTrue("Tree results must retain contiguous devices", seenDevices.add(nextDevice)); + currentDevice = nextDevice; + deviceRows = 0; + } + assertEquals(++deviceRows, result.getLong("Time")); assertEquals(result.getLong("Time"), result.getLong("value")); rows++; } + assertEquals(ROWS, deviceRows); + assertEquals(DEVICES, seenDevices.size()); assertEquals(DEVICES * ROWS, rows); } } @@ -300,7 +317,7 @@ public void testOrderByAfterGlobalWindowAndConstantFill() throws Exception { } @Test - public void testTopKAfterGlobalWindowConsumesUnorderedInput() throws Exception { + public void testExcludedTopKPlanRetainsQueryResults() throws Exception { try (Connection connection = EnvFactory.getEnv().getTableConnection(); Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java index bd7dec028b137..c6ee9577efe65 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java @@ -39,7 +39,9 @@ public class CollectOperator implements ProcessOperator { private final CommonOperatorContext operatorContext; protected final List children; - private final AnyChildBlocked anyChildBlocked; + private AnyChildBlocked anyChildBlocked; + private boolean inited; + protected int currentIndex; private ListenableFuture selectionFuture; private ListenableFuture exposedFuture; private int remainingChildren; @@ -48,17 +50,36 @@ public class CollectOperator implements ProcessOperator { public CollectOperator(CommonOperatorContext operatorContext, List children) { this.operatorContext = operatorContext; this.children = children; - this.anyChildBlocked = new AnyChildBlocked(children); this.remainingChildren = children.size(); } + /** + * Enable only after the complete table FI has passed eligibility checks, before drivers start. + */ + public void enableReadyInputConsumption() { + if (anyChildBlocked == null) { + anyChildBlocked = new AnyChildBlocked(children); + } + } + @Override public boolean hasNext() { - return !closed && remainingChildren > 0; + return anyChildBlocked == null + ? currentIndex < children.size() + : !closed && remainingChildren > 0; } @Override public TsBlock next() throws Exception { + if (anyChildBlocked == null) { + if (children.get(currentIndex).hasNextWithTimer()) { + TsBlock block = children.get(currentIndex).nextWithTimer(); + return block == null ? null : transformBlock(currentIndex, block); + } + closeCurrentChild(currentIndex); + currentIndex++; + return null; + } anyChildBlocked.checkFailure(); if (!hasNext()) { return null; @@ -98,6 +119,11 @@ protected TsBlock transformBlock(int childId, TsBlock block) { } protected void closeCurrentChild(int index) throws Exception { + if (anyChildBlocked == null) { + children.get(index).close(); + children.set(index, null); + return; + } Operator child = children.get(index); if (child == null) { return; @@ -111,6 +137,15 @@ protected void closeCurrentChild(int index) throws Exception { @Override public ListenableFuture isBlocked() { + if (anyChildBlocked == null) { + if (!inited) { + inited = true; + for (Operator child : children) { + child.isBlocked(); + } + } + return currentIndex >= children.size() ? NOT_BLOCKED : children.get(currentIndex).isBlocked(); + } if (!hasNext()) { return NOT_BLOCKED; } @@ -135,6 +170,14 @@ public CommonOperatorContext getOperatorContext() { @Override public void close() throws Exception { + if (anyChildBlocked == null) { + for (int i = currentIndex; i < children.size(); i++) { + if (children.get(i) != null) { + closeCurrentChild(i); + } + } + return; + } if (closed) { return; } @@ -161,6 +204,13 @@ public void close() throws Exception { @Override public long calculateMaxPeekMemory() { + if (anyChildBlocked == null) { + long maxPeekMemory = 0; + for (Operator child : children) { + maxPeekMemory = Math.max(maxPeekMemory, child.calculateMaxPeekMemoryWithCounter()); + } + return maxPeekMemory; + } long retained = calculateRetainedSizeAfterCallingNext(); long peak = retained; for (Operator child : children) { @@ -189,6 +239,9 @@ public long calculateMaxReturnSize() { @Override public long calculateRetainedSizeAfterCallingNext() { + if (anyChildBlocked == null) { + return 0; + } long retained = 0; for (Operator child : children) { if (child != null) { @@ -206,7 +259,7 @@ public List getChildren() { @Override public long ramBytesUsed() { return INSTANCE_SIZE - + anyChildBlocked.ramBytesUsed() + + (anyChildBlocked == null ? 0 : anyChildBlocked.ramBytesUsed()) + children.stream() .mapToLong(MemoryEstimationHelper::getEstimatedSizeOfAccountableObject) .sum() diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java index 6de8128fe70a3..6ab0b3a35f92d 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java @@ -71,7 +71,10 @@ private static TsBlock block(int... values) { } private static CollectOperator collect(Operator... children) { - return new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(children))); + CollectOperator collect = + new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(children))); + collect.enableReadyInputConsumption(); + return collect; } @Test @@ -100,6 +103,7 @@ public void testThousandChildrenAndOneListenerPerPendingPeriod() throws Exceptio children.add(new Child(block(i))); } CollectOperator collect = new CollectOperator(new TestContext(), children); + collect.enableReadyInputConsumption(); Child ready = (Child) children.get(count - 1); for (int round = 0; round < 5; round++) { ListenableFuture wait = collect.isBlocked(); @@ -133,6 +137,7 @@ public void testAllReadyChildrenHaveLinearSelectionCost() throws Exception { } CountingList countedChildren = new CountingList(children); CollectOperator collect = new CollectOperator(new TestContext(), countedChildren); + collect.enableReadyInputConsumption(); try { for (int round = 0; round < rounds; round++) { assertTrue(collect.isBlocked().isDone()); @@ -178,6 +183,7 @@ public void testClosedChildrenDoNotIncreaseRemainingChildSelectionCost() throws } CountingList countedChildren = new CountingList(children); CollectOperator collect = new CollectOperator(new TestContext(), countedChildren); + collect.enableReadyInputConsumption(); try { // Retire the first 999 children and consume one block from the remaining child. for (int i = 0; i < childCount; i++) { @@ -378,6 +384,47 @@ public void testCloseWakesWaitAndClosesEveryChildDespiteException() throws Excep assertEquals(1, b.closes); } + @Test + public void testDefaultCollectRetainsSequentialConsumption() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + b.ready(); + CollectOperator collect = + new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(a, b))); + assertFalse(collect.isBlocked().isDone()); + a.ready(); + assertSame(a.output, collect.next()); + a.hasNext = false; + a.finished = true; + assertNull(collect.next()); + assertSame(b.output, collect.next()); + assertEquals(0, collect.calculateRetainedSizeAfterCallingNext()); + collect.close(); + assertEquals(1, a.closes); + assertEquals(1, b.closes); + } + + @Test + public void testDefaultMappingCollectRetainsSequentialConsumption() throws Exception { + Child a = new Child(block(10, 11)); + Child b = new Child(block(20, 21)); + b.ready(); + List> mappings = new ArrayList<>(); + mappings.add(Collections.singletonList(0)); + mappings.add(Collections.singletonList(1)); + MappingCollectOperator collect = + new MappingCollectOperator( + new TestContext(), new ArrayList<>(Arrays.asList(a, b)), mappings); + assertFalse(collect.isBlocked().isDone()); + a.ready(); + assertEquals(10, collect.next().getColumn(0).getInt(0)); + a.hasNext = false; + a.finished = true; + assertNull(collect.next()); + assertEquals(21, collect.next().getColumn(0).getInt(0)); + collect.close(); + } + @Test public void testMappingUsesActualChildIdentity() throws Exception { Child a = new Child(block(10, 11)); @@ -389,6 +436,7 @@ public void testMappingUsesActualChildIdentity() throws Exception { MappingCollectOperator collect = new MappingCollectOperator( new TestContext(), new ArrayList<>(Arrays.asList(a, b)), mappings); + collect.enableReadyInputConsumption(); assertTrue(collect.isBlocked().isDone()); assertEquals(21, collect.next().getColumn(0).getInt(0)); a.ready(); @@ -473,6 +521,7 @@ public void testConcurrentCompletionsConsumeEveryChildExactlyOnce() throws Excep List children = new ArrayList<>(); for (int i = 0; i < count; i++) children.add(new Child(block(i))); CollectOperator collect = new CollectOperator(new TestContext(), new ArrayList<>(children)); + collect.enableReadyInputConsumption(); ListenableFuture pending = collect.isBlocked(); assertFalse(pending.isDone()); ExecutorService executor = Executors.newFixedThreadPool(count); diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index f28a007654ca8..37bee9557acd9 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4053,4 +4053,5 @@ private DataNodeQueryMessages() {} "Dispatch LOAD piece success: loadId={}, regionId={}, pieceIndex={}, dataSize={}"; public static final String EXCEPTION_PLAN_NODE_ARG_IS_ALREADY_REGISTERED_ON_MEMORY_SIDE_ARG_NOT_ARG_4C1BB625 = "Plan node %s is already registered on memory side %s, not %s"; public static final String EXCEPTION_MEMORY_RESERVATION_IS_NOT_REGISTERED_FOR_QUERY_ARG_FRAGMENT_ARG_PLAN_NODE_ARG_67A4B81F = "Memory reservation is not registered for query %s, fragment %s, plan node %s"; + public static final String LOG_SHARED_EXCHANGE_BUDGET_FOR_FI_ARG_INPUT_ARG_OUTPUT_ARG_98AE1109 = "Shared exchange budget for FI {}: INPUT={}, OUTPUT={}"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 57d2ef87150c8..4c098ecb3157e 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4808,4 +4808,5 @@ private DataNodeQueryMessages() {} "下发 LOAD piece 成功：loadId={}，regionId={}，pieceIndex={}，dataSize={}"; public static final String EXCEPTION_PLAN_NODE_ARG_IS_ALREADY_REGISTERED_ON_MEMORY_SIDE_ARG_NOT_ARG_4C1BB625 = "PlanNode %s 已注册到内存侧 %s，不能注册到 %s"; public static final String EXCEPTION_MEMORY_RESERVATION_IS_NOT_REGISTERED_FOR_QUERY_ARG_FRAGMENT_ARG_PLAN_NODE_ARG_67A4B81F = "查询 %s、Fragment %s、PlanNode %s 尚未注册内存账目"; + public static final String LOG_SHARED_EXCHANGE_BUDGET_FOR_FI_ARG_INPUT_ARG_OUTPUT_ARG_98AE1109 = "FI {} 的共享 exchange 预算：INPUT={}，OUTPUT={}"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 2fae9aa537081..4b4ca52b8558f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -22,6 +22,7 @@ import org.apache.iotdb.calc.execution.filter.TopKRuntimeFilter; import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; import org.apache.iotdb.calc.execution.operator.Operator; +import org.apache.iotdb.calc.execution.operator.process.CollectOperator; import org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator; import org.apache.iotdb.calc.execution.operator.process.LimitOperator; import org.apache.iotdb.calc.execution.operator.process.OffsetOperator; @@ -47,7 +48,9 @@ import org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.commons.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; @@ -301,6 +304,20 @@ protected OperatorContext addOperatorContext( .addOperatorContext(context.getNextOperatorId(), planNodeId, operatorType); } + @Override + public Operator visitCollect(CollectNode node, LocalExecutionPlanContext context) { + CollectOperator collect = (CollectOperator) super.visitCollect(node, context); + context.addCollectOperator(collect); + return collect; + } + + @Override + public Operator visitUnion(UnionNode node, LocalExecutionPlanContext context) { + CollectOperator collect = (CollectOperator) super.visitUnion(node, context); + context.addCollectOperator(collect); + return collect; + } + @Override public Operator visitCteScan(CteScanNode node, LocalExecutionPlanContext context) { OperatorContext operatorContext = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java index f5ecc27fe45a2..9e5cd2e0ea003 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.planner; import org.apache.iotdb.calc.execution.operator.Operator; +import org.apache.iotdb.calc.execution.operator.process.CollectOperator; import org.apache.iotdb.calc.plan.planner.ITableOperatorGeneratorContext; import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.calc.plan.planner.memory.PipelineMemoryEstimator; @@ -51,6 +52,8 @@ import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nullable; @@ -73,6 +76,8 @@ // Attention: We should use thread-safe data structure for members that are shared by all pipelines public class LocalExecutionPlanContext implements ITableOperatorGeneratorContext { + private static final Logger LOGGER = LoggerFactory.getLogger(LocalExecutionPlanContext.class); + // Save operators in this pipeline, a new one will be created when creating another pipeline private final DriverContext driverContext; private final AtomicInteger nextOperatorId; @@ -87,6 +92,7 @@ public class LocalExecutionPlanContext implements ITableOperatorGeneratorContext private int exchangeSumNum = 0; private final long fragmentExchangeMaxBytes; private Map legacySinkLimits = new IdentityHashMap<>(); + private List collectOperators = new ArrayList<>(); private List cachedDataTypes; @@ -129,6 +135,7 @@ public LocalExecutionPlanContext( public LocalExecutionPlanContext(LocalExecutionPlanContext parentContext) { this.fragmentExchangeMaxBytes = parentContext.fragmentExchangeMaxBytes; this.legacySinkLimits = parentContext.legacySinkLimits; + this.collectOperators = parentContext.collectOperators; this.nextOperatorId = parentContext.nextOperatorId; this.typeProvider = parentContext.typeProvider; this.allSensorsMap = parentContext.allSensorsMap; @@ -257,6 +264,7 @@ public void setExchangeSumNum(int exchangeSumNum) { this.exchangeSumNum = exchangeSumNum; } + /** Original per-handle quota for tree planning and independent table FI accounting. */ public long getMaxBytesOneHandleCanReserve() { return exchangeSumNum == 0 ? fragmentExchangeMaxBytes @@ -271,6 +279,23 @@ public void addExchangeOperator(ExchangeOperator exchangeOperator) { this.exchangeOperatorList.add(exchangeOperator); } + public void addCollectOperator(CollectOperator collectOperator) { + collectOperators.add(collectOperator); + } + + public void setMaxBytesOneHandleCanReserve() { + long maxBytesOneHandleCanReserve = getMaxBytesOneHandleCanReserve(); + LOGGER.debug( + DataNodeQueryMessages + .MAXBYTESONEHANDLECANRESERVE_FOR_EXCHANGEOPERATOR_IS_ARG_EXCHANGESUMNUM_IS_ARG, + maxBytesOneHandleCanReserve, + exchangeSumNum); + exchangeOperatorList.forEach( + exchangeOperator -> + exchangeOperator.getSourceHandle().setMaxBytesCanReserve(maxBytesOneHandleCanReserve)); + } + + /** Called after operator generation, before root memory estimation and factory registration. */ public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { // Only remote sources charge this FI's INPUT. Local inputs are owned by the producing FI; // internal pipeline queues cannot enable sharing because of the pipeline check below. @@ -288,9 +313,10 @@ public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { boolean sharedBudget = (inputMaxBytes > 0 || outputMaxBytes > 0) && SharedExchangeMemoryEligibility.isEligible(plan) - // Internal pipeline queues and their downstream sinks must not compete for OUTPUT. - && pipelineDriverFactories.size() == 1 - && pipelineDriverFactories.get(0).getDependencyPipelineIndex() == -1 + // Only the root pipeline may exist. Child factories are registered during generation; + // the root factory is registered after its memory has been estimated below. + && pipelineDriverFactories.isEmpty() + && driverContext.getDependencyDriverIndex() == -1 && (inputExchangeCount == 0 || inputMaxBytes >= maxBlockBytes) && (outputSinkCount == 0 || outputMaxBytes >= maxBlockBytes); FragmentInstanceContext instanceContext = driverContext.getFragmentInstanceContext(); @@ -299,6 +325,18 @@ public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { if (sharedBudget) { memoryPool.registerFragmentInstanceSharedBudget( instanceContext.getId().getQueryId().getId(), instanceContext.getId().getFullId()); + collectOperators.forEach(CollectOperator::enableReadyInputConsumption); + LOGGER.debug( + DataNodeQueryMessages.LOG_SHARED_EXCHANGE_BUDGET_FOR_FI_ARG_INPUT_ARG_OUTPUT_ARG_98AE1109, + instanceContext.getId(), + instanceContext.getInputMaxBytes(), + instanceContext.getOutputMaxBytes()); + } else { + LOGGER.debug( + DataNodeQueryMessages + .MAXBYTESONEHANDLECANRESERVE_FOR_EXCHANGEOPERATOR_IS_ARG_EXCHANGESUMNUM_IS_ARG, + getMaxBytesOneHandleCanReserve(), + exchangeSumNum); } legacySinkLimits.forEach( (sink, legacyLimit) -> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java index 8e020cf736453..c600b4eb9016d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java @@ -116,6 +116,9 @@ public List plan( Operator root = generateOperator(instanceContext, context, plan); + // Select Collect consumption before estimating its retained input memory. + setExchangeMemoryLimits(plan, context); + PipelineMemoryEstimator memoryEstimator = context.constructPipelineMemoryEstimator(root, null, plan, -1); // set the map to null for gc @@ -133,14 +136,6 @@ public List plan( context.getTimePartitions().ifPresent(instanceContext::setTimePartitions); - // Choose the accounting policy only after all FI operators and pipelines are known. - context.setExchangeMemoryLimits( - plan, - MPPDataExchangeService.getInstance() - .getMPPDataExchangeManager() - .getLocalMemoryManager() - .getQueryPool()); - return context.getPipelineDriverFactories(); } @@ -161,6 +156,9 @@ public List plan( Operator root = generateOperator(instanceContext, context, plan); + // Select Collect consumption before estimating its retained input memory. + setExchangeMemoryLimits(plan, context); + PipelineMemoryEstimator memoryEstimator = context.constructPipelineMemoryEstimator(root, null, plan, -1); // set the map to null for gc @@ -171,14 +169,6 @@ public List plan( context.addPipelineDriverFactory(root, context.getDriverContext(), 0); - // Choose the accounting policy only after all FI operators and pipelines are known. - context.setExchangeMemoryLimits( - plan, - MPPDataExchangeService.getInstance() - .getMPPDataExchangeManager() - .getLocalMemoryManager() - .getQueryPool()); - return context.getPipelineDriverFactories(); } @@ -207,6 +197,22 @@ private Operator generateOperator( return root; } + private void setExchangeMemoryLimits(PlanNode plan, LocalExecutionPlanContext context) { + FragmentInstanceContext instanceContext = context.getInstanceContext(); + if (instanceContext.getSessionInfo() == null + || instanceContext.getSessionInfo().getSqlDialect() == TREE) { + context.setMaxBytesOneHandleCanReserve(); + return; + } + // Choose the table FI policy only after all operators and pipelines are known. + context.setExchangeMemoryLimits( + plan, + MPPDataExchangeService.getInstance() + .getMPPDataExchangeManager() + .getLocalMemoryManager() + .getQueryPool()); + } + private long checkMemory( final PipelineMemoryEstimator memoryEstimator, FragmentInstanceContext instanceContext) throws MemoryNotEnoughException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index a46877ac1d5f6..b6a57db5445d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -2577,6 +2577,7 @@ public Operator visitExchange(ExchangeNode node, LocalExecutionPlanContext conte if (!isSameNode) { context.addExchangeSumNum(1); } + sourceHandle.setMaxBytesCanReserve(context.getMaxBytesOneHandleCanReserve()); ExchangeOperator exchangeOperator = new ExchangeOperator(operatorContext, sourceHandle, node.getUpstreamPlanNodeId()); context.addExchangeOperator(exchangeOperator); @@ -2615,7 +2616,8 @@ public Operator visitIdentitySink(IdentitySinkNode node, LocalExecutionPlanConte node.getPlanNodeId().getId(), context.getInstanceContext()); List children = dealWithConsumeChildrenOneByOneNode(node, context); - context.setISink(sinkHandle); + sinkHandle.setMaxBytesCanReserve(context.getMaxBytesOneHandleCanReserve()); + context.getDriverContext().setSink(sinkHandle); return new IdentitySinkOperator(operatorContext, children, downStreamChannelIndex, sinkHandle); } @@ -2656,7 +2658,8 @@ public Operator visitShuffleSink(ShuffleSinkNode node, LocalExecutionPlanContext localInstanceId.toThrift(), node.getPlanNodeId().getId(), context.getInstanceContext()); - context.setISink(sinkHandle); + sinkHandle.setMaxBytesCanReserve(context.getMaxBytesOneHandleCanReserve()); + context.getDriverContext().setSink(sinkHandle); return new ShuffleHelperOperator(operatorContext, children, downStreamChannelIndex, sinkHandle); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java index f5a716a72b8f7..47c31964a3ff9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java @@ -20,8 +20,9 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import java.util.ArrayDeque; import java.util.Deque; @@ -37,20 +38,16 @@ static boolean isEligible(PlanNode root) { nodes.push(root); while (!nodes.isEmpty()) { PlanNode node = nodes.pop(); - if (node instanceof ExchangeNode - || node - instanceof - org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode) { + if (node instanceof ExchangeNode) { // An upstream plan, if still attached, belongs to a different FI. continue; } + // TopK's input ordering assumptions remain unchanged, including its single-child form. + if (node instanceof TopKNode) { + return false; + } // Core relational nodes need not implement the serialization-only PlanNode.getType(). - boolean anyReady = - node instanceof CollectNode - || node instanceof UnionNode - || node - instanceof - org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.CollectNode; + boolean anyReady = node instanceof CollectNode || node instanceof UnionNode; hasCollect |= anyReady; List children = node.getChildren(); // Unknown multi-input operators are not implicitly safe. In particular, sequential sinks diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index d7461431824e2..016ee576584bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -630,27 +630,11 @@ public List visitTopK(TopKNode node, PlanContext context) { if (canTopKEliminated(node.getOrderingScheme(), node.getCount(), childrenNodes.get(0))) { return childrenNodes; } - TopKNode physical = - new TopKNode( - node.getPlanNodeId(), - childrenNodes, - node.getOrderingScheme(), - node.getCount(), - node.getOutputSymbols(), - canSortEliminated( - node.getOrderingScheme(), - nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()))); - physical.setTopKRuntimeFilterSourceId(node.getTopKRuntimeFilterSourceId()); - return Collections.singletonList(physical); + node.setChildren(Collections.singletonList(childrenNodes.get(0))); + return Collections.singletonList(node); } - TopKNode newTopKNode = - new TopKNode( - node.getPlanNodeId(), - node.getOrderingScheme(), - node.getCount(), - node.getOutputSymbols(), - true); + TopKNode newTopKNode = (TopKNode) node.clone(); // root TopK is not a runtime filter producer. newTopKNode.setTopKRuntimeFilterSourceId(null); for (PlanNode child : childrenNodes) { @@ -665,8 +649,7 @@ public List visitTopK(TopKNode node, PlanContext context) { node.getOrderingScheme(), node.getCount(), node.getOutputSymbols(), - canSortEliminated( - node.getOrderingScheme(), nodeOrderingMap.get(child.getPlanNodeId()))); + node.isChildrenDataInOrder()); regionTopK.setTopKRuntimeFilterSourceId(node.getTopKRuntimeFilterSourceId()); newChild = regionTopK; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java index ebcc275717f6f..49623ab9e45ec 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -21,6 +21,7 @@ import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; import org.apache.iotdb.calc.execution.operator.Operator; +import org.apache.iotdb.calc.execution.operator.process.CollectOperator; import org.apache.iotdb.calc.execution.operator.process.TableMergeSortOperator; import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.commons.memory.MemoryManager; @@ -28,6 +29,7 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; @@ -45,8 +47,10 @@ import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import com.google.common.collect.ArrayListMultimap; +import com.google.common.util.concurrent.SettableFuture; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; @@ -63,6 +67,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -99,7 +105,6 @@ private LocalExecutionPlanContext newContext() { mock(MemoryReservationManager.class)); LocalExecutionPlanContext result = new LocalExecutionPlanContext(new TypeProvider(), instance, (DataNodeQueryContext) null); - result.addPipelineDriverFactory(mock(Operator.class), result.getDriverContext(), 0); result.setExchangeSumNum(4); return result; } @@ -117,6 +122,9 @@ private static PlanNode node(PlanNodeType type, PlanNode... children) { case TABLE_COLLECT_NODE: nodeClass = CollectNode.class; break; + case TABLE_TOPK_NODE: + nodeClass = TopKNode.class; + break; case TREE_COLLECT: nodeClass = org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.CollectNode.class; @@ -139,6 +147,21 @@ private static PlanNode collect() { Collections.emptyList()); } + private CollectOperator generateTableCollect(PlanNode plan) { + Operator first = mock(Operator.class); + Operator second = mock(Operator.class); + when(first.isBlocked()).thenReturn(SettableFuture.create()); + doReturn(Operator.NOT_BLOCKED).when(second).isBlocked(); + when(first.calculateMaxPeekMemoryWithCounter()).thenReturn(100L); + when(first.calculateRetainedSizeAfterCallingNext()).thenReturn(30L); + when(second.calculateMaxPeekMemoryWithCounter()).thenReturn(80L); + when(second.calculateRetainedSizeAfterCallingNext()).thenReturn(50L); + doReturn(first).when(plan.getChildren().get(0)).accept(any(), same(context)); + doReturn(second).when(plan.getChildren().get(1)).accept(any(), same(context)); + return (CollectOperator) + plan.accept(new DataNodeTableOperatorGenerator(mock(Metadata.class)), context); + } + private ISourceHandle addSource(LocalExecutionPlanContext target) { ISourceHandle handle = mock(SourceHandle.class); ExchangeOperator operator = mock(ExchangeOperator.class); @@ -296,14 +319,82 @@ public void testMappingCollectUsesSharedBudget() { } @Test - public void testTreeCollectUsesSharedBudget() { + public void testTreePlanningRetainsLegacySourceQuota() { + ISourceHandle source = addSource(context); + context.setMaxBytesOneHandleCanReserve(); + verify(source).setMaxBytesCanReserve(256); + assertFalse( + SharedExchangeMemoryEligibility.isEligible( + node( + PlanNodeType.TREE_COLLECT, + node(PlanNodeType.EXCHANGE), + node(PlanNodeType.EXCHANGE)))); + assertAdmissionIsShared(false); + } + + @Test + public void testTableCollectEnablesReadyConsumptionOnlyForEligibleFragment() { + PlanNode plan = collect(); + CollectOperator collect = generateTableCollect(plan); addSource(context); - context.setExchangeMemoryLimits( - node(PlanNodeType.TREE_COLLECT, node(PlanNodeType.EXCHANGE), node(PlanNodeType.EXCHANGE)), - pool); + context.setExchangeMemoryLimits(plan, pool); + assertTrue(collect.isBlocked().isDone()); + assertEquals(80, collect.calculateRetainedSizeAfterCallingNext()); + assertEquals(150, collect.calculateMaxPeekMemory()); assertAdmissionIsShared(true); } + @Test + public void testTableUnionEnablesReadyConsumptionOnlyForEligibleFragment() { + PlanNode union = + new UnionNode( + new PlanNodeId("union"), + Arrays.asList( + node(PlanNodeType.TABLE_EXCHANGE_NODE), node(PlanNodeType.TABLE_EXCHANGE_NODE)), + ArrayListMultimap.create(), + Collections.emptyList()); + CollectOperator collect = generateTableCollect(union); + addSource(context); + context.setExchangeMemoryLimits(union, pool); + assertTrue(collect.isBlocked().isDone()); + assertEquals(80, collect.calculateRetainedSizeAfterCallingNext()); + assertEquals(150, collect.calculateMaxPeekMemory()); + assertAdmissionIsShared(true); + } + + @Test + public void testSingleChildTopKRetainsLegacyBudgetAndConsumption() { + PlanNode child = collect(); + CollectOperator collect = generateTableCollect(child); + ISourceHandle source = addSource(context); + PlanNode topK = node(PlanNodeType.TABLE_TOPK_NODE, child); + context.setExchangeMemoryLimits(topK, pool); + verify(source).setMaxBytesCanReserve(256); + assertFalse(collect.isBlocked().isDone()); + assertAdmissionIsShared(false); + assertFalse( + SharedExchangeMemoryEligibility.isEligible( + node(PlanNodeType.TABLE_COLLECT_NODE, topK, collect()))); + assertTrue( + SharedExchangeMemoryEligibility.isEligible( + node( + PlanNodeType.TABLE_COLLECT_NODE, + node(PlanNodeType.TABLE_EXCHANGE_NODE, topK), + node(PlanNodeType.TABLE_EXCHANGE_NODE)))); + } + + @Test + public void testMultiplePipelinesRetainSequentialCollectConsumption() { + PlanNode plan = collect(); + CollectOperator collect = generateTableCollect(plan); + LocalExecutionPlanContext child = context.createSubContext(); + child.addPipelineDriverFactory(mock(Operator.class), child.getDriverContext(), 0); + addSource(context); + context.setExchangeMemoryLimits(plan, pool); + assertFalse(collect.isBlocked().isDone()); + assertAdmissionIsShared(false); + } + @Test public void testNestedDependencyDisablesSharingForEntireFragment() { for (PlanNodeType dependency : @@ -393,7 +484,7 @@ public void testInternalPipelineDisablesSharingForEntireFragment() { @Test public void testPipelineSchedulingDependencyDisablesSharing() { - context.getCurrentPipelineDriverFactory().setDependencyPipeline(0); + context.getDriverContext().setDependencyDriverIndex(0); context.setExchangeMemoryLimits(collect(), pool); assertAdmissionIsShared(false); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java index 5bae986bf4762..aa20dd0005bc2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java @@ -25,7 +25,6 @@ import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GroupNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.StreamSortNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.WindowNode; import org.junit.Test; @@ -148,24 +147,6 @@ public void testUnionFinalAggregationDoesNotInheritBranchGrouping() { assertTrue(foundFinalCollect); } - @Test - public void testTopKAboveGlobalWindowDoesNotAssumeOrderedInput() { - PlanNode root = - plan( - "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n FROM table1 " - + "ORDER BY s1 DESC, tag1, tag2, tag3 LIMIT 5"); - List topKs = nodes(root, TopKNode.class); - assertFalse(topKs.isEmpty()); - boolean foundUnorderedInput = false; - for (TopKNode topK : topKs) { - if (!nodes(topK.getChildren().get(0), CollectNode.class).isEmpty()) { - foundUnorderedInput = true; - assertFalse(topK.isChildrenDataInOrder()); - } - } - assertTrue(foundUnorderedInput); - } - @Test public void testOrderedScanBranchesKeepStreamingAggregation() { PlanNode root = From f034960172e521aa67556feb36bb4a353a9dfc0d Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 16 Oct 2026 01:59:20 +0800 Subject: [PATCH 13/29] Address ready Collect review findings and deterministic replica checks Cancel pending reservations before FI deregistration and keep legacy accounting independent per plan node. Guard late Collect mode changes, cover planner entry points, and order pipe verification queries by their complete primary key. --- .../it/dual/tablemodel/TableModelUtils.java | 6 +- .../it/query/recent/IoTDBCollectFanInIT.java | 21 ++ .../apache/iotdb/calc/i18n/CalcMessages.java | 3 + .../apache/iotdb/calc/i18n/CalcMessages.java | 3 + .../operator/process/CollectOperator.java | 10 + .../operator/process/CollectOperatorTest.java | 114 ++++++++ .../execution/memory/MemoryPool.java | 33 ++- .../execution/memory/MemoryPoolTest.java | 156 ++++++++++- .../LocalExecutionPlannerBudgetTest.java | 245 ++++++++++++++++++ 9 files changed, 580 insertions(+), 11 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java index 2aa5b49f331f6..700d552f0c9a0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java @@ -366,7 +366,11 @@ public static String generateHeaderResults() { } public static String getQuerySql(final String table) { - return "select s0,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,time from " + table; + // ClusterTestResultSet compares replicas row by row. All tags plus time form a unique key, + // so require the same row order even when Collect consumes inputs in readiness order. + return "select s0,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,time from " + + table + + " order by s0,s1,s2,s3,time"; } public static String getQueryCountSql(final String table) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java index 356908b5affc2..72e75a9cfb9c3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java @@ -19,10 +19,12 @@ package org.apache.iotdb.relational.it.query.recent; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -34,6 +36,7 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -55,6 +58,7 @@ public static void setUp() throws Exception { EnvFactory.getEnv() .getConfig() .getCommonConfig() + .setDegreeOfParallelism(1) .setMaxTsBlockLineNumber(2) .setSortBufferSize(128 * 1024) .setDataRegionGroupExtensionPolicy("CUSTOM") @@ -110,6 +114,23 @@ public void testUnorderedUnionPreservesEveryDuplicateOnEachNode() throws Excepti } } + @Test + public void testPipeVerificationQueryUsesDeterministicPrimaryKeyOrder() throws Exception { + TableModelUtils.createDataBaseAndTable(EnvFactory.getEnv(), "pipe_readings", DATABASE); + TableModelUtils.insertData(DATABASE, "pipe_readings", 0, DEVICES * ROWS, EnvFactory.getEnv()); + List expected = + new ArrayList<>(TableModelUtils.generateExpectedResults(0, DEVICES * ROWS)); + Collections.sort(expected); + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + try (ResultSet result = + statement.executeQuery(TableModelUtils.getQuerySql("pipe_readings"))) { + TestUtils.assertResultSetEqual(result, TableModelUtils.generateHeaderResults(), expected); + } + } + } + @Test public void testLegacyTreeAndSharedTreeToTablePaths() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java b/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java index 750def8c4a3b0..33976ed731f22 100644 --- a/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java +++ b/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java @@ -582,4 +582,7 @@ private CalcMessages() {} EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_ARG_ARGUMENTS_BUT_GOT_ARG_D6249DD2 = "Aggregate function [%s] requires %d arguments, but got %d"; + public static final String + EXCEPTION_READY_INPUT_CONSUMPTION_MUST_BE_ENABLED_BEFORE_COLLECT_EXECUTION_STARTS_5FAF2DC4 = + "Ready input consumption must be enabled before Collect execution starts"; } diff --git a/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java b/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java index 06ac8d62cbbfd..19a56413d8b0f 100644 --- a/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java +++ b/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java @@ -558,4 +558,7 @@ private CalcMessages() {} public static final String EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_ARG_ARGUMENTS_BUT_GOT_ARG_D6249DD2 = "聚合函数 [%s] 需要 %d 个参数，但实际传入 %d 个"; + public static final String + EXCEPTION_READY_INPUT_CONSUMPTION_MUST_BE_ENABLED_BEFORE_COLLECT_EXECUTION_STARTS_5FAF2DC4 = + "必须在 Collect 开始执行前启用就绪输入消费"; } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java index c6ee9577efe65..e94139ff0cdc2 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java @@ -21,6 +21,7 @@ import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; import org.apache.iotdb.calc.execution.operator.Operator; +import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.commons.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.commons.utils.TestOnly; @@ -30,6 +31,7 @@ import java.util.List; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.util.concurrent.Futures.getDone; import static com.google.common.util.concurrent.Futures.nonCancellationPropagating; @@ -41,6 +43,7 @@ public class CollectOperator implements ProcessOperator { protected final List children; private AnyChildBlocked anyChildBlocked; private boolean inited; + private boolean executionStarted; protected int currentIndex; private ListenableFuture selectionFuture; private ListenableFuture exposedFuture; @@ -58,6 +61,10 @@ public CollectOperator(CommonOperatorContext operatorContext, List chi */ public void enableReadyInputConsumption() { if (anyChildBlocked == null) { + checkState( + !executionStarted, + CalcMessages + .EXCEPTION_READY_INPUT_CONSUMPTION_MUST_BE_ENABLED_BEFORE_COLLECT_EXECUTION_STARTS_5FAF2DC4); anyChildBlocked = new AnyChildBlocked(children); } } @@ -71,6 +78,7 @@ public boolean hasNext() { @Override public TsBlock next() throws Exception { + executionStarted = true; if (anyChildBlocked == null) { if (children.get(currentIndex).hasNextWithTimer()) { TsBlock block = children.get(currentIndex).nextWithTimer(); @@ -137,6 +145,7 @@ protected void closeCurrentChild(int index) throws Exception { @Override public ListenableFuture isBlocked() { + executionStarted = true; if (anyChildBlocked == null) { if (!inited) { inited = true; @@ -170,6 +179,7 @@ public CommonOperatorContext getOperatorContext() { @Override public void close() throws Exception { + executionStarted = true; if (anyChildBlocked == null) { for (int i = currentIndex; i < children.size(); i++) { if (children.get(i) != null) { diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java index 6ab0b3a35f92d..ac05610712aaa 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java @@ -55,6 +55,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -425,6 +426,119 @@ public void testDefaultMappingCollectRetainsSequentialConsumption() throws Excep collect.close(); } + @Test + public void testLegacyInitializesOtherInputsOnceAndFinishesAtLastChild() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + b.ready(); + CollectOperator collect = + new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(a, b))); + for (int i = 0; i < 10; i++) { + assertFalse(collect.isBlocked().isDone()); + } + assertEquals(11, a.waitCalls); + assertEquals(1, b.waitCalls); + assertEquals(0, b.advances); + a.ready(); + assertSame(a.output, collect.next()); + a.hasNext = false; + assertNull(collect.next()); + assertTrue(collect.hasNext()); + assertFalse(collect.isFinished()); + for (int i = 0; i < 10; i++) { + assertTrue(collect.isBlocked().isDone()); + } + assertEquals(11, a.waitCalls); + assertEquals(11, b.waitCalls); + assertSame(b.output, collect.next()); + b.hasNext = false; + assertNull(collect.next()); + assertFalse(collect.hasNext()); + assertTrue(collect.isFinished()); + assertTrue(collect.isBlocked().isDone()); + collect.close(); + assertEquals(1, a.closes); + assertEquals(1, b.closes); + } + + @Test + public void testLegacyMemoryUsesLargestChildPeak() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + a.peak = 100; + a.retained = 30; + a.returnSize = 10; + b.peak = 80; + b.retained = 50; + b.returnSize = 20; + CollectOperator collect = + new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(a, b))); + assertEquals(100, collect.calculateMaxPeekMemory()); + assertEquals(0, collect.calculateRetainedSizeAfterCallingNext()); + assertEquals(20, collect.calculateMaxReturnSize()); + collect.close(); + } + + @Test + public void testCannotEnableReadyConsumptionAfterLegacyInputIsRetired() throws Exception { + Child a = new Child(block(1)); + Child b = new Child(block(2)); + a.hasNext = false; + b.ready(); + CollectOperator collect = + new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(a, b))); + assertNull(collect.next()); + assertThrows(IllegalStateException.class, collect::enableReadyInputConsumption); + assertSame(b.output, collect.next()); + b.hasNext = false; + assertNull(collect.next()); + assertTrue(collect.isFinished()); + } + + @Test + public void testCannotEnableReadyConsumptionAfterLegacyReadinessObservation() throws Exception { + Child child = new Child(block(1)); + CollectOperator collect = + new CollectOperator(new TestContext(), new ArrayList<>(Collections.singletonList(child))); + assertFalse(collect.isBlocked().isDone()); + assertThrows(IllegalStateException.class, collect::enableReadyInputConsumption); + collect.close(); + } + + @Test + public void testCannotEnableReadyConsumptionAfterLegacyNextWithoutIsBlocked() throws Exception { + Child child = new Child(block(1)); + child.ready(); + CollectOperator collect = + new CollectOperator(new TestContext(), new ArrayList<>(Collections.singletonList(child))); + assertSame(child.output, collect.next()); + assertThrows(IllegalStateException.class, collect::enableReadyInputConsumption); + collect.close(); + } + + @Test + public void testCannotEnableReadyConsumptionAfterLegacyClose() throws Exception { + Child child = new Child(block(1)); + CollectOperator collect = + new CollectOperator(new TestContext(), new ArrayList<>(Collections.singletonList(child))); + collect.close(); + assertThrows(IllegalStateException.class, collect::enableReadyInputConsumption); + assertEquals(1, child.closes); + } + + @Test + public void testEnablingReadyConsumptionIsIdempotent() throws Exception { + Child child = new Child(block(1)); + child.ready(); + CollectOperator collect = collect(child); + collect.enableReadyInputConsumption(); + ListenableFuture selection = collect.isBlocked(); + collect.enableReadyInputConsumption(); + assertSame(selection, collect.isBlocked()); + assertSame(child.output, collect.next()); + collect.close(); + } + @Test public void testMappingUsesActualChildIdentity() throws Exception { Child a = new Child(block(10, 11)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index 219372c9bd34b..6a98ca7c4828c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -174,7 +174,7 @@ private static class FragmentMemoryReservation { private final AtomicLong inputUsedBytes = new AtomicLong(); private final AtomicLong outputUsedBytes = new AtomicLong(); // Enabled by local planning before drivers start; request limits are supplied by handles. - private boolean sharedBudget; + private volatile boolean sharedBudget; private AtomicLong getUsedBytes(MemorySide side) { return side == MemorySide.INPUT ? inputUsedBytes : outputUsedBytes; @@ -299,6 +299,18 @@ public void deRegisterFragmentInstanceFromQueryMemoryMap( // forceDeregister == true, after all LocalSourceChannel/LocalSourceHandles are closed. return; } + // Pending requests must not outlive their FI registration. Cancel before removing the + // account so another query's free() cannot retry a request against a missing FI. Do not + // hold the registration lock while notifying cancellation listeners. + Iterator> pending = memoryReservationFutures.iterator(); + while (pending.hasNext()) { + MemoryReservationFuture future = pending.next(); + if (queryId.equals(future.getQueryId()) + && fragmentInstanceId.equals(future.getFragmentInstanceId())) { + tryCancel(future); + pending.remove(); + } + } synchronized (queryMemoryReservations) { queryRelatedMemory.remove(fragmentInstanceId); // Another fragment may have recreated the query entry while this cleanup waited for the @@ -451,10 +463,11 @@ public void free(String queryId, String fragmentInstanceId, String planNodeId, l DataNodeQueryMessages .QUERY_EXCEPTION_RELATEDMEMORYRESERVED_CAN_T_BE_NULL_WHEN_FREEING_MEMORY_C80009F2); } - synchronized (fragment) { - PlanNodeReservation reservation = - requireReservation( - fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); + PlanNodeReservation reservation = + requireReservation( + fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); + Object accountingLock = fragment.sharedBudget ? fragment : reservation; + synchronized (accountingLock) { if (reservation.bytesReserved < bytes) { throw new IllegalArgumentException( DataNodeQueryMessages.FREE_MORE_MEMORY_THAN_HAS_BEEN_RESERVED); @@ -540,10 +553,12 @@ public boolean tryReserve( long maxBytesCanReserve) { FragmentMemoryReservation fragment = getFragmentReservation(queryId, fragmentInstanceId, planNodeId); - synchronized (fragment) { - PlanNodeReservation reservation = - requireReservation( - fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); + PlanNodeReservation reservation = + requireReservation( + fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId, planNodeId); + // Legacy quotas are independent per plan node. Only shared quotas require the FI lock. + Object accountingLock = fragment.sharedBudget ? fragment : reservation; + synchronized (accountingLock) { long usedBytes = fragment.sharedBudget ? fragment.getUsedBytes(reservation.side).get() diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index ed19981b400d7..25957b97897d3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -180,6 +180,85 @@ public void testForceDeregisterReportsMemoryLeak() { Assert.assertEquals(256L, pool.getReservedBytes()); } + @Test + public void testDeregisterCancelsPendingReservationsBeforeUnrelatedFree() { + pool.registerPlanNodeIdToQueryMemoryMap("healthy", "fi", "a", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap("healthy", "fi", "b", MemorySide.INPUT); + Assert.assertTrue(pool.tryReserve("healthy", "fi", "a", 512, 512)); + Assert.assertTrue(pool.tryReserve("healthy", "fi", "b", 512, 512)); + MemoryReservationResult pending = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256, 512, false); + Assert.assertFalse(pending.getFuture().isDone()); + + pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); + + Assert.assertTrue(pending.getFuture().isCancelled()); + pool.free("healthy", "fi", "a", 512); + pool.free("healthy", "fi", "b", 512); + Assert.assertEquals(0, pool.getReservedBytes()); + } + + @Test + public void testForceDeregisterCancelsPendingReservationsEvenWhenReportingLeak() { + Assert.assertTrue(pool.tryReserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512, 512)); + MemoryReservationResult pending = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256, 512, false); + pool.registerPlanNodeIdToQueryMemoryMap("healthy", "fi", "a", MemorySide.INPUT); + Assert.assertTrue(pool.tryReserve("healthy", "fi", "a", 256, 512)); + + Assert.assertThrows( + MemoryLeakException.class, + () -> + pool.deRegisterFragmentInstanceFromQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, true)); + + Assert.assertTrue(pending.getFuture().isCancelled()); + pool.free("healthy", "fi", "a", 256); + // Forced cleanup still reports and retains the original leaked allocation. + Assert.assertEquals(512, pool.getReservedBytes()); + } + + @Test + public void testNonForcedDeregisterKeepsPendingReservationWhileMemoryIsLive() { + Assert.assertTrue(pool.tryReserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512, 512)); + MemoryReservationResult pending = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256, 512, false); + + pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, false); + + Assert.assertFalse(pending.getFuture().isDone()); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512); + Assert.assertTrue(pending.getFuture().isDone()); + Assert.assertFalse(pending.getFuture().isCancelled()); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256); + pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, false); + Assert.assertEquals(0, pool.getReservedBytes()); + } + + @Test + public void testDeregisterDoesNotCancelOtherFragmentPendingReservation() { + pool.registerPlanNodeIdToQueryMemoryMap("healthy", "fi", "a", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap("healthy", "fi", "b", MemorySide.INPUT); + Assert.assertTrue(pool.tryReserve("healthy", "fi", "a", 512, 512)); + Assert.assertTrue(pool.tryReserve("healthy", "fi", "b", 512, 512)); + pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, "other", PLAN_NODE_ID, MemorySide.INPUT); + MemoryReservationResult pending = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256, 512, false); + MemoryReservationResult other = + pool.reserveWithPriority(QUERY_ID, "other", PLAN_NODE_ID, 256, 512, false); + + pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); + + Assert.assertTrue(pending.getFuture().isCancelled()); + Assert.assertFalse(other.getFuture().isDone()); + pool.free("healthy", "fi", "a", 512); + Assert.assertTrue(other.getFuture().isDone()); + Assert.assertFalse(other.getFuture().isCancelled()); + pool.free(QUERY_ID, "other", PLAN_NODE_ID, 256); + pool.free("healthy", "fi", "b", 512); + Assert.assertEquals(0, pool.getReservedBytes()); + } + @Test public void testTryReserve() { @@ -891,7 +970,19 @@ public void testSharedAndLegacyFragmentsDoNotChangeEachOthersPolicy() { @Test public void testConcurrentAdmissionAlwaysGrantsOneRequestWhenOneFits() throws Exception { - pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); + assertConcurrentAdmissionAlwaysGrantsOneRequestWhenOneFits(true); + } + + @Test + public void testLegacyConcurrentAdmissionAlwaysGrantsOneRequestWhenOneFits() throws Exception { + assertConcurrentAdmissionAlwaysGrantsOneRequestWhenOneFits(false); + } + + private void assertConcurrentAdmissionAlwaysGrantsOneRequestWhenOneFits(boolean shared) + throws Exception { + if (shared) { + pool.registerFragmentInstanceSharedBudget(QUERY_ID, FRAGMENT_INSTANCE_ID); + } for (int i = 0; i < 3; i++) { String fi = "other" + i; pool.registerPlanNodeIdToQueryMemoryMap(QUERY_ID, fi, PLAN_NODE_ID, MemorySide.OUTPUT); @@ -938,4 +1029,67 @@ public void testConcurrentAdmissionAlwaysGrantsOneRequestWhenOneFits() throws Ex for (int i = 0; i < 3; i++) pool.free(QUERY_ID, "other" + i, PLAN_NODE_ID, 256); Assert.assertEquals(0, pool.getReservedBytes()); } + + @Test + public void testConcurrentLegacyReservationsKeepIndependentQuotasAndAccurateSideTotals() + throws Exception { + for (boolean samePlanNode : new boolean[] {true, false}) { + MemoryPool legacyPool = new MemoryPool("legacy", new MemoryManager(1024), 512); + ExecutorService executor = Executors.newFixedThreadPool(16); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch attempted = new CountDownLatch(16); + CountDownLatch release = new CountDownLatch(1); + List> tasks = new ArrayList<>(); + try { + for (int i = 0; i < 16; i++) { + String id = samePlanNode ? "same" : "pipeline-" + i; + legacyPool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, id, MemorySide.INPUT); + tasks.add( + executor.submit( + () -> { + start.await(); + boolean reserved = + legacyPool.tryReserve(QUERY_ID, FRAGMENT_INSTANCE_ID, id, 32, 128); + attempted.countDown(); + release.await(); + if (reserved) { + legacyPool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, id, 32); + } + return null; + })); + } + start.countDown(); + Assert.assertTrue(attempted.await(10, TimeUnit.SECONDS)); + long expected = samePlanNode ? 128 : 512; + Assert.assertEquals(expected, legacyPool.getReservedBytes()); + Assert.assertEquals( + expected, + legacyPool.getFragmentMemoryReservedBytes( + QUERY_ID, FRAGMENT_INSTANCE_ID, MemorySide.INPUT)); + if (samePlanNode) { + Assert.assertEquals( + 128, + legacyPool.getPlanNodeMemoryReservedBytes(QUERY_ID, FRAGMENT_INSTANCE_ID, "same")); + } else { + for (int i = 0; i < 16; i++) { + Assert.assertEquals( + 32, + legacyPool.getPlanNodeMemoryReservedBytes( + QUERY_ID, FRAGMENT_INSTANCE_ID, "pipeline-" + i)); + } + } + release.countDown(); + for (Future task : tasks) { + task.get(10, TimeUnit.SECONDS); + } + Assert.assertEquals(0, legacyPool.getReservedBytes()); + Assert.assertEquals(0, legacyPool.getQueryMemoryReservedBytes(QUERY_ID)); + } finally { + start.countDown(); + release.countDown(); + executor.shutdownNow(); + } + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java new file mode 100644 index 0000000000000..ac32c831c43ec --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner; + +import org.apache.iotdb.calc.execution.operator.Operator; +import org.apache.iotdb.calc.execution.operator.process.CollectOperator; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; +import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.queryengine.common.SqlDialect; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; +import org.apache.iotdb.db.queryengine.common.PlanFragmentId; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeService; +import org.apache.iotdb.db.queryengine.execution.exchange.source.SourceHandle; +import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine; +import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; +import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; +import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; +import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; + +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class LocalExecutionPlannerBudgetTest { + private long previousBudget; + private int previousBlockSize; + private boolean previousEstimation; + private int previousDataNodeId; + + @Before + public void setUp() { + previousDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0); + previousBudget = + IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); + previousBlockSize = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); + previousEstimation = + IoTDBDescriptor.getInstance().getMemoryConfig().isEnableQueryMemoryEstimation(); + IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(1024); + TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(64); + IoTDBDescriptor.getInstance().getMemoryConfig().setEnableQueryMemoryEstimation(true); + } + + @After + public void tearDown() { + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(previousDataNodeId); + IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(previousBudget); + TSFileDescriptor.getInstance().getConfig().setMaxTsBlockSizeInBytes(previousBlockSize); + IoTDBDescriptor.getInstance() + .getMemoryConfig() + .setEnableQueryMemoryEstimation(previousEstimation); + } + + @Test + public void testTablePlanEnablesSharingBeforeMemoryEstimationAndRootFactory() throws Exception { + LocalExecutionPlanner planner = LocalExecutionPlanner.getInstance(); + MemoryPool pool = + MPPDataExchangeService.getInstance() + .getMPPDataExchangeManager() + .getLocalMemoryManager() + .getQueryPool(); + FragmentInstanceContext instance = instance("table_planner_budget", SqlDialect.TABLE); + PlanNode first = mock(PlanNode.class); + PlanNode second = mock(PlanNode.class); + SourceHandle firstHandle = mock(SourceHandle.class); + SourceHandle secondHandle = mock(SourceHandle.class); + addInput(first, firstHandle, false, 100, 30); + addInput(second, secondHandle, true, 80, 50); + CollectNode plan = + spy( + new CollectNode( + new PlanNodeId("collect"), Arrays.asList(first, second), Collections.emptyList())); + doReturn(PlanNodeType.TABLE_COLLECT_NODE).when(plan).getType(); + when(first.getChildren()).thenReturn(Collections.emptyList()); + when(second.getChildren()).thenReturn(Collections.emptyList()); + long freeBefore = planner.getFreeMemoryForOperators(); + try { + List factories = + planner.plan(plan, new TypeProvider(), instance, new DataNodeQueryContext(1)); + assertEquals(1, factories.size()); + CollectOperator collect = (CollectOperator) factories.get(0).getOperation(); + // Check the actual reservation, not only the formula after planning has completed. + assertEquals(150 + collect.ramBytesUsed(), freeBefore - planner.getFreeMemoryForOperators()); + assertEquals(1024, instance.getInputMaxBytes()); + assertEquals(0, instance.getOutputMaxBytes()); + assertTrue(collect.isBlocked().isDone()); + verify(firstHandle).setMaxBytesCanReserve(1024); + verify(secondHandle).setMaxBytesCanReserve(1024); + + String query = instance.getId().getQueryId().getId(); + String fi = instance.getId().getFullId(); + pool.registerPlanNodeIdToQueryMemoryMap(query, fi, "first", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap(query, fi, "second", MemorySide.INPUT); + assertTrue( + pool.reserveWithPriority(query, fi, "first", 1024, 1024, false).isReserveSuccess()); + MemoryPool.MemoryReservationResult pending = + pool.reserveWithPriority(query, fi, "second", 64, 1024, false); + assertFalse(pending.getFuture().isDone()); + pool.free(query, fi, "first", 1024); + assertTrue(pending.getFuture().isDone()); + pool.free(query, fi, "second", 64); + collect.close(); + } finally { + pool.deRegisterFragmentInstanceFromQueryMemoryMap( + instance.getId().getQueryId().getId(), instance.getId().getFullId(), true); + long reserved = freeBefore - planner.getFreeMemoryForOperators(); + if (reserved > 0) { + planner.releaseToFreeMemoryForOperators(reserved); + } + } + } + + @Test + public void testNullSessionRetainsTreeGeneratorAndIndependentInputQuotas() throws Exception { + FragmentInstanceContext instance = instance("null_session_planner_budget", null); + PlanNode plan = mock(PlanNode.class); + when(plan.getType()).thenReturn(PlanNodeType.TREE_COLLECT); + when(plan.getPlanNodeId()).thenReturn(new PlanNodeId("collect")); + SourceHandle firstHandle = mock(SourceHandle.class); + SourceHandle secondHandle = mock(SourceHandle.class); + Operator first = input(false, 100, 30); + Operator second = input(true, 80, 50); + doAnswer( + invocation -> { + assertTrue(invocation.getArgument(0) instanceof OperatorTreeGenerator); + LocalExecutionPlanContext context = invocation.getArgument(1); + registerInput(context, firstHandle); + registerInput(context, secondHandle); + CollectOperator collect = + new CollectOperator( + context + .getDriverContext() + .addOperatorContext(0, new PlanNodeId("collect"), "CollectOperator"), + new java.util.ArrayList<>(Arrays.asList(first, second))); + return collect; + }) + .when(plan) + .accept(any(), any()); + LocalExecutionPlanner planner = LocalExecutionPlanner.getInstance(); + long freeBefore = planner.getFreeMemoryForOperators(); + try { + List factories = + planner.plan(plan, new TypeProvider(), instance, new DataNodeQueryContext(1)); + assertEquals(1, factories.size()); + CollectOperator collect = (CollectOperator) factories.get(0).getOperation(); + assertEquals(100 + collect.ramBytesUsed(), freeBefore - planner.getFreeMemoryForOperators()); + assertFalse(collect.isBlocked().isDone()); + assertEquals(0, instance.getInputMaxBytes()); + verify(firstHandle).setMaxBytesCanReserve(512); + verify(secondHandle).setMaxBytesCanReserve(512); + collect.close(); + } finally { + long reserved = freeBefore - planner.getFreeMemoryForOperators(); + if (reserved > 0) { + planner.releaseToFreeMemoryForOperators(reserved); + } + } + } + + private static FragmentInstanceContext instance(String query, SqlDialect dialect) { + FragmentInstanceContext instance = + spy( + FragmentInstanceContext.createFragmentInstanceContext( + new FragmentInstanceId(new PlanFragmentId(new QueryId(query), 0), "0"), + mock(FragmentInstanceStateMachine.class), + mock(MemoryReservationManager.class))); + SessionInfo session = null; + if (dialect != null) { + session = mock(SessionInfo.class); + when(session.getSqlDialect()).thenReturn(dialect); + } + doReturn(session).when(instance).getSessionInfo(); + return instance; + } + + private static void addInput( + PlanNode node, SourceHandle handle, boolean ready, long peak, long retained) { + Operator operator = input(ready, peak, retained); + doAnswer( + invocation -> { + LocalExecutionPlanContext context = invocation.getArgument(1); + registerInput(context, handle); + return operator; + }) + .when(node) + .accept(any(), any()); + } + + private static void registerInput(LocalExecutionPlanContext context, SourceHandle handle) { + ExchangeOperator exchange = mock(ExchangeOperator.class); + when(exchange.getSourceHandle()).thenReturn(handle); + context.addExchangeOperator(exchange); + context.addExchangeSumNum(1); + } + + private static Operator input(boolean ready, long peak, long retained) { + Operator operator = mock(Operator.class); + doReturn(ready ? Operator.NOT_BLOCKED : SettableFuture.create()).when(operator).isBlocked(); + when(operator.calculateMaxPeekMemoryWithCounter()).thenReturn(peak); + when(operator.calculateRetainedSizeAfterCallingNext()).thenReturn(retained); + return operator; + } +} From d5245b3ab8930aa304a8ee665aa343d08f6901ee Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 16 Oct 2026 09:56:33 +0800 Subject: [PATCH 14/29] Check receiver snapshots independently on each node --- .../it/triple/IoTDBShowReceiversTripleIT.java | 103 ++++++++++++------ 1 file changed, 67 insertions(+), 36 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java index 0712387775b9d..5d5313d9f891f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java @@ -54,6 +54,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getUnorderedResultSets; + @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT3.class}) public class IoTDBShowReceiversTripleIT { @@ -214,37 +216,66 @@ private boolean hasReceiversOnDifferentDataNodes( final String secondPipeName, final int secondReceiverDataNodeId) throws SQLException { - boolean hasFirstPipe = false; - boolean hasSecondPipe = false; - final Set receiverDataNodeIds = new HashSet<>(); try (final Connection connection = receiverEnv.getConnection( SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect); final Statement statement = connection.createStatement(); final ResultSet resultSet = statement.executeQuery(sql)) { - while (resultSet.next()) { - final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids"); - if (!"DataNode".equals(getString(resultSet, "ReceiverNodeType", "receiver_node_type")) - || !"thrift".equals(getString(resultSet, "Protocol", "protocol")) - || pipeIds == null) { - continue; - } - final int receiverDataNodeId = getInt(resultSet, "ReceiverNodeId", "receiver_node_id"); - if (receiverDataNodeId == firstReceiverDataNodeId - && pipeIds.contains(firstPipeName + "@") - && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 - && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { - hasFirstPipe = true; - receiverDataNodeIds.add(receiverDataNodeId); - } - if (receiverDataNodeId == secondReceiverDataNodeId - && pipeIds.contains(secondPipeName + "@") - && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 - && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { - hasSecondPipe = true; - receiverDataNodeIds.add(receiverDataNodeId); + // Each node must see both senders, but unordered table queries may return different row + // orders. + for (final ResultSet nodeResult : + BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) + ? getUnorderedResultSets(sql, resultSet) + : Collections.singletonList(resultSet)) { + if (!hasExpectedReceivers( + nodeResult, + sqlDialect, + firstPipeName, + firstReceiverDataNodeId, + secondPipeName, + secondReceiverDataNodeId)) { + return false; } } + return true; + } + } + + private static boolean hasExpectedReceivers( + final ResultSet resultSet, + final String sqlDialect, + final String firstPipeName, + final int firstReceiverDataNodeId, + final String secondPipeName, + final int secondReceiverDataNodeId) + throws SQLException { + boolean hasFirstPipe = false; + boolean hasSecondPipe = false; + final Set receiverDataNodeIds = new HashSet<>(); + while (resultSet.next()) { + final String pipeIds = getString(resultSet, sqlDialect, "PipeIDs", "pipe_ids"); + if (!"DataNode" + .equals(getString(resultSet, sqlDialect, "ReceiverNodeType", "receiver_node_type")) + || !"thrift".equals(getString(resultSet, sqlDialect, "Protocol", "protocol")) + || pipeIds == null) { + continue; + } + final int receiverDataNodeId = + getInt(resultSet, sqlDialect, "ReceiverNodeId", "receiver_node_id"); + if (receiverDataNodeId == firstReceiverDataNodeId + && pipeIds.contains(firstPipeName + "@") + && getInt(resultSet, sqlDialect, "ConnectionCount", "connection_count") >= 1 + && getInt(resultSet, sqlDialect, "PipeCount", "pipe_count") >= 1) { + hasFirstPipe = true; + receiverDataNodeIds.add(receiverDataNodeId); + } + if (receiverDataNodeId == secondReceiverDataNodeId + && pipeIds.contains(secondPipeName + "@") + && getInt(resultSet, sqlDialect, "ConnectionCount", "connection_count") >= 1 + && getInt(resultSet, sqlDialect, "PipeCount", "pipe_count") >= 1) { + hasSecondPipe = true; + receiverDataNodeIds.add(receiverDataNodeId); + } } return hasFirstPipe && hasSecondPipe && receiverDataNodeIds.size() >= 2; } @@ -266,22 +297,22 @@ private int getDataNodeId(final DataNodeWrapper targetDataNode) throws Exception } private static String getString( - final ResultSet resultSet, final String treeColumnName, final String tableColumnName) + final ResultSet resultSet, + final String sqlDialect, + final String treeColumnName, + final String tableColumnName) throws SQLException { - try { - return resultSet.getString(treeColumnName); - } catch (final SQLException ignored) { - return resultSet.getString(tableColumnName); - } + return resultSet.getString( + BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) ? tableColumnName : treeColumnName); } private static int getInt( - final ResultSet resultSet, final String treeColumnName, final String tableColumnName) + final ResultSet resultSet, + final String sqlDialect, + final String treeColumnName, + final String tableColumnName) throws SQLException { - try { - return resultSet.getInt(treeColumnName); - } catch (final SQLException ignored) { - return resultSet.getInt(tableColumnName); - } + return resultSet.getInt( + BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) ? tableColumnName : treeColumnName); } } From 577f9d09a72f2ca992a80eb62cba6d2187fd5045 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 16 Oct 2026 11:42:06 +0800 Subject: [PATCH 15/29] Fix shared FI account keys and indexed exchange waiter cleanup --- .../execution/memory/MemoryPool.java | 58 +++-- .../planner/LocalExecutionPlanContext.java | 3 +- .../execution/memory/MemoryPoolTest.java | 205 ++++++++++++++++++ .../LocalExecutionPlanContextBudgetTest.java | 62 +++++- .../LocalExecutionPlannerBudgetTest.java | 4 +- 5 files changed, 312 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index 6a98ca7c4828c..3664a845d4f82 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -37,9 +37,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -56,6 +55,8 @@ public static class MemoryReservationFuture extends AbstractFuture { private final String fragmentInstanceId; private final String planNodeId; private final long bytesToReserve; + // Assigned once by the pool before publication. Zero denotes an untracked future. + private long reservationId; /** * MemoryReservationFuture is created when an exchange handle tries to reserve memory from the @@ -173,6 +174,8 @@ private static class FragmentMemoryReservation { private final Map planNodes = new ConcurrentHashMap<>(); private final AtomicLong inputUsedBytes = new AtomicLong(); private final AtomicLong outputUsedBytes = new AtomicLong(); + private final Map> pendingReservations = + new ConcurrentHashMap<>(); // Enabled by local planning before drivers start; request limits are supplied by handles. private volatile boolean sharedBudget; @@ -185,8 +188,11 @@ private AtomicLong getUsedBytes(MemorySide side) { private final Map> queryMemoryReservations = new ConcurrentHashMap<>(); - private final Queue> memoryReservationFutures = - new ConcurrentLinkedQueue<>(); + // Sequence keys retain FIFO wakeup order and allow removal without scanning unrelated requests. + // Only blocked reservations use these indexes; successful admission keeps its existing fast path. + private final Map> memoryReservationFutures = + new ConcurrentSkipListMap<>(); + private final AtomicLong nextReservationId = new AtomicLong(); public MemoryPool(String id, MemoryManager memoryManager, long maxBytesPerFragmentInstance) { this.id = Validate.notNull(id, DataNodeQueryMessages.EXCEPTION_ID_CAN_NOT_BE_NULL_DOT_BDD2AD7D); @@ -302,13 +308,10 @@ public void deRegisterFragmentInstanceFromQueryMemoryMap( // Pending requests must not outlive their FI registration. Cancel before removing the // account so another query's free() cannot retry a request against a missing FI. Do not // hold the registration lock while notifying cancellation listeners. - Iterator> pending = memoryReservationFutures.iterator(); - while (pending.hasNext()) { - MemoryReservationFuture future = pending.next(); - if (queryId.equals(future.getQueryId()) - && fragmentInstanceId.equals(future.getFragmentInstanceId())) { + if (fragmentRelatedMemory != null) { + for (MemoryReservationFuture future : + fragmentRelatedMemory.pendingReservations.values()) { tryCancel(future); - pending.remove(); } } synchronized (queryMemoryReservations) { @@ -392,11 +395,21 @@ public MemoryReservationResult reserveWithPriority( MemoryReservationFuture result = MemoryReservationFuture.create( queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve); - memoryReservationFutures.add(result); + FragmentMemoryReservation fragment = + getFragmentReservation(queryId, fragmentInstanceId, planNodeId); + result.reservationId = nextReservationId.incrementAndGet(); + fragment.pendingReservations.put(result.reservationId, result); + memoryReservationFutures.put(result.reservationId, result); // A concurrent free may have missed this request before it was published. Retry after // publication, under the same future lock used by free and tryCancel to avoid charging twice. - if (tryCompleteReservation(result)) { - memoryReservationFutures.remove(result); + try { + if (tryCompleteReservation(result)) { + removePendingReservation(result); + } + } catch (RuntimeException | Error e) { + // A concurrently removed registration must not leave a published request behind. + tryCancel(result); + throw e; } return new MemoryReservationResult(result, false, bytesToReserve); } @@ -440,6 +453,9 @@ public synchronized long tryCancel(ListenableFuture future) { synchronized (future) { // If the future is not a MemoryReservationFuture, it must have been completed. if (future.isDone()) { + if (future instanceof MemoryReservationFuture) { + removePendingReservation((MemoryReservationFuture) future); + } return 0L; } Validate.isTrue( @@ -447,6 +463,7 @@ public synchronized long tryCancel(ListenableFuture future) { DataNodeQueryMessages.EXCEPTION_INVALID_FUTURE_TYPE_14507EF5 + future.getClass().getSimpleName()); future.cancel(true); + removePendingReservation((MemoryReservationFuture) future); } return ((MemoryReservationFuture) future).getBytesToReserve(); } @@ -481,15 +498,26 @@ public void free(String queryId, String fragmentInstanceId, String planNodeId, l if (memoryReservationFutures.isEmpty()) { return; } - Iterator> iterator = memoryReservationFutures.iterator(); + Iterator> iterator = memoryReservationFutures.values().iterator(); while (iterator.hasNext()) { MemoryReservationFuture future = iterator.next(); if (tryCompleteReservation(future)) { - iterator.remove(); + removePendingReservation(future); } } } + private void removePendingReservation(MemoryReservationFuture future) { + memoryReservationFutures.remove(future.reservationId, future); + Map fragments = + queryMemoryReservations.get(future.getQueryId()); + FragmentMemoryReservation fragment = + fragments == null ? null : fragments.get(future.getFragmentInstanceId()); + if (fragment != null) { + fragment.pendingReservations.remove(future.reservationId, future); + } + } + private boolean tryCompleteReservation(MemoryReservationFuture future) { synchronized (future) { if (future.isDone()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java index 9e5cd2e0ea003..055bc4b19f98f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java @@ -324,7 +324,8 @@ public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { sharedBudget ? inputMaxBytes : 0, sharedBudget ? outputMaxBytes : 0); if (sharedBudget) { memoryPool.registerFragmentInstanceSharedBudget( - instanceContext.getId().getQueryId().getId(), instanceContext.getId().getFullId()); + instanceContext.getId().getQueryId().getId(), + instanceContext.getId().getFragmentInstanceId()); collectOperators.forEach(CollectOperator::enableReadyInputConsumption); LOGGER.debug( DataNodeQueryMessages.LOG_SHARED_EXCHANGE_BUDGET_FOR_FI_ARG_INPUT_ARG_OUTPUT_ARG_98AE1109, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index 25957b97897d3..d2b901d7f2355 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -32,8 +32,15 @@ import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.reflect.Field; +import java.util.AbstractCollection; import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -41,6 +48,7 @@ import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.awaitility.Awaitility.await; @@ -259,6 +267,203 @@ public void testDeregisterDoesNotCancelOtherFragmentPendingReservation() { Assert.assertEquals(0, pool.getReservedBytes()); } + @Test + public void testOwnerCancellationAndDeregistrationDoNotScanUnrelatedWaiters() throws Exception { + assertDeregistrationDoesNotScanUnrelatedWaiters(true); + } + + @Test + public void testDeregistrationCancelsOnlyIndexedWaiters() throws Exception { + assertDeregistrationDoesNotScanUnrelatedWaiters(false); + } + + private void assertDeregistrationDoesNotScanUnrelatedWaiters(boolean ownerCancels) + throws Exception { + AtomicInteger visits = countGlobalPendingVisits(); + pool.tryReserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512, 512); + pool.registerPlanNodeIdToQueryMemoryMap("holder", "fi", "p", MemorySide.INPUT); + pool.tryReserve("holder", "fi", "p", 512, 512); + int count = 1024; + List victims = new ArrayList<>(); + List unrelated = new ArrayList<>(); + for (int i = 0; i < count; i++) { + String fi = Integer.toString(i); + pool.registerPlanNodeIdToQueryMemoryMap("victim", fi, "p", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap("unrelated", fi, "p", MemorySide.INPUT); + victims.add(pool.reserveWithPriority("victim", fi, "p", 8, 512, false)); + unrelated.add(pool.reserveWithPriority("unrelated", fi, "p", 8, 512, false)); + } + for (int i = 0; i < count; i++) { + if (ownerCancels) { + Assert.assertEquals(8, pool.tryCancel(victims.get(i).getFuture())); + } + pool.deRegisterFragmentInstanceFromQueryMemoryMap("victim", Integer.toString(i), true); + Assert.assertTrue(victims.get(i).getFuture().isCancelled()); + Assert.assertFalse(unrelated.get(i).getFuture().isDone()); + } + // An operation-count assertion avoids timing noise and detects a global scan for every FI. + Assert.assertEquals(0, visits.get()); + Assert.assertEquals(count, pool.getMemoryReservationSize()); + for (int i = 0; i < count; i++) { + Assert.assertEquals(8, pool.tryCancel(unrelated.get(i).getFuture())); + pool.deRegisterFragmentInstanceFromQueryMemoryMap("unrelated", Integer.toString(i), true); + } + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512); + pool.free("holder", "fi", "p", 512); + pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); + pool.deRegisterFragmentInstanceFromQueryMemoryMap("holder", "fi", true); + Assert.assertEquals(0, pool.getMemoryReservationSize()); + Assert.assertEquals(0, pool.getQueryMemoryReservationSize()); + Assert.assertEquals(0, pool.getReservedBytes()); + } + + @SuppressWarnings("unchecked") + private AtomicInteger countGlobalPendingVisits() throws Exception { + AtomicInteger visits = new AtomicInteger(); + Field field = MemoryPool.class.getDeclaredField("memoryReservationFutures"); + field.setAccessible(true); + Object pending = field.get(pool); + if (pending instanceof Queue) { + // Keep this branch so the regression also runs against the former queue implementation. + Queue> counted = + new ConcurrentLinkedQueue>() { + @Override + public Iterator> iterator() { + return countVisits(super.iterator(), visits); + } + }; + counted.addAll((Queue>) pending); + field.set(pool, counted); + } else { + Map> counted = + new ConcurrentSkipListMap>() { + @Override + public Collection> values() { + Collection> values = super.values(); + return new AbstractCollection>() { + @Override + public Iterator> iterator() { + return countVisits(values.iterator(), visits); + } + + @Override + public int size() { + return values.size(); + } + }; + } + }; + counted.putAll((Map>) pending); + field.set(pool, counted); + } + return visits; + } + + private static Iterator countVisits(Iterator delegate, AtomicInteger visits) { + return new Iterator() { + @Override + public boolean hasNext() { + return delegate.hasNext(); + } + + @Override + public T next() { + visits.incrementAndGet(); + return delegate.next(); + } + + @Override + public void remove() { + delegate.remove(); + } + }; + } + + @Test + public void testCancellationKeepsFifoWakeupOrder() { + pool.tryReserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512, 512); + pool.registerPlanNodeIdToQueryMemoryMap("holder", "fi", "p", MemorySide.INPUT); + pool.tryReserve("holder", "fi", "p", 512, 512); + List requests = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + String fi = Integer.toString(i); + pool.registerPlanNodeIdToQueryMemoryMap("waiting", fi, "p", MemorySide.INPUT); + requests.add(pool.reserveWithPriority("waiting", fi, "p", 64, 512, false)); + } + Assert.assertEquals(64, pool.tryCancel(requests.get(1).getFuture())); + Assert.assertEquals(0, pool.tryCancel(requests.get(1).getFuture())); + Assert.assertEquals(2, pool.getMemoryReservationSize()); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 64); + Assert.assertTrue(requests.get(0).getFuture().isDone()); + Assert.assertFalse(requests.get(2).getFuture().isDone()); + pool.free("waiting", "0", "p", 64); + Assert.assertTrue(requests.get(2).getFuture().isDone()); + pool.free("waiting", "2", "p", 64); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 448); + pool.free("holder", "fi", "p", 512); + for (int i = 0; i < 3; i++) { + pool.deRegisterFragmentInstanceFromQueryMemoryMap("waiting", Integer.toString(i), true); + } + Assert.assertEquals(0, pool.getMemoryReservationSize()); + Assert.assertEquals(0, pool.getReservedBytes()); + } + + @Test + public void testConcurrentCancellationAndGrantRemoveAllPendingIndexes() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + for (int round = 0; round < 32; round++) { + MemoryPool concurrent = new MemoryPool("race", new MemoryManager(1024), 512); + concurrent.registerPlanNodeIdToQueryMemoryMap("holder", "fi", "a", MemorySide.INPUT); + concurrent.registerPlanNodeIdToQueryMemoryMap("holder", "fi", "b", MemorySide.INPUT); + concurrent.tryReserve("holder", "fi", "a", 512, 512); + concurrent.tryReserve("holder", "fi", "b", 512, 512); + List requests = new ArrayList<>(); + for (int i = 0; i < 16; i++) { + String fi = Integer.toString(i); + concurrent.registerPlanNodeIdToQueryMemoryMap("waiting", fi, "p", MemorySide.INPUT); + requests.add(concurrent.reserveWithPriority("waiting", fi, "p", 16, 512, false)); + } + CountDownLatch start = new CountDownLatch(1); + Future grant = + executor.submit( + () -> { + start.await(); + concurrent.free("holder", "fi", "a", 512); + return null; + }); + Future cancel = + executor.submit( + () -> { + start.await(); + for (MemoryReservationResult request : requests) { + concurrent.tryCancel(request.getFuture()); + } + return null; + }); + start.countDown(); + grant.get(10, TimeUnit.SECONDS); + cancel.get(10, TimeUnit.SECONDS); + for (int i = 0; i < requests.size(); i++) { + Assert.assertTrue(requests.get(i).getFuture().isDone()); + if (!requests.get(i).getFuture().isCancelled()) { + concurrent.free("waiting", Integer.toString(i), "p", 16); + } + concurrent.deRegisterFragmentInstanceFromQueryMemoryMap( + "waiting", Integer.toString(i), true); + } + concurrent.free("holder", "fi", "b", 512); + concurrent.deRegisterFragmentInstanceFromQueryMemoryMap("holder", "fi", true); + Assert.assertEquals(0, concurrent.getMemoryReservationSize()); + Assert.assertEquals(0, concurrent.getQueryMemoryReservationSize()); + Assert.assertEquals(0, concurrent.getReservedBytes()); + } + } finally { + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + } + @Test public void testTryReserve() { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java index 49623ab9e45ec..2b3c847e89e46 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -24,6 +24,8 @@ import org.apache.iotdb.calc.execution.operator.process.CollectOperator; import org.apache.iotdb.calc.execution.operator.process.TableMergeSortOperator; import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.memory.MemoryManager; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; @@ -35,6 +37,7 @@ import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.common.PlanFragmentId; import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager.SourceHandleListener; import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISinkHandle; import org.apache.iotdb.db.queryengine.execution.exchange.source.ISourceHandle; import org.apache.iotdb.db.queryengine.execution.exchange.source.LocalSourceHandle; @@ -43,11 +46,13 @@ import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine; +import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; import com.google.common.collect.ArrayListMultimap; import com.google.common.util.concurrent.SettableFuture; @@ -55,6 +60,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.TsBlockSerde; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -63,6 +69,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.List; +import java.util.concurrent.ExecutorService; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -188,7 +196,7 @@ private void assertAdmissionIsShared(boolean shared, MemorySide side, long limit assertEquals(0, instance.getInputMaxBytes()); assertEquals(0, instance.getOutputMaxBytes()); } - String fi = context.getFragmentInstanceId().getFullId(); + String fi = context.getFragmentInstanceId().getFragmentInstanceId(); pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "a" + side, side); pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "b" + side, side); assertTrue( @@ -206,6 +214,56 @@ private void assertAdmissionIsShared(boolean shared, MemorySide side, long limit pool.free("budget", fi, "b" + side, limit); } + @Test + public void testRealSourceHandlesShareCanonicalFragmentAccountAndCleanup() throws Exception { + FragmentInstanceId id = context.getFragmentInstanceId(); + String fi = FragmentInstanceId.createFragmentInstanceIdFromTFragmentInstanceId(id.toThrift()); + LocalMemoryManager manager = mock(LocalMemoryManager.class); + when(manager.getQueryPool()).thenReturn(pool); + SourceHandle first = realSourceHandle(id, "first", manager); + SourceHandle second = realSourceHandle(id, "second", manager); + for (SourceHandle source : Arrays.asList(first, second)) { + ExchangeOperator exchange = mock(ExchangeOperator.class); + when(exchange.getSourceHandle()).thenReturn(source); + context.addExchangeOperator(exchange); + } + try { + context.setExchangeMemoryLimits(collect(), pool); + assertEquals(1024, context.getInstanceContext().getInputMaxBytes()); + first.isBlocked(); + second.isBlocked(); + first.updatePendingDataBlockInfo(0, List.of(1024L)); + second.updatePendingDataBlockInfo(0, List.of(1024L)); + assertEquals(1024, pool.getFragmentMemoryReservedBytes("budget", fi, MemorySide.INPUT)); + assertEquals(0, pool.getPlanNodeMemoryReservedBytes("budget", fi, "second")); + assertEquals(1, pool.getMemoryReservationSize()); + first.abort(); + assertEquals(1024, pool.getPlanNodeMemoryReservedBytes("budget", fi, "second")); + } finally { + first.abort(); + second.abort(); + pool.deRegisterFragmentInstanceFromQueryMemoryMap("budget", fi, true); + } + assertEquals(0, pool.getMemoryReservationSize()); + assertEquals(0, pool.getReservedBytes()); + assertEquals(0, pool.getQueryMemoryReservationSize()); + } + + private static SourceHandle realSourceHandle( + FragmentInstanceId id, String planNode, LocalMemoryManager manager) { + return new SourceHandle( + new TEndPoint("remote", 12345), + new TFragmentInstanceId(id.getQueryId().getId(), 1, "0"), + id.toThrift(), + planNode, + 0, + manager, + mock(ExecutorService.class), + new TsBlockSerde(), + mock(SourceHandleListener.class), + mock(IClientManager.class)); + } + @Test public void testPureCollectUsesSharedSideBudgets() { ISourceHandle source = addSource(context); @@ -515,7 +573,7 @@ public void testMergeSortGetsBothHeadsDespitePrefetchOnOneInput() throws Excepti context.setExchangeMemoryLimits( node(PlanNodeType.TABLE_MERGESORT_NODE, collect(), node(PlanNodeType.TABLE_EXCHANGE_NODE)), pool); - String fi = context.getFragmentInstanceId().getFullId(); + String fi = context.getFragmentInstanceId().getFragmentInstanceId(); pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "a", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "b", MemorySide.INPUT); for (int i = 0; i < 4; i++) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java index ac32c831c43ec..31c562199fd63 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java @@ -130,7 +130,7 @@ public void testTablePlanEnablesSharingBeforeMemoryEstimationAndRootFactory() th verify(secondHandle).setMaxBytesCanReserve(1024); String query = instance.getId().getQueryId().getId(); - String fi = instance.getId().getFullId(); + String fi = instance.getId().getFragmentInstanceId(); pool.registerPlanNodeIdToQueryMemoryMap(query, fi, "first", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap(query, fi, "second", MemorySide.INPUT); assertTrue( @@ -144,7 +144,7 @@ public void testTablePlanEnablesSharingBeforeMemoryEstimationAndRootFactory() th collect.close(); } finally { pool.deRegisterFragmentInstanceFromQueryMemoryMap( - instance.getId().getQueryId().getId(), instance.getId().getFullId(), true); + instance.getId().getQueryId().getId(), instance.getId().getFragmentInstanceId(), true); long reserved = freeBefore - planner.getFreeMemoryForOperators(); if (reserved > 0) { planner.releaseToFreeMemoryForOperators(reserved); From 5a5b9a37dce73576db0aea01bc3c539f5907c5b8 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 20 Oct 2026 11:00:09 +0800 Subject: [PATCH 16/29] Preserve sequential grouping plans in ineligible Collect fragments Keep GroupNode and preGrouped aggregation properties when a fragment retains sequential Collect consumption. Disable ready consumption for local consumers that require grouped or ordered input instead of inserting unnecessary full sorts or converting streaming aggregation to hash aggregation. Add planner regressions for the reviewed TopK/window queries and retain ready consumption coverage for eligible unordered queries. Review: https://github.com/apache/iotdb/pull/18869#discussion_r4300855002 --- .../SharedExchangeMemoryEligibility.java | 56 +++++- .../TableDistributedPlanGenerator.java | 97 ++--------- .../LocalExecutionPlanContextBudgetTest.java | 4 + .../planner/CollectInputPropertyTest.java | 162 ++++++++---------- 4 files changed, 145 insertions(+), 174 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java index 47c31964a3ff9..e66f1719bec61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java @@ -19,20 +19,35 @@ package org.apache.iotdb.db.queryengine.plan.planner; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.FillNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.LimitNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.OffsetNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CopyToNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import java.util.ArrayDeque; import java.util.Deque; import java.util.List; +import java.util.Locale; +import java.util.Set; /** Select shared budgets only for FIs whose fan-in can consume any ready input. */ -final class SharedExchangeMemoryEligibility { +public final class SharedExchangeMemoryEligibility { + private static final Set UNORDERED_AGGREGATIONS = + Set.of("count", "sum", "avg", "min", "max"); + private SharedExchangeMemoryEligibility() {} - static boolean isEligible(PlanNode root) { + public static boolean isEligible(PlanNode root) { boolean hasCollect = false; Deque nodes = new ArrayDeque<>(); nodes.push(root); @@ -49,6 +64,9 @@ static boolean isEligible(PlanNode root) { // Core relational nodes need not implement the serialization-only PlanNode.getType(). boolean anyReady = node instanceof CollectNode || node instanceof UnionNode; hasCollect |= anyReady; + if (!canConsumeUnordered(node)) { + return false; + } List children = node.getChildren(); // Unknown multi-input operators are not implicitly safe. In particular, sequential sinks // and joins can wait for an input while another input holds the entire shared budget. @@ -59,4 +77,38 @@ static boolean isEligible(PlanNode root) { } return hasCollect; } + + private static boolean canConsumeUnordered(PlanNode node) { + if (node instanceof AggregationNode) { + AggregationNode aggregation = (AggregationNode) node; + return aggregation.getPreGroupedSymbols().isEmpty() + && aggregation.getAggregations().values().stream() + .allMatch( + function -> + function.getOrderingScheme().isEmpty() + && !function.isInputOrderedByTimeAscending() + && UNORDERED_AGGREGATIONS.contains( + function + .getResolvedFunction() + .getSignature() + .getName() + .toLowerCase(Locale.ROOT))); + } + // Do not infer safety from the number of children. Windows, streaming grouping and + // TopK can depend on order even when they have only one input. Unknown + // consumers conservatively retain sequential consumption within this FI. + return node instanceof CollectNode + || node instanceof UnionNode + || node instanceof OutputNode + || node instanceof ProjectNode + || node instanceof FilterNode + || node instanceof IdentitySinkNode + || node instanceof ExchangeNode + || node instanceof FillNode + || node instanceof CopyToNode + || node instanceof LimitNode + || node instanceof OffsetNode + || node.getClass() == SortNode.class + || node.getChildren().isEmpty(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 016ee576584bd..d4bab67db0616 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -689,12 +689,6 @@ public List visitGroup(GroupNode node, PlanContext context) { for (PlanNode child : childrenNodes) { if (canSortEliminated(node.getOrderingScheme(), nodeOrderingMap.get(child.getPlanNodeId()))) { result.add(child); - } else if (groupingInvalidatedByCollect( - child, node.getOrderingScheme().getOrderBy().subList(0, node.getPartitionKeyCount()))) { - SortNode sort = - new SortNode(queryId.genPlanNodeId(), child, node.getOrderingScheme(), false, false); - nodeOrderingMap.put(sort.getPlanNodeId(), sort.getOrderingScheme()); - result.add(sort); } else { GroupNode subGroupNode = new GroupNode( @@ -1106,7 +1100,7 @@ public List visitExternalTsFileAggregationScan( if (needFinalAggregation) { OrderingScheme childOrdering = nodeOrderingMap.get(result.get(0).getPlanNodeId()); finalAggregation.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, result)); - return Collections.singletonList(validateAggregationGrouping(finalAggregation)); + return Collections.singletonList(finalAggregation); } return result; } @@ -2051,7 +2045,7 @@ public List visitAggregation(AggregationNode node, PlanContext context } List childrenNodes = node.getChild().accept(this, context); - OrderingScheme childOrdering = commonOrdering(childrenNodes); + OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); if (node.isStreamable()) { // Child has Ordering, we need to check if it is the Ordering we expected if (childOrdering != null) { @@ -2124,7 +2118,7 @@ public List visitAggregation(AggregationNode node, PlanContext context if (physicalAggregation.isStreamable() && childOrdering != null) { nodeOrderingMap.put(planNodeId, expectedOrderingSchema); } - return validateAggregationGrouping(aggregationNode); + return aggregationNode; }) .collect(Collectors.toList()); @@ -2134,10 +2128,10 @@ public List visitAggregation(AggregationNode node, PlanContext context new CollectNode(queryId.genPlanNodeId(), aggregationNodes.get(0).getOutputSymbols()); collectNode.setChildren(aggregationNodes); splitResult.left.setChild(collectNode); - return Collections.singletonList(validateAggregationGrouping(splitResult.left)); + return Collections.singletonList(splitResult.left); } - return Collections.singletonList(validateAggregationGrouping(physicalAggregation)); + return Collections.singletonList(physicalAggregation); } // We cannot do multi-stage Aggregate if any aggregation-function is distinct. @@ -2150,9 +2144,7 @@ public List visitAggregation(AggregationNode node, PlanContext context nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()), childrenNodes); node.setChild(physicalChild); return Collections.singletonList( - validateAggregationGrouping( - withRateFunctionInputOrdering( - node, nodeOrderingMap.get(physicalChild.getPlanNodeId())))); + withRateFunctionInputOrdering(node, nodeOrderingMap.get(physicalChild.getPlanNodeId()))); } Pair splitResult = split(node, symbolAllocator, queryId); AggregationNode intermediate = splitResult.right; @@ -2175,73 +2167,13 @@ public List visitAggregation(AggregationNode node, PlanContext context if (node.isStreamable() && childOrdering != null) { nodeOrderingMap.put(planNodeId, expectedOrderingSchema); } - return validateAggregationGrouping(aggregationNode); + return aggregationNode; }) .collect(Collectors.toList()); splitResult.left.setChild( mergeChildrenViaCollectOrMergeSort( nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()), childrenNodes)); - return Collections.singletonList(validateAggregationGrouping(splitResult.left)); - } - - private AggregationNode validateAggregationGrouping(AggregationNode node) { - if (!node.getPreGroupedSymbols().isEmpty() - && groupingInvalidatedByCollect(node.getChild(), node.getPreGroupedSymbols())) { - node.setPreGroupedSymbols(ImmutableList.of()); - nodeOrderingMap.remove(node.getPlanNodeId()); - } - return node; - } - - /** Whether an unordered fan-in invalidates a previously established grouping property. */ - private boolean groupingInvalidatedByCollect(PlanNode node, List grouping) { - if (grouping.isEmpty()) { - return false; - } - if ((node instanceof CollectNode || node instanceof UnionNode) - && node.getChildren().size() > 1) { - return true; - } - OrderingScheme ordering = nodeOrderingMap.get(node.getPlanNodeId()); - if (ordering != null && prefixMatched(ordering, grouping)) { - return false; - } - // A completed aggregation emits each full grouping key once, even when it hashes its input. - if (node instanceof AggregationNode) { - AggregationNode aggregation = (AggregationNode) node; - if (!aggregation.getStep().isOutputPartial() - && aggregation.getGroupingSetCount() == 1 - && grouping.containsAll(aggregation.getGroupingKeys())) { - return false; - } - } - if (node instanceof ProjectNode) { - ProjectNode project = (ProjectNode) node; - List inputs = new ArrayList<>(); - for (Symbol symbol : grouping) { - Expression expression = project.getAssignments().get(symbol); - if (!(expression instanceof SymbolReference)) { - // A computed key cannot inherit a branch-local continuity proof. - return containsUnorderedCollect(project.getChild()); - } - inputs.add(Symbol.from(expression)); - } - return groupingInvalidatedByCollect(project.getChild(), inputs); - } - for (PlanNode child : node.getChildren()) { - if (groupingInvalidatedByCollect(child, grouping)) { - return true; - } - } - return false; - } - - private boolean containsUnorderedCollect(PlanNode node) { - if ((node instanceof CollectNode || node instanceof UnionNode) - && node.getChildren().size() > 1) { - return true; - } - return node.getChildren().stream().anyMatch(this::containsUnorderedCollect); + return Collections.singletonList(splitResult.left); } private static AggregationNode withRateFunctionInputOrdering( @@ -2369,8 +2301,7 @@ public List visitAggregationTableScan( throw new IllegalStateException( DataNodeQueryMessages.LIST_PLANNODE_SIZE_SHOULD_1_BUT_NOW_IS); } - resultTableScanNodeList = - Collections.singletonList(validateAggregationGrouping(finalAggregation)); + resultTableScanNodeList = Collections.singletonList(finalAggregation); } return resultTableScanNodeList; @@ -2529,7 +2460,7 @@ private List constructSpilledAggregationTableScanByRegionReplicaSet( mergeChildrenViaCollectOrMergeSort( nodeOrderingMap.get(result.get(0).getPlanNodeId()), result)); } - return Collections.singletonList(validateAggregationGrouping(finalAggregation)); + return Collections.singletonList(finalAggregation); } return result; } @@ -2746,8 +2677,7 @@ public List visitAggregationTreeDeviceViewScan( throw new IllegalStateException( DataNodeQueryMessages.LIST_PLANNODE_SIZE_SHOULD_1_BUT_NOW_IS); } - resultTableScanNodeList = - Collections.singletonList(validateAggregationGrouping(finalAggregation)); + resultTableScanNodeList = Collections.singletonList(finalAggregation); } return resultTableScanNodeList; @@ -2933,7 +2863,7 @@ private List constructSpilledAggregationTreeDeviceViewScanByRegionRepl mergeChildrenViaCollectOrMergeSort( nodeOrderingMap.get(result.get(0).getPlanNodeId()), result)); } - return Collections.singletonList(validateAggregationGrouping(finalAggregation)); + return Collections.singletonList(finalAggregation); } return result; } @@ -4077,8 +4007,7 @@ public NodeDistribution getNodeDistribution(PlanNodeId nodeId) { public void clearExpectedOrderingScheme() { expectedOrderingScheme = null; hasSortProperty = false; - // Grouping pushdown depends on this ordering. A boundary such as a global window - // must not pass an outer Group's partitioning request to its scans. + // Grouping pushdown requires an ordering; a global window clears that requirement. pushDownGrouping = false; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java index 2b3c847e89e46..e00322050fdfe 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -120,6 +120,10 @@ private LocalExecutionPlanContext newContext() { private static PlanNode node(PlanNodeType type, PlanNode... children) { Class nodeClass; switch (type) { + case IDENTITY_SINK: + nodeClass = + org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode.class; + break; case TABLE_EXCHANGE_NODE: nodeClass = org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode.class; break; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java index aa20dd0005bc2..28663c2c0a7d9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java @@ -24,8 +24,10 @@ import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GroupNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.StreamSortNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.WindowNode; +import org.apache.iotdb.db.queryengine.plan.planner.SharedExchangeMemoryEligibility; +import org.apache.iotdb.db.queryengine.plan.planner.plan.DistributedQueryPlan; +import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.junit.Test; @@ -37,122 +39,106 @@ import static org.junit.Assert.assertTrue; public class CollectInputPropertyTest { - private static PlanNode plan(String sql) { + private static DistributedQueryPlan plan(String sql) { PlanTester tester = new PlanTester(); tester.createPlan(sql); - return tester.getFragmentPlan(0); + return tester.getDistributedPlan(); } private static List nodes(PlanNode node, Class type) { List result = new ArrayList<>(); if (type.isInstance(node)) result.add(type.cast(node)); - for (PlanNode child : node.getChildren()) result.addAll(nodes(child, type)); + if (!(node instanceof ExchangeNode)) + for (PlanNode child : node.getChildren()) result.addAll(nodes(child, type)); return result; } - @Test - public void testAggregationAboveGlobalWindowHashesInterleavedGroups() { - for (String aggregate : new String[] {"SUM(s1)", "COUNT(DISTINCT s1)"}) { - PlanNode root = - plan( - "SELECT tag1, tag2, tag3, " - + aggregate - + " FROM (" - + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " - + "FROM table1 WHERE tag1 != 'shenzhen') t WHERE n > 0 GROUP BY tag1, tag2, tag3"); - List aggregations = nodes(root, AggregationNode.class); - assertFalse(aggregations.isEmpty()); - assertFalse(nodes(root, CollectNode.class).isEmpty()); - for (AggregationNode aggregation : aggregations) { - if (!nodes(aggregation.getChild(), CollectNode.class).isEmpty()) { - assertTrue(aggregation.getPreGroupedSymbols().isEmpty()); - assertFalse(aggregation.isStreamable()); - } - } - assertTrue( - "Hash aggregation should not add a grouping sort", nodes(root, SortNode.class).isEmpty()); - } + private static void assertSequential(DistributedQueryPlan plan) { + assertTrue( + plan.getInstances().stream() + .anyMatch( + fi -> !nodes(fi.getFragment().getPlanNodeTree(), CollectNode.class).isEmpty())); + for (FragmentInstance fi : plan.getInstances()) + assertFalse( + fi.getId().toString(), + SharedExchangeMemoryEligibility.isEligible(fi.getFragment().getPlanNodeTree())); } @Test - public void testPartialPreGroupingAlsoFallsBackToHash() { - PlanNode root = + public void testTopKWindowKeepsGroupInsteadOfFullSort() { + DistributedQueryPlan query = plan( - "SELECT tag1, tag2, tag3, s1, COUNT(*) FROM (" - + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " - + "FROM table1 WHERE tag1 != 'shenzhen') t WHERE n > 0 GROUP BY tag1, tag2, tag3, s1"); - List aggregations = nodes(root, AggregationNode.class); - assertFalse(aggregations.isEmpty()); - assertFalse(nodes(root, CollectNode.class).isEmpty()); - assertTrue(aggregations.get(0).getPreGroupedSymbols().isEmpty()); + "SELECT tag1, tag2, tag3, s1, SUM(s1) OVER (PARTITION BY tag1, tag2, tag3 ORDER BY s1) rs " + + "FROM (SELECT tag1, tag2, tag3, s1, ROW_NUMBER() OVER () n FROM table1) t " + + "WHERE n > 0 ORDER BY rs DESC LIMIT 10"); + assertSequential(query); + PlanNode root = query.getInstances().get(0).getFragment().getPlanNodeTree(); + assertEquals(1, nodes(root, GroupNode.class).size()); + assertTrue( + nodes(root, SortNode.class).stream().noneMatch(sort -> sort.getClass() == SortNode.class)); } @Test - public void testNestedWindowRestoresPartitionContiguityWithSort() { - PlanNode root = + public void testTopKAggregationKeepsStreamingGroups() { + DistributedQueryPlan query = plan( - "SELECT tag1, tag2, tag3, SUM(s1) OVER (" - + "PARTITION BY tag1, tag2, tag3 ORDER BY s1) AS running_sum " - + "FROM (SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n FROM table1) t WHERE n > 0"); - assertFalse(nodes(root, CollectNode.class).isEmpty()); - assertTrue(nodes(root, GroupNode.class).isEmpty()); - List windows = nodes(root, WindowNode.class); - assertEquals(2, windows.size()); - WindowNode outer = - windows.stream() - .filter(w -> !w.getSpecification().getPartitionBy().isEmpty()) - .findFirst() - .get(); - assertFalse(nodes(outer.getChild(), SortNode.class).isEmpty()); + "SELECT tag1, tag2, tag3, SUM(s1) AS s FROM (SELECT tag1, tag2, tag3, s1, " + + "COUNT(*) OVER () AS n FROM table1 WHERE tag1 != 'shenzhen') t " + + "WHERE n > 0 GROUP BY tag1, tag2, tag3 ORDER BY s DESC LIMIT 10"); + assertSequential(query); + List aggregations = + nodes(query.getInstances().get(0).getFragment().getPlanNodeTree(), AggregationNode.class); + assertEquals(1, aggregations.size()); + assertEquals(3, aggregations.get(0).getPreGroupedSymbols().size()); + assertTrue(aggregations.get(0).isStreamable()); } @Test - public void testStreamSortAfterGlobalWindowUsesFullSort() { - PlanNode root = + public void testNestedWindowKeepsSequentialGroupingPlan() { + DistributedQueryPlan query = plan( - "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " - + "FROM table1 ORDER BY tag1, tag2, tag3, s1"); - assertFalse(nodes(root, CollectNode.class).isEmpty()); - assertFalse(nodes(root, SortNode.class).isEmpty()); - assertTrue(nodes(root, StreamSortNode.class).isEmpty()); - } - - @Test - public void testConstantFillDoesNotRemoveRestoredSort() { - PlanNode root = - plan( - "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n FROM table1 " - + "FILL METHOD CONSTANT 0 ORDER BY tag1, tag2, tag3"); - assertFalse(nodes(root, CollectNode.class).isEmpty()); - assertFalse(nodes(root, SortNode.class).isEmpty()); - assertTrue(nodes(root, StreamSortNode.class).isEmpty()); + "SELECT tag1, tag2, tag3, SUM(s1) OVER (PARTITION BY tag1, tag2, tag3 ORDER BY s1) AS rs " + + "FROM (SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n FROM table1) t WHERE n > 0"); + assertSequential(query); + assertFalse( + nodes(query.getInstances().get(0).getFragment().getPlanNodeTree(), GroupNode.class) + .isEmpty()); } @Test - public void testUnionFinalAggregationDoesNotInheritBranchGrouping() { - PlanNode root = - plan( - "SELECT tag1, tag2, tag3, SUM(s1) FROM (" - + "SELECT tag1, tag2, tag3, s1 FROM table1 UNION ALL " - + "SELECT tag1, tag2, tag3, s1 FROM table2) u GROUP BY tag1, tag2, tag3"); - List aggregations = nodes(root, AggregationNode.class); - assertFalse(aggregations.isEmpty()); - boolean foundFinalCollect = false; - for (AggregationNode aggregation : aggregations) { - if (aggregation.getChild() instanceof CollectNode) { - foundFinalCollect = true; - assertTrue(aggregation.getPreGroupedSymbols().isEmpty()); - } + public void testUnorderedScansProjectionsAggregationsAndUnionRetainReadyMode() { + for (String sql : + new String[] { + "SELECT * FROM table1", + "SELECT time, s1 + 1 AS x FROM table1", + "SELECT time, diff(s1) AS d FROM table1", + "SELECT time, abs(diff(s1)) AS d FROM table1 " + + "WHERE tag1='shenzhen' AND tag2='B2' AND tag3='ZZ'", + "SELECT time, s1 FROM table1 WHERE diff(s1) > 0", + "SELECT COUNT(*), SUM(s1), MIN(s1), MAX(s1) FROM table1", + "SELECT COUNT(DISTINCT s1) FROM table1", + "SELECT s1 FROM table1 UNION ALL SELECT s1 FROM table2" + }) { + DistributedQueryPlan query = plan(sql); + assertTrue( + sql, + query.getInstances().stream() + .anyMatch( + fi -> + SharedExchangeMemoryEligibility.isEligible( + fi.getFragment().getPlanNodeTree()))); } - assertTrue(foundFinalCollect); } @Test public void testOrderedScanBranchesKeepStreamingAggregation() { - PlanNode root = + DistributedQueryPlan query = plan("SELECT tag1, tag2, tag3, SUM(s1 + 1) FROM table1 GROUP BY tag1, tag2, tag3"); - List aggregations = nodes(root, AggregationNode.class); - assertFalse(aggregations.isEmpty()); - assertTrue(aggregations.stream().anyMatch(AggregationNode::isStreamable)); + assertTrue( + query.getInstances().stream() + .anyMatch( + fi -> + nodes(fi.getFragment().getPlanNodeTree(), AggregationNode.class).stream() + .anyMatch(AggregationNode::isStreamable))); } } From 48674aee40b818d150053ecce1512a0e9a3aeb49 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 20 Oct 2026 11:00:43 +0800 Subject: [PATCH 17/29] Detach pending exchange requests safely during FI deregistration Detach pending requests from both indexes without cancelling or completing their futures, preserving owner cancellation accounting and avoiding spurious admission callbacks. Protect publication and account closure with the same fragment monitor, skip closed accounts during retries, and retain the original account identity for indexed cleanup. Keep these lifecycle changes together so detachment remains safe when requests are published or retried concurrently with deregistration. Add regressions for SourceHandle abort/close, publication during deregistration, and recreation of a fragment with the same ID. Review: https://github.com/apache/iotdb/pull/18869#discussion_r4300855005 Review: https://github.com/apache/iotdb/pull/18869#discussion_r4300855008 --- .../execution/memory/MemoryPool.java | 171 +++++++++++------- .../execution/exchange/SourceHandleTest.java | 64 +++++++ .../execution/memory/MemoryPoolTest.java | 106 ++++++++++- 3 files changed, 268 insertions(+), 73 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index 3664a845d4f82..62b95d80d281e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -57,6 +57,7 @@ public static class MemoryReservationFuture extends AbstractFuture { private final long bytesToReserve; // Assigned once by the pool before publication. Zero denotes an untracked future. private long reservationId; + private FragmentMemoryReservation fragment; /** * MemoryReservationFuture is created when an exchange handle tries to reserve memory from the @@ -159,6 +160,7 @@ public long getReservedBytes() { private static class PlanNodeReservation { private final MemorySide side; private volatile long bytesReserved; + private boolean deregistered; private PlanNodeReservation(MemorySide side) { this.side = side; @@ -178,6 +180,8 @@ private static class FragmentMemoryReservation { new ConcurrentHashMap<>(); // Enabled by local planning before drivers start; request limits are supplied by handles. private volatile boolean sharedBudget; + // Protected by this monitor, together with waiter publication and retry. + private boolean deregistered; private AtomicLong getUsedBytes(MemorySide side) { return side == MemorySide.INPUT ? inputUsedBytes : outputUsedBytes; @@ -286,56 +290,72 @@ public void deRegisterFragmentInstanceFromQueryMemoryMap( String queryId, String fragmentInstanceId, boolean forceDeregister) { Map queryRelatedMemory = queryMemoryReservations.get(queryId); - if (queryRelatedMemory != null) { - FragmentMemoryReservation fragmentRelatedMemory = queryRelatedMemory.get(fragmentInstanceId); - boolean hasPotentialMemoryLeak = false; - // fragmentRelatedMemory could be null if the FI has not reserved any memory(For example, - // next() of root operator returns no data) - if (fragmentRelatedMemory != null) { - hasPotentialMemoryLeak = - fragmentRelatedMemory.inputUsedBytes.get() != 0L - || fragmentRelatedMemory.outputUsedBytes.get() != 0L - || fragmentRelatedMemory.planNodes.values().stream() - .anyMatch(value -> value.bytesReserved != 0L); - } - if (!forceDeregister && hasPotentialMemoryLeak) { - // If hasPotentialMemoryLeak is true, it means that LocalSourceChannel/LocalSourceHandles - // have not been closed. - // We should wait for them to be closed. Make sure this method is called again with - // forceDeregister == true, after all LocalSourceChannel/LocalSourceHandles are closed. - return; - } - // Pending requests must not outlive their FI registration. Cancel before removing the - // account so another query's free() cannot retry a request against a missing FI. Do not - // hold the registration lock while notifying cancellation listeners. - if (fragmentRelatedMemory != null) { - for (MemoryReservationFuture future : - fragmentRelatedMemory.pendingReservations.values()) { - tryCancel(future); + if (queryRelatedMemory == null) { + return; + } + FragmentMemoryReservation fragment = queryRelatedMemory.get(fragmentInstanceId); + boolean hasPotentialMemoryLeak = false; + synchronized (queryMemoryReservations) { + if (fragment != null) { + synchronized (fragment) { + if (queryRelatedMemory.get(fragmentInstanceId) != fragment) { + return; + } + if (!forceDeregister && hasReservedMemory(fragment)) { + // Local queues can still hold this FI's allocation after its driver has closed. + return; + } + fragment.deregistered = true; + setPlanNodesDeregistered(fragment, true); + // Wait for reservations that had already acquired their plan-node accounting lock. + hasPotentialMemoryLeak = hasReservedMemory(fragment); + if (!forceDeregister && hasPotentialMemoryLeak) { + setPlanNodesDeregistered(fragment, false); + fragment.deregistered = false; + return; + } + queryRelatedMemory.remove(fragmentInstanceId, fragment); } } - synchronized (queryMemoryReservations) { - queryRelatedMemory.remove(fragmentInstanceId); - // Another fragment may have recreated the query entry while this cleanup waited for the - // lock. - if (queryRelatedMemory.isEmpty() - && queryMemoryReservations.get(queryId) == queryRelatedMemory) { - queryMemoryReservations.remove(queryId); - } + if (queryRelatedMemory.isEmpty() + && queryMemoryReservations.get(queryId) == queryRelatedMemory) { + queryMemoryReservations.remove(queryId); + } + } + // Detach without completing the futures. Only owners may cancel and deduct pending bytes; + // cancellation listeners otherwise mistake cancellation for admission. Publication and + // retries cannot enter this account after it has been marked as deregistered. + if (fragment != null) { + for (MemoryReservationFuture future : fragment.pendingReservations.values()) { + removePendingReservation(future); } - if (hasPotentialMemoryLeak) { - // hasPotentialMemoryLeak means that fragmentRelatedMemory is not null - List> invalidEntryList = - fragmentRelatedMemory.planNodes.entrySet().stream() - .filter(entry -> entry.getValue().bytesReserved != 0L) - .collect(Collectors.toList()); - throw new MemoryLeakException( - String.format( - DataNodeQueryMessages - .QUERY_EXCEPTION_PLANNODE_RELATED_MEMORY_IS_NOT_ZERO_WHEN_TRYING_TO_DEREGISTER_E01109C5, - queryId, - fragmentInstanceId, - invalidEntryList)); + } + if (hasPotentialMemoryLeak) { + List> invalidEntryList = + fragment.planNodes.entrySet().stream() + .filter(entry -> entry.getValue().bytesReserved != 0L) + .collect(Collectors.toList()); + throw new MemoryLeakException( + String.format( + DataNodeQueryMessages + .QUERY_EXCEPTION_PLANNODE_RELATED_MEMORY_IS_NOT_ZERO_WHEN_TRYING_TO_DEREGISTER_E01109C5, + queryId, + fragmentInstanceId, + invalidEntryList)); + } + } + + private static boolean hasReservedMemory(FragmentMemoryReservation fragment) { + return fragment.inputUsedBytes.get() != 0L + || fragment.outputUsedBytes.get() != 0L + || fragment.planNodes.values().stream().anyMatch(value -> value.bytesReserved != 0L); + } + + private static void setPlanNodesDeregistered( + FragmentMemoryReservation fragment, boolean deregistered) { + for (PlanNodeReservation reservation : fragment.planNodes.values()) { + synchronized (reservation) { + reservation.deregistered = deregistered; } } } @@ -397,9 +417,22 @@ public MemoryReservationResult reserveWithPriority( queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve); FragmentMemoryReservation fragment = getFragmentReservation(queryId, fragmentInstanceId, planNodeId); - result.reservationId = nextReservationId.incrementAndGet(); - fragment.pendingReservations.put(result.reservationId, result); - memoryReservationFutures.put(result.reservationId, result); + synchronized (fragment) { + // Deregistration must not miss a request published after its pending-request scan. + if (fragment.deregistered) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_MEMORY_RESERVATION_IS_NOT_REGISTERED_FOR_QUERY_ARG_FRAGMENT_ARG_PLAN_NODE_ARG_67A4B81F, + queryId, + fragmentInstanceId, + planNodeId)); + } + result.fragment = fragment; + result.reservationId = nextReservationId.incrementAndGet(); + fragment.pendingReservations.put(result.reservationId, result); + memoryReservationFutures.put(result.reservationId, result); + } // A concurrent free may have missed this request before it was published. Retry after // publication, under the same future lock used by free and tryCancel to avoid charging twice. try { @@ -407,8 +440,7 @@ public MemoryReservationResult reserveWithPriority( removePendingReservation(result); } } catch (RuntimeException | Error e) { - // A concurrently removed registration must not leave a published request behind. - tryCancel(result); + removePendingReservation(result); throw e; } return new MemoryReservationResult(result, false, bytesToReserve); @@ -509,12 +541,8 @@ public void free(String queryId, String fragmentInstanceId, String planNodeId, l private void removePendingReservation(MemoryReservationFuture future) { memoryReservationFutures.remove(future.reservationId, future); - Map fragments = - queryMemoryReservations.get(future.getQueryId()); - FragmentMemoryReservation fragment = - fragments == null ? null : fragments.get(future.getFragmentInstanceId()); - if (fragment != null) { - fragment.pendingReservations.remove(future.reservationId, future); + if (future.fragment != null) { + future.fragment.pendingReservations.remove(future.reservationId, future); } } @@ -527,16 +555,22 @@ private boolean tryCompleteReservation(MemoryReservationFuture future) { String queryId = future.getQueryId(); String fragmentInstanceId = future.getFragmentInstanceId(); String planNodeId = future.getPlanNodeId(); - if (tryReserve( - queryId, - fragmentInstanceId, - planNodeId, - bytesToReserve, - future.getMaxBytesCanReserve())) { - future.set(null); - return true; + synchronized (future.fragment) { + if (future.fragment.deregistered) { + return true; + } + if (!tryReserve( + queryId, + fragmentInstanceId, + planNodeId, + bytesToReserve, + future.getMaxBytesCanReserve())) { + return false; + } } - return false; + // Owner callbacks may register or release other accounts. + future.set(null); + return true; } } @@ -587,6 +621,9 @@ public boolean tryReserve( // Legacy quotas are independent per plan node. Only shared quotas require the FI lock. Object accountingLock = fragment.sharedBudget ? fragment : reservation; synchronized (accountingLock) { + if (reservation.deregistered) { + return false; + } long usedBytes = fragment.sharedBudget ? fragment.getUsedBytes(reservation.side).get() diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java index 3cfebb1ba295c..df59dffb6c37a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java @@ -77,6 +77,70 @@ public static void afterClass() { IoTDBDescriptor.getInstance().getMemoryConfig().setMaxBytesPerFragmentInstance(maxBytesPerFI); } + @Test + public void testPendingHandleCanCloseAfterForcedMemoryDeregistration() throws Exception { + for (int closeMode = 0; closeMode < 3; closeMode++) { + MemoryPool memoryPool = + new MemoryPool("pending", new MemoryManager(MOCK_TSBLOCK_SIZE), MOCK_TSBLOCK_SIZE); + memoryPool.registerPlanNodeIdToQueryMemoryMap( + "holder", + "0.0", + "holder", + org.apache.iotdb.db.queryengine.execution.memory.MemorySide.INPUT); + Assert.assertTrue( + memoryPool.tryReserve("holder", "0.0", "holder", MOCK_TSBLOCK_SIZE, MOCK_TSBLOCK_SIZE)); + LocalMemoryManager manager = Mockito.mock(LocalMemoryManager.class); + Mockito.when(manager.getQueryPool()).thenReturn(memoryPool); + ExecutorService executor = Mockito.mock(ExecutorService.class); + Mockito.doAnswer( + invocation -> { + ((Runnable) invocation.getArgument(0)).run(); + return null; + }) + .when(executor) + .execute(Mockito.any(Runnable.class)); + SourceHandleListener listener = Mockito.mock(SourceHandleListener.class); + SourceHandle source = + new SourceHandle( + new TEndPoint("remote", 12345), + new TFragmentInstanceId("pending", 1, "0"), + new TFragmentInstanceId("pending", 0, "0"), + "src", + 0, + manager, + executor, + new TsBlockSerde(), + listener, + Mockito.mock(IClientManager.class)); + source.isBlocked(); + source.updatePendingDataBlockInfo(0, List.of(MOCK_TSBLOCK_SIZE)); + Assert.assertEquals(1, memoryPool.getMemoryReservationSize()); + memoryPool.deRegisterFragmentInstanceFromQueryMemoryMap("pending", "0.0", true); + // Detachment must not schedule an RPC for an unreserved block. + Mockito.verify(executor, Mockito.never()).execute(Mockito.any(Runnable.class)); + Mockito.verify(executor, Mockito.never()).submit(Mockito.any(Runnable.class)); + memoryPool.free("holder", "0.0", "holder", MOCK_TSBLOCK_SIZE); + if (closeMode == 0) { + source.abort(); + } else if (closeMode == 1) { + source.abort(new RuntimeException("cancel query")); + } else { + source.close(); + } + Assert.assertEquals(0, source.getBufferRetainedSizeInBytes()); + Assert.assertEquals(0, memoryPool.getMemoryReservationSize()); + Assert.assertEquals(0, memoryPool.getReservedBytes()); + if (closeMode < 2) { + Assert.assertTrue(source.isAborted()); + Mockito.verify(listener).onAborted(source); + } else { + Mockito.verify(listener).onFinished(source); + } + memoryPool.deRegisterFragmentInstanceFromQueryMemoryMap("holder", "0.0", true); + Assert.assertEquals(0, memoryPool.getQueryMemoryReservationSize()); + } + } + @Test public void testCloseStopsFragmentFetchAfterInFlightRpc() throws Exception { testCancellationStopsFragmentFetchAfterInFlightRpc(false); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index d2b901d7f2355..411e249bd2382 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -189,7 +189,7 @@ public void testForceDeregisterReportsMemoryLeak() { } @Test - public void testDeregisterCancelsPendingReservationsBeforeUnrelatedFree() { + public void testDeregisterDetachesPendingReservationsBeforeUnrelatedFree() { pool.registerPlanNodeIdToQueryMemoryMap("healthy", "fi", "a", MemorySide.INPUT); pool.registerPlanNodeIdToQueryMemoryMap("healthy", "fi", "b", MemorySide.INPUT); Assert.assertTrue(pool.tryReserve("healthy", "fi", "a", 512, 512)); @@ -200,14 +200,15 @@ public void testDeregisterCancelsPendingReservationsBeforeUnrelatedFree() { pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); - Assert.assertTrue(pending.getFuture().isCancelled()); + Assert.assertFalse(pending.getFuture().isDone()); + Assert.assertEquals(256, pool.tryCancel(pending.getFuture())); pool.free("healthy", "fi", "a", 512); pool.free("healthy", "fi", "b", 512); Assert.assertEquals(0, pool.getReservedBytes()); } @Test - public void testForceDeregisterCancelsPendingReservationsEvenWhenReportingLeak() { + public void testForceDeregisterDetachesPendingReservationsEvenWhenReportingLeak() { Assert.assertTrue(pool.tryReserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512, 512)); MemoryReservationResult pending = pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256, 512, false); @@ -220,12 +221,100 @@ public void testForceDeregisterCancelsPendingReservationsEvenWhenReportingLeak() pool.deRegisterFragmentInstanceFromQueryMemoryMap( QUERY_ID, FRAGMENT_INSTANCE_ID, true)); - Assert.assertTrue(pending.getFuture().isCancelled()); + Assert.assertFalse(pending.getFuture().isDone()); + Assert.assertEquals(256, pool.tryCancel(pending.getFuture())); pool.free("healthy", "fi", "a", 256); // Forced cleanup still reports and retains the original leaked allocation. Assert.assertEquals(512, pool.getReservedBytes()); } + @Test + @SuppressWarnings("unchecked") + public void testPublicationDuringDeregistrationCannotPoisonOtherQueries() throws Exception { + pool.registerPlanNodeIdToQueryMemoryMap("holder", "fi", "a", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap("holder", "fi", "b", MemorySide.INPUT); + Assert.assertTrue(pool.tryReserve("holder", "fi", "a", 512, 512)); + Assert.assertTrue(pool.tryReserve("holder", "fi", "b", 512, 512)); + Field accounts = MemoryPool.class.getDeclaredField("queryMemoryReservations"); + accounts.setAccessible(true); + Map> queries = + (Map>) accounts.get(pool); + Object fragment = queries.get(QUERY_ID).get(FRAGMENT_INSTANCE_ID); + Field pendingField = fragment.getClass().getDeclaredField("pendingReservations"); + pendingField.setAccessible(true); + AtomicInteger publicationAttempts = new AtomicInteger(); + List detached = new ArrayList<>(); + List healthy = new ArrayList<>(); + pendingField.set( + fragment, + new java.util.concurrent.ConcurrentHashMap< + Long, MemoryPool.MemoryReservationFuture>() { + @Override + public Collection> values() { + // A waiter published after this snapshot was the missed-request race in deregistration. + Collection> snapshot = + new ArrayList<>(super.values()); + publicationAttempts.incrementAndGet(); + try { + detached.add( + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 32, 512, false)); + } catch (IllegalArgumentException expected) { + // Publication must be refused once deregistration has closed this account. + } + pool.registerPlanNodeIdToQueryMemoryMap("waiting", "fi", "later", MemorySide.INPUT); + healthy.add(pool.reserveWithPriority("waiting", "fi", "later", 32, 512, false)); + return snapshot; + } + }); + pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); + Assert.assertEquals(1, publicationAttempts.get()); + pool.free("holder", "fi", "a", 512); + Assert.assertTrue(healthy.get(0).getFuture().isDone()); + pool.free("holder", "fi", "b", 512); + pool.free("waiting", "fi", "later", 32); + for (MemoryReservationResult request : detached) { + Assert.assertFalse(request.getFuture().isDone()); + Assert.assertEquals(32, pool.tryCancel(request.getFuture())); + } + pool.deRegisterFragmentInstanceFromQueryMemoryMap("holder", "fi", true); + pool.deRegisterFragmentInstanceFromQueryMemoryMap("waiting", "fi", true); + Assert.assertEquals(0, pool.getReservedBytes()); + Assert.assertEquals(0, pool.getMemoryReservationSize()); + Assert.assertEquals(0, pool.getQueryMemoryReservationSize()); + } + + @Test + public void testDetachedWaiterDoesNotUseRecreatedFragmentAccount() { + pool.registerPlanNodeIdToQueryMemoryMap("holder", "fi", "a", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap("holder", "fi", "b", MemorySide.INPUT); + Assert.assertTrue(pool.tryReserve("holder", "fi", "a", 512, 512)); + Assert.assertTrue(pool.tryReserve("holder", "fi", "b", 512, 512)); + MemoryReservationResult old = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 32, 512, false); + AtomicInteger callbacks = new AtomicInteger(); + old.getFuture().addListener(callbacks::incrementAndGet, Runnable::run); + pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); + Assert.assertFalse(old.getFuture().isDone()); + Assert.assertEquals(0, callbacks.get()); + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, MemorySide.INPUT); + MemoryReservationResult replacement = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 32, 512, false); + Assert.assertEquals(32, pool.tryCancel(old.getFuture())); + Assert.assertEquals(1, pool.getMemoryReservationSize()); + pool.free("holder", "fi", "a", 512); + Assert.assertTrue(replacement.getFuture().isDone()); + Assert.assertFalse(replacement.getFuture().isCancelled()); + pool.free("holder", "fi", "b", 512); + pool.free(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 32); + pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); + pool.deRegisterFragmentInstanceFromQueryMemoryMap("holder", "fi", true); + Assert.assertEquals(0, pool.getReservedBytes()); + Assert.assertEquals(0, pool.getMemoryReservationSize()); + Assert.assertEquals(0, pool.getQueryMemoryReservationSize()); + } + @Test public void testNonForcedDeregisterKeepsPendingReservationWhileMemoryIsLive() { Assert.assertTrue(pool.tryReserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512, 512)); @@ -257,7 +346,8 @@ public void testDeregisterDoesNotCancelOtherFragmentPendingReservation() { pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); - Assert.assertTrue(pending.getFuture().isCancelled()); + Assert.assertFalse(pending.getFuture().isDone()); + Assert.assertEquals(256, pool.tryCancel(pending.getFuture())); Assert.assertFalse(other.getFuture().isDone()); pool.free("healthy", "fi", "a", 512); Assert.assertTrue(other.getFuture().isDone()); @@ -273,7 +363,7 @@ public void testOwnerCancellationAndDeregistrationDoNotScanUnrelatedWaiters() th } @Test - public void testDeregistrationCancelsOnlyIndexedWaiters() throws Exception { + public void testDeregistrationDetachesOnlyIndexedWaiters() throws Exception { assertDeregistrationDoesNotScanUnrelatedWaiters(false); } @@ -298,6 +388,10 @@ private void assertDeregistrationDoesNotScanUnrelatedWaiters(boolean ownerCancel Assert.assertEquals(8, pool.tryCancel(victims.get(i).getFuture())); } pool.deRegisterFragmentInstanceFromQueryMemoryMap("victim", Integer.toString(i), true); + if (!ownerCancels) { + Assert.assertFalse(victims.get(i).getFuture().isDone()); + Assert.assertEquals(8, pool.tryCancel(victims.get(i).getFuture())); + } Assert.assertTrue(victims.get(i).getFuture().isCancelled()); Assert.assertFalse(unrelated.get(i).getFuture().isDone()); } From 3f184ea5c94a7b53a13275151a66b1679405b605 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 20 Oct 2026 11:47:41 +0800 Subject: [PATCH 18/29] Fix targeted Region lookup mocks in peer removal tests RegionMaintainHandler now looks up a Region through getRegionDatabase and getReplicaSets. Stubbing only getAllReplicaSets makes both removal regression fixtures report the Region as absent and skip peer-list reconciliation. Delegate the targeted lookups to the fixture PartitionInfo and update the deleted-Region case to return an empty result from the targeted lookup. CI: https://github.com/apache/iotdb/actions/runs/39362051673/job/118430666646 Validation: 12 targeted tests; English and Chinese reactor compilation. --- .../impl/region/RemovePeerRestartConsensusTest.java | 10 ++++++++-- .../impl/region/RestartDuringRemovePeerTest.java | 10 +++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RemovePeerRestartConsensusTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RemovePeerRestartConsensusTest.java index b03f06c95a2c6..d0e75de97bc9a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RemovePeerRestartConsensusTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RemovePeerRestartConsensusTest.java @@ -99,7 +99,9 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; /** * Runs the ConfigNode removal state machine against real IoTConsensus replication services. Only @@ -159,8 +161,12 @@ public void removalProcedureRepairsRestartedPeerAndWritesContinue() throws Excep final PartitionInfo partitionInfo = newPartitionInfo(locations); final ConfigManager configManager = Mockito.mock(ConfigManager.class, Mockito.RETURNS_DEEP_STUBS); - Mockito.when(configManager.getPartitionManager().getAllReplicaSets()) - .thenAnswer(ignored -> partitionInfo.getAllReplicaSets()); + Mockito.when( + configManager.getPartitionManager().getRegionDatabase(any(TConsensusGroupId.class))) + .thenAnswer(call -> partitionInfo.getRegionDatabase(call.getArgument(0))); + Mockito.when(configManager.getPartitionManager().getReplicaSets(anyString(), anyList())) + .thenAnswer( + call -> partitionInfo.getReplicaSets(call.getArgument(0), call.getArgument(1))); Mockito.when( configManager .getPartitionManager() diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RestartDuringRemovePeerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RestartDuringRemovePeerTest.java index be868283edcc8..12f7769eae8e8 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RestartDuringRemovePeerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RestartDuringRemovePeerTest.java @@ -71,6 +71,7 @@ import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; @@ -117,8 +118,10 @@ public void setUp() throws Exception { loadManager = Mockito.mock(LoadManager.class, Mockito.RETURNS_DEEP_STUBS); when(configManager.getPartitionManager()).thenReturn(partitionManager); when(configManager.getLoadManager()).thenReturn(loadManager); - when(partitionManager.getAllReplicaSets()) - .thenAnswer(ignored -> partitionInfo.getAllReplicaSets()); + when(partitionManager.getRegionDatabase(any(TConsensusGroupId.class))) + .thenAnswer(call -> partitionInfo.getRegionDatabase(call.getArgument(0))); + when(partitionManager.getReplicaSets(anyString(), anyList())) + .thenAnswer(call -> partitionInfo.getReplicaSets(call.getArgument(0), call.getArgument(1))); when(partitionManager.removeRegionLocation(any())) .thenAnswer(call -> partitionInfo.removeRegionLocation(call.getArgument(0))); handler = Mockito.spy(new RegionMaintainHandler(configManager)); @@ -342,7 +345,8 @@ public void correctionRejectsStaleMetadataAndNullResponsesButAcceptsDeletedRegio partitionInfo.removeRegionLocation(new RemoveRegionLocationPlan(regionId, removedNode)); doReturn(null).when(handler).resetPeerList(any(), anyList(), anyMap()); assertFalse(handler.resetPeerListAfterRemoveForIoTConsensusV1(regionId, removedNode)); - when(partitionManager.getAllReplicaSets()).thenReturn(Collections.emptyList()); + when(partitionManager.getReplicaSets(anyString(), anyList())) + .thenReturn(Collections.emptyList()); assertTrue(handler.resetPeerListAfterRemoveForIoTConsensusV1(regionId, removedNode)); } From 45f2be92506e62e878332a2c4e7db0ce6c3d104d Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 20 Oct 2026 16:21:43 +0800 Subject: [PATCH 19/29] Restrict test changes to ready Collect coverage Revert out-of-scope test modifications: pipe replica ordering and receiver snapshot rewrites, peer-removal mock updates, defensive ORDER BY additions, per-node EXPLAIN helpers and the prophylactic UDTF assertion change. The remaining test surface covers only the shared exchange budget and ready Collect feature. --- .../db/it/utils/CollectQueryAssertions.java | 9 -- .../it/dual/tablemodel/TableModelUtils.java | 6 +- .../it/triple/IoTDBShowReceiversTripleIT.java | 103 ++++++------------ .../it/db/it/IoTDBMultiDeviceTableIT.java | 2 +- .../udf/IoTDBUserDefinedTableFunctionIT.java | 57 +++++----- .../it/insertquery/IoTDBInsertQueryIT.java | 4 +- .../query/recent/IoTDBClusterResultSetIT.java | 10 -- .../it/query/recent/IoTDBCollectFanInIT.java | 4 +- .../it/query/recent/IoTDBCteIT.java | 82 ++++++-------- .../query/recent/IoTDBTableAggregationIT.java | 29 ++--- .../IoTDBTopKRuntimeFilterScanPathIT.java | 3 +- .../RemovePeerRestartConsensusTest.java | 10 +- .../region/RestartDuringRemovePeerTest.java | 10 +- 13 files changed, 123 insertions(+), 206 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java index e53ad150ca431..0c5906a9e3c37 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QueryBody; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QuerySpecification; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.TableSubquery; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; import org.apache.iotdb.itbase.runtime.ClusterTestResultSet; @@ -78,14 +77,6 @@ public static List getUnorderedResultSets(String sql, ResultSet resul return nodeResults(resultSet); } - /** EXPLAIN text may contain node-specific details; each node must satisfy the plan assertions. */ - public static List getExplainResultSets(String sql, ResultSet resultSet) { - assertTrue( - "Plan assertions require EXPLAIN: " + sql, - new SqlParser().createStatement(sql, ZoneOffset.UTC, null) instanceof Explain); - return nodeResults(resultSet); - } - private static List nodeResults(ResultSet resultSet) { return resultSet instanceof ClusterTestResultSet ? ClusterTestResultSet.getNodeResultSets(resultSet) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java index 700d552f0c9a0..2aa5b49f331f6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java @@ -366,11 +366,7 @@ public static String generateHeaderResults() { } public static String getQuerySql(final String table) { - // ClusterTestResultSet compares replicas row by row. All tags plus time form a unique key, - // so require the same row order even when Collect consumes inputs in readiness order. - return "select s0,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,time from " - + table - + " order by s0,s1,s2,s3,time"; + return "select s0,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,time from " + table; } public static String getQueryCountSql(final String table) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java index 5d5313d9f891f..0712387775b9d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java @@ -54,8 +54,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getUnorderedResultSets; - @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT3.class}) public class IoTDBShowReceiversTripleIT { @@ -216,65 +214,36 @@ private boolean hasReceiversOnDifferentDataNodes( final String secondPipeName, final int secondReceiverDataNodeId) throws SQLException { + boolean hasFirstPipe = false; + boolean hasSecondPipe = false; + final Set receiverDataNodeIds = new HashSet<>(); try (final Connection connection = receiverEnv.getConnection( SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect); final Statement statement = connection.createStatement(); final ResultSet resultSet = statement.executeQuery(sql)) { - // Each node must see both senders, but unordered table queries may return different row - // orders. - for (final ResultSet nodeResult : - BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) - ? getUnorderedResultSets(sql, resultSet) - : Collections.singletonList(resultSet)) { - if (!hasExpectedReceivers( - nodeResult, - sqlDialect, - firstPipeName, - firstReceiverDataNodeId, - secondPipeName, - secondReceiverDataNodeId)) { - return false; + while (resultSet.next()) { + final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids"); + if (!"DataNode".equals(getString(resultSet, "ReceiverNodeType", "receiver_node_type")) + || !"thrift".equals(getString(resultSet, "Protocol", "protocol")) + || pipeIds == null) { + continue; + } + final int receiverDataNodeId = getInt(resultSet, "ReceiverNodeId", "receiver_node_id"); + if (receiverDataNodeId == firstReceiverDataNodeId + && pipeIds.contains(firstPipeName + "@") + && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 + && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { + hasFirstPipe = true; + receiverDataNodeIds.add(receiverDataNodeId); + } + if (receiverDataNodeId == secondReceiverDataNodeId + && pipeIds.contains(secondPipeName + "@") + && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 + && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { + hasSecondPipe = true; + receiverDataNodeIds.add(receiverDataNodeId); } - } - return true; - } - } - - private static boolean hasExpectedReceivers( - final ResultSet resultSet, - final String sqlDialect, - final String firstPipeName, - final int firstReceiverDataNodeId, - final String secondPipeName, - final int secondReceiverDataNodeId) - throws SQLException { - boolean hasFirstPipe = false; - boolean hasSecondPipe = false; - final Set receiverDataNodeIds = new HashSet<>(); - while (resultSet.next()) { - final String pipeIds = getString(resultSet, sqlDialect, "PipeIDs", "pipe_ids"); - if (!"DataNode" - .equals(getString(resultSet, sqlDialect, "ReceiverNodeType", "receiver_node_type")) - || !"thrift".equals(getString(resultSet, sqlDialect, "Protocol", "protocol")) - || pipeIds == null) { - continue; - } - final int receiverDataNodeId = - getInt(resultSet, sqlDialect, "ReceiverNodeId", "receiver_node_id"); - if (receiverDataNodeId == firstReceiverDataNodeId - && pipeIds.contains(firstPipeName + "@") - && getInt(resultSet, sqlDialect, "ConnectionCount", "connection_count") >= 1 - && getInt(resultSet, sqlDialect, "PipeCount", "pipe_count") >= 1) { - hasFirstPipe = true; - receiverDataNodeIds.add(receiverDataNodeId); - } - if (receiverDataNodeId == secondReceiverDataNodeId - && pipeIds.contains(secondPipeName + "@") - && getInt(resultSet, sqlDialect, "ConnectionCount", "connection_count") >= 1 - && getInt(resultSet, sqlDialect, "PipeCount", "pipe_count") >= 1) { - hasSecondPipe = true; - receiverDataNodeIds.add(receiverDataNodeId); } } return hasFirstPipe && hasSecondPipe && receiverDataNodeIds.size() >= 2; @@ -297,22 +266,22 @@ private int getDataNodeId(final DataNodeWrapper targetDataNode) throws Exception } private static String getString( - final ResultSet resultSet, - final String sqlDialect, - final String treeColumnName, - final String tableColumnName) + final ResultSet resultSet, final String treeColumnName, final String tableColumnName) throws SQLException { - return resultSet.getString( - BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) ? tableColumnName : treeColumnName); + try { + return resultSet.getString(treeColumnName); + } catch (final SQLException ignored) { + return resultSet.getString(tableColumnName); + } } private static int getInt( - final ResultSet resultSet, - final String sqlDialect, - final String treeColumnName, - final String tableColumnName) + final ResultSet resultSet, final String treeColumnName, final String tableColumnName) throws SQLException { - return resultSet.getInt( - BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) ? tableColumnName : treeColumnName); + try { + return resultSet.getInt(treeColumnName); + } catch (final SQLException ignored) { + return resultSet.getInt(tableColumnName); + } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java index 999f16e4b0806..16d92c213436d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java @@ -264,7 +264,7 @@ public void testMultiDeviceQueryAndDelete() { } private void testSelectAll() { - String selectSql = "select * from t ORDER BY tag1, tag2, time"; + String selectSql = "select * from t"; try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java index 25c7176ca9fc8..9db3acb13e098 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java @@ -39,7 +39,6 @@ import java.util.HashSet; import java.util.Set; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getUnorderedResultSets; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -201,41 +200,39 @@ public void testLargeResultIsSplitWithoutDataLoss() throws Exception { "large_result", "org.apache.iotdb.db.query.udf.example.relational.LargeResultTableFunction"); + Set returnedRows = new HashSet<>(); try (Connection connection = EnvFactory.getEnv().getTableConnection(); Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); - String sql = - "SELECT * FROM large_result(vehicle, " - + LARGE_RESULT_REPEAT_COUNT - + ", " - + LARGE_RESULT_PAYLOAD_SIZE - + ")"; - try (ResultSet clusterResult = statement.executeQuery(sql)) { - for (ResultSet resultSet : getUnorderedResultSets(sql, clusterResult)) { - Set returnedRows = new HashSet<>(); - while (resultSet.next()) { - int repeatIndex = resultSet.getInt("repeat_index"); - long time = resultSet.getLong("time"); - Assert.assertTrue(repeatIndex >= 0 && repeatIndex < LARGE_RESULT_REPEAT_COUNT); - Assert.assertEquals( - repeatIndex + ":" + "x".repeat(LARGE_RESULT_PAYLOAD_SIZE), - resultSet.getString("payload")); - assertPassThroughColumns(resultSet, time); - Assert.assertTrue( - "Duplicate result row for time " + time + " and repeat index " + repeatIndex, - returnedRows.add(time + ":" + repeatIndex)); - } - - long[] inputTimes = new long[] {1, 2, 3, 5}; - Assert.assertEquals(inputTimes.length * LARGE_RESULT_REPEAT_COUNT, returnedRows.size()); - for (long time : inputTimes) { - for (int repeatIndex = 0; repeatIndex < LARGE_RESULT_REPEAT_COUNT; repeatIndex++) { - Assert.assertTrue(returnedRows.contains(time + ":" + repeatIndex)); - } - } + try (ResultSet resultSet = + statement.executeQuery( + "SELECT * FROM large_result(vehicle, " + + LARGE_RESULT_REPEAT_COUNT + + ", " + + LARGE_RESULT_PAYLOAD_SIZE + + ")")) { + while (resultSet.next()) { + int repeatIndex = resultSet.getInt("repeat_index"); + long time = resultSet.getLong("time"); + Assert.assertTrue(repeatIndex >= 0 && repeatIndex < LARGE_RESULT_REPEAT_COUNT); + Assert.assertEquals( + repeatIndex + ":" + "x".repeat(LARGE_RESULT_PAYLOAD_SIZE), + resultSet.getString("payload")); + assertPassThroughColumns(resultSet, time); + Assert.assertTrue( + "Duplicate result row for time " + time + " and repeat index " + repeatIndex, + returnedRows.add(time + ":" + repeatIndex)); } } } + + long[] inputTimes = new long[] {1, 2, 3, 5}; + Assert.assertEquals(inputTimes.length * LARGE_RESULT_REPEAT_COUNT, returnedRows.size()); + for (long time : inputTimes) { + for (int repeatIndex = 0; repeatIndex < LARGE_RESULT_REPEAT_COUNT; repeatIndex++) { + Assert.assertTrue(returnedRows.contains(time + ":" + repeatIndex)); + } + } } private static void assertPassThroughColumns(ResultSet resultSet, long time) throws SQLException { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java index dc32124cc64d3..75a7725c41498 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java @@ -280,9 +280,7 @@ public void testPartialColumn() throws SQLException { statement.execute("DELETE FROM vehicle1"); statement.execute( "INSERT INTO vehicle1(time, s2, manufacturer) SELECT time, s2, manufacturer FROM vehicle0"); - resultSet = - statement.executeQuery( - "SELECT deviceId, manufacturer, s1, s2 FROM vehicle1 ORDER BY manufacturer, s2"); + resultSet = statement.executeQuery("SELECT deviceId, manufacturer, s1, s2 FROM vehicle1"); while (resultSet.next()) { for (int i = 1; i <= 4; i++) { if (i % 2 == 1) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java index 332c69477bdad..3371c3d4dd988 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java @@ -40,7 +40,6 @@ import java.util.List; import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getExplainResultSets; import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getUnorderedResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; @@ -138,15 +137,6 @@ public void testSingleNodeAndNullValues() throws SQLException { } } - @Test - public void testExplainChecksAreExplicit() throws SQLException { - try (ResultSet result = cluster(new String[] {"plan-a"}, new String[] {"plan-b"})) { - assertThrows(AssertionError.class, () -> getExplainResultSets("SELECT value FROM t", result)); - assertEquals( - 2, getExplainResultSets("EXPLAIN SELECT value FROM t ORDER BY value", result).size()); - } - } - private static ResultSet cluster(String[]... nodeRows) throws SQLException { List statements = new ArrayList<>(); List endpoints = new ArrayList<>(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java index 72e75a9cfb9c3..4c04dfc68898a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java @@ -125,7 +125,9 @@ public void testPipeVerificationQueryUsesDeterministicPrimaryKeyOrder() throws E Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE); try (ResultSet result = - statement.executeQuery(TableModelUtils.getQuerySql("pipe_readings"))) { + statement.executeQuery( + "select s0,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,time from pipe_readings " + + "order by s0,s1,s2,s3,time")) { TestUtils.assertResultSetEqual(result, TableModelUtils.generateHeaderResults(), expected); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java index 3d2966b1872c4..65d3f277e52de 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java @@ -58,7 +58,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getExplainResultSets; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.assertEquals; @@ -362,22 +361,19 @@ public void testNestExplain1() throws SQLException { statement.execute("USE testdb"); // explain - try (ResultSet clusterResult = statement.executeQuery(sql)) { - for (ResultSet resultSet : getExplainResultSets(sql, clusterResult)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(metaData.getColumnCount(), 1); - assertEquals(metaData.getColumnName(1), "distribution plan"); - - StringBuilder sb = new StringBuilder(); - while (resultSet.next()) { - sb.append(resultSet.getString(1)).append(System.lineSeparator()); - } - String result = sb.toString(); - assertFalse(result.contains("CTE Query : 'cte1'")); - assertTrue(result.contains("CTE Query : 'cte2'")); - assertTrue(result.contains("Main Query")); - } + ResultSet resultSet = statement.executeQuery(sql); + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnCount(), 1); + assertEquals(metaData.getColumnName(1), "distribution plan"); + + StringBuilder sb = new StringBuilder(); + while (resultSet.next()) { + sb.append(resultSet.getString(1)).append(System.lineSeparator()); } + String result = sb.toString(); + assertFalse(result.contains("CTE Query : 'cte1'")); + assertTrue(result.contains("CTE Query : 'cte2'")); + assertTrue(result.contains("Main Query")); } } @@ -392,22 +388,19 @@ public void testNestExplain2() throws SQLException { statement.execute("USE testdb"); // explain - try (ResultSet clusterResult = statement.executeQuery(sql)) { - for (ResultSet resultSet : getExplainResultSets(sql, clusterResult)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(metaData.getColumnCount(), 1); - assertEquals(metaData.getColumnName(1), "distribution plan"); - - StringBuilder sb = new StringBuilder(); - while (resultSet.next()) { - sb.append(resultSet.getString(1)).append(System.lineSeparator()); - } - String result = sb.toString(); - assertTrue(result.contains("CTE Query : 'cte1'")); - assertTrue(result.contains("CTE Query : 'cte2'")); - assertTrue(result.contains("Main Query")); - } + ResultSet resultSet = statement.executeQuery(sql); + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnCount(), 1); + assertEquals(metaData.getColumnName(1), "distribution plan"); + + StringBuilder sb = new StringBuilder(); + while (resultSet.next()) { + sb.append(resultSet.getString(1)).append(System.lineSeparator()); } + String result = sb.toString(); + assertTrue(result.contains("CTE Query : 'cte1'")); + assertTrue(result.contains("CTE Query : 'cte2'")); + assertTrue(result.contains("Main Query")); } } @@ -422,22 +415,19 @@ public void testExplainJoin() throws SQLException { statement.execute("USE testdb"); // explain - try (ResultSet clusterResult = statement.executeQuery(sql)) { - for (ResultSet resultSet : getExplainResultSets(sql, clusterResult)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(metaData.getColumnCount(), 1); - assertEquals(metaData.getColumnName(1), "distribution plan"); - - StringBuilder sb = new StringBuilder(); - while (resultSet.next()) { - sb.append(resultSet.getString(1)).append(System.lineSeparator()); - } - String result = sb.toString(); - assertFalse(result.contains("CTE Query : 'cte1'")); - assertTrue(result.contains("CTE Query : 'cte2'")); - assertTrue(result.contains("Main Query")); - } + ResultSet resultSet = statement.executeQuery(sql); + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnCount(), 1); + assertEquals(metaData.getColumnName(1), "distribution plan"); + + StringBuilder sb = new StringBuilder(); + while (resultSet.next()) { + sb.append(resultSet.getString(1)).append(System.lineSeparator()); } + String result = sb.toString(); + assertFalse(result.contains("CTE Query : 'cte1'")); + assertTrue(result.contains("CTE Query : 'cte2'")); + assertTrue(result.contains("Main Query")); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index d707cd65bdc51..518b3da1fdd21 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -41,7 +41,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getExplainResultSets; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; @@ -5903,23 +5902,19 @@ private void assertRateQueryUsesIntermediateAggregation(String query) { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); - try (ResultSet clusterResult = statement.executeQuery("EXPLAIN (FORMAT JSON) " + query)) { - for (ResultSet resultSet : - getExplainResultSets("EXPLAIN (FORMAT JSON) " + query, clusterResult)) { - Assert.assertTrue(resultSet.next()); - String plan = resultSet.getString(1); - // The PARTIAL/FINAL split only exists when the scanned data spans multiple DataRegions. - // Under randomized partition allocation (e.g. the SHUFFLE strategy used by - // IoTDBTableAggregation2IT), all devices may occasionally land in one single region, - // where a SINGLE-step aggregation is the correct plan, so the check must be skipped. - if (countDistinctRegionIds(plan) < 2) { - continue; - } - Assert.assertTrue( - "Expected a PARTIAL aggregation in plan: " + plan, plan.contains("PARTIAL")); - Assert.assertTrue( - "Expected a FINAL aggregation in plan: " + plan, plan.contains("FINAL")); + try (ResultSet resultSet = statement.executeQuery("EXPLAIN (FORMAT JSON) " + query)) { + Assert.assertTrue(resultSet.next()); + String plan = resultSet.getString(1); + // The PARTIAL/FINAL split only exists when the scanned data spans multiple DataRegions. + // Under randomized partition allocation (e.g. the SHUFFLE strategy used by + // IoTDBTableAggregation2IT), all devices may occasionally land in one single region, + // where a SINGLE-step aggregation is the correct plan, so the check must be skipped. + if (countDistinctRegionIds(plan) < 2) { + return; } + Assert.assertTrue( + "Expected a PARTIAL aggregation in plan: " + plan, plan.contains("PARTIAL")); + Assert.assertTrue("Expected a FINAL aggregation in plan: " + plan, plan.contains("FINAL")); } } catch (Exception exception) { throw new AssertionError(exception); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java index 59499aa5db0de..7697bc81649bd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java @@ -140,8 +140,7 @@ public void unionAllSiblingTopKOrderByTimeLimit() { tableResultSetEqualTest( "(SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd1' ORDER BY time DESC LIMIT 1)" + " UNION ALL" - + " (SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd3' ORDER BY time DESC LIMIT 1)" - + " ORDER BY time, device_id, s1", + + " (SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd3' ORDER BY time DESC LIMIT 1)", expectedHeader, retArray, TABLE_DATABASE); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RemovePeerRestartConsensusTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RemovePeerRestartConsensusTest.java index d0e75de97bc9a..b03f06c95a2c6 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RemovePeerRestartConsensusTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RemovePeerRestartConsensusTest.java @@ -99,9 +99,7 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; /** * Runs the ConfigNode removal state machine against real IoTConsensus replication services. Only @@ -161,12 +159,8 @@ public void removalProcedureRepairsRestartedPeerAndWritesContinue() throws Excep final PartitionInfo partitionInfo = newPartitionInfo(locations); final ConfigManager configManager = Mockito.mock(ConfigManager.class, Mockito.RETURNS_DEEP_STUBS); - Mockito.when( - configManager.getPartitionManager().getRegionDatabase(any(TConsensusGroupId.class))) - .thenAnswer(call -> partitionInfo.getRegionDatabase(call.getArgument(0))); - Mockito.when(configManager.getPartitionManager().getReplicaSets(anyString(), anyList())) - .thenAnswer( - call -> partitionInfo.getReplicaSets(call.getArgument(0), call.getArgument(1))); + Mockito.when(configManager.getPartitionManager().getAllReplicaSets()) + .thenAnswer(ignored -> partitionInfo.getAllReplicaSets()); Mockito.when( configManager .getPartitionManager() diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RestartDuringRemovePeerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RestartDuringRemovePeerTest.java index 12f7769eae8e8..be868283edcc8 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RestartDuringRemovePeerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/RestartDuringRemovePeerTest.java @@ -71,7 +71,6 @@ import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; @@ -118,10 +117,8 @@ public void setUp() throws Exception { loadManager = Mockito.mock(LoadManager.class, Mockito.RETURNS_DEEP_STUBS); when(configManager.getPartitionManager()).thenReturn(partitionManager); when(configManager.getLoadManager()).thenReturn(loadManager); - when(partitionManager.getRegionDatabase(any(TConsensusGroupId.class))) - .thenAnswer(call -> partitionInfo.getRegionDatabase(call.getArgument(0))); - when(partitionManager.getReplicaSets(anyString(), anyList())) - .thenAnswer(call -> partitionInfo.getReplicaSets(call.getArgument(0), call.getArgument(1))); + when(partitionManager.getAllReplicaSets()) + .thenAnswer(ignored -> partitionInfo.getAllReplicaSets()); when(partitionManager.removeRegionLocation(any())) .thenAnswer(call -> partitionInfo.removeRegionLocation(call.getArgument(0))); handler = Mockito.spy(new RegionMaintainHandler(configManager)); @@ -345,8 +342,7 @@ public void correctionRejectsStaleMetadataAndNullResponsesButAcceptsDeletedRegio partitionInfo.removeRegionLocation(new RemoveRegionLocationPlan(regionId, removedNode)); doReturn(null).when(handler).resetPeerList(any(), anyList(), anyMap()); assertFalse(handler.resetPeerListAfterRemoveForIoTConsensusV1(regionId, removedNode)); - when(partitionManager.getReplicaSets(anyString(), anyList())) - .thenReturn(Collections.emptyList()); + when(partitionManager.getAllReplicaSets()).thenReturn(Collections.emptyList()); assertTrue(handler.resetPeerListAfterRemoveForIoTConsensusV1(regionId, removedNode)); } From 82813cbf7dfd3a98b8bac143e781bf6063ed12db Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 20 Oct 2026 22:34:40 +0800 Subject: [PATCH 20/29] Fix ready-first Collect ordering and exchange budget validation --- .../IoTDBMultiTAGsWithAttributesTableIT.java | 5 +- .../it/db/it/IoTDBPatternAggregationIT.java | 14 +- .../it/db/it/IoTDBPatternRecognitionIT.java | 151 +++++--- .../it/db/it/IoTDBWindowFunction3IT.java | 2 +- .../relational/it/db/it/IoTDBWindowTVFIT.java | 74 ++-- .../udf/IoTDBUserDefinedTableFunctionIT.java | 8 +- .../query/old/IoTDBDatetimeFormatTableIT.java | 3 +- .../IoTDBAlignByDeviceTableIT.java | 8 +- .../IoTDBOrderByWithAlignByDeviceTableIT.java | 2 +- .../scalar/IoTDBDiffFunctionTableIT.java | 6 +- .../scalar/IoTDBRoundFunctionTableIT.java | 80 +++- .../IoTDBTopKRuntimeFilterScanPathIT.java | 3 +- .../IoTDBCorrelatedExistsSubqueryIT.java | 183 +++++----- ...IoTDBCorrelatedQuantifiedComparisonIT.java | 67 ++-- .../IoTDBCorrelatedScalarSubqueryIT.java | 139 +++---- .../IoTDBUncorrelatedExistsSubqueryIT.java | 203 ++++++----- ...oTDBUncorrelatedInPredicateSubqueryIT.java | 127 ++++--- ...TDBUncorrelatedQuantifiedComparisonIT.java | 344 +++++++++--------- .../IoTDBUncorrelatedScalarSubqueryIT.java | 168 ++++----- .../IoTDBAlignByDeviceTableViewIT.java | 8 +- ...DBOrderByWithAlignByDeviceTableViewIT.java | 2 +- .../recent/IoTDBTableViewAggregationIT.java | 2 +- .../IoTDBTableViewAggregationNonStreamIT.java | 2 +- .../apache/iotdb/calc/i18n/CalcMessages.java | 4 - .../apache/iotdb/calc/i18n/CalcMessages.java | 3 - .../operator/process/AnyChildBlocked.java | 4 - .../operator/process/CollectOperator.java | 77 +--- .../operator/process/CollectOperatorTest.java | 161 -------- .../DataNodeTableOperatorGenerator.java | 17 - .../planner/LocalExecutionPlanContext.java | 10 +- .../SharedExchangeBudgetEligibility.java | 70 ++++ .../SharedExchangeMemoryEligibility.java | 114 ------ .../TableDistributedPlanGenerator.java | 95 ++++- .../LocalExecutionPlanContextBudgetTest.java | 23 +- .../LocalExecutionPlannerBudgetTest.java | 4 +- .../planner/CollectInputPropertyTest.java | 184 ++++++---- 36 files changed, 1172 insertions(+), 1195 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeBudgetEligibility.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java index 1b293f75865c3..f5af1271ea4a0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java @@ -2087,7 +2087,7 @@ public void fourTableJoinTest() { + " c.course_name," + " g.score " + "from students s, teachers t, courses c, grades g " - + "where s.time=c.time and c.time=g.time and t.teacher_tag = 1005 limit 1"; + + "where s.time=c.time and c.time=g.time and t.teacher_tag = 1005 order by g.score desc limit 1"; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @@ -2196,7 +2196,8 @@ public void innerJoinOnMultiColumns() { expectedHeader = new String[] {"attr1", "attr2"}; sql = - "select t0.attr1,t0.attr2 from table0 t0 join table1 t1 on t0.attr1=t1.attr1 AND t0.attr2=t1.attr2"; + "select t0.attr1,t0.attr2 from table0 t0 join table1 t1 on t0.attr1=t1.attr1 AND t0.attr2=t1.attr2" + + " ORDER BY t0.attr1, t0.attr2"; retArray = new String[] { "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java index e2000380b48db..850d3480bb068 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java @@ -227,7 +227,7 @@ public void test1() { + " B AS B.totalprice = 20, " + " C AS C.totalprice = 30, " + " D AS D.totalprice = 40 " - + ") AS m ", + + ") AS m ORDER BY m.time ", expectedHeader, retArray, DATABASE_NAME); @@ -282,7 +282,7 @@ public void test2() { + " B AS B.totalprice = 20, " + " C AS C.totalprice = 30, " + " D AS D.totalprice = 40 " - + ") AS m ", + + ") AS m ORDER BY m.time ", expectedHeader, retArray, DATABASE_NAME); @@ -333,7 +333,7 @@ public void test3() { + " B AS B.totalprice = 20, " + " C AS C.totalprice = 30, " + " D AS D.totalprice = 40 " - + ") AS m ", + + ") AS m ORDER BY m.time ", expectedHeader, retArray, DATABASE_NAME); @@ -384,7 +384,7 @@ public void test4() { + " B AS B.totalprice = 20, " + " C AS C.totalprice = 30, " + " D AS D.totalprice = 40 " - + ") AS m ", + + ") AS m ORDER BY m.time ", expectedHeader, retArray, DATABASE_NAME); @@ -426,7 +426,7 @@ public void test5() { + " B AS totalprice = 20, " + " C AS totalprice = 30, " + " D AS totalprice = 40 " - + ") AS m ", + + ") AS m ORDER BY m.time ", expectedHeader, retArray, DATABASE_NAME); @@ -476,7 +476,7 @@ public void test6() { + " PATTERN (A+) " + " DEFINE " + " A AS true " - + ") AS m ", + + ") AS m ORDER BY m.time ", expectedHeader, retArray, DATABASE_NAME); @@ -507,7 +507,7 @@ public void testAggregationsInDefineClause() { + " PATTERN ((A | B)*) " + " DEFINE " + " A AS AVG(totalprice) = 5 " - + ") AS m ", + + ") AS m ORDER BY m.time ", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java index 070ba0b6348b4..9dd2ec8809bd0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java @@ -31,9 +31,13 @@ import org.junit.runner.RunWith; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; import static java.lang.String.format; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -279,32 +283,24 @@ public void testOutputMode() { + " B AS B.totalprice < PREV(B.totalprice) " + ") AS m"; - tableResultSetEqualTest( - format(sql, "ONE ROW PER MATCH", "PATTERN (B*)"), expectedHeader, retArray1, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "ONE ROW PER MATCH", "PATTERN (B*)"), expectedHeader, retArray1); - tableResultSetEqualTest( - format(sql, "ALL ROWS PER MATCH", "PATTERN (B*)"), - expectedHeader, - retArray2, - DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "ALL ROWS PER MATCH", "PATTERN (B*)"), expectedHeader, retArray2); - tableResultSetEqualTest( - format(sql, "ALL ROWS PER MATCH", "PATTERN (B+)"), - expectedHeader, - retArray3, - DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "ALL ROWS PER MATCH", "PATTERN (B+)"), expectedHeader, retArray3); - tableResultSetEqualTest( + assertUnorderedTableResult( format(sql, "ALL ROWS PER MATCH OMIT EMPTY MATCHES", "PATTERN (B*)"), expectedHeader, - retArray4, - DATABASE_NAME); + retArray4); - tableResultSetEqualTest( + assertUnorderedTableResult( format(sql, "ALL ROWS PER MATCH WITH UNMATCHED ROWS", "PATTERN (B+)"), expectedHeader, - retArray5, - DATABASE_NAME); + retArray5); } @Test @@ -359,7 +355,7 @@ public void testLogicalNavigationFunction() { + " PATTERN (A+) " + " DEFINE " + " A AS true " - + ") AS m"; + + ") AS m ORDER BY m.time"; // LAST(totalprice) tableResultSetEqualTest(format(sql, "totalprice"), expectedHeader, retArray1, DATABASE_NAME); tableResultSetEqualTest( @@ -454,7 +450,7 @@ public void testPhysicalNavigationFunction() { + " PATTERN (A+) " + " DEFINE " + " A AS true " - + ") AS m"; + + ") AS m ORDER BY m.time"; // PREV(totalprice) tableResultSetEqualTest( format(sql1, "PREV(totalprice)"), expectedHeader1, retArray1, DATABASE_NAME); @@ -588,7 +584,7 @@ public void testNestedNavigation() { + " PATTERN (A+) " + " DEFINE " + " A AS true " - + ") AS m"; + + ") AS m ORDER BY m.time"; // PREV(FIRST(totalprice)) tableResultSetEqualTest( format(sql, "PREV(RPR_FIRST(totalprice))"), expectedHeader, retArray1, DATABASE_NAME); @@ -677,7 +673,7 @@ public void testUnionVariable() { + " A AS A.totalprice = 80, " + " L AS L.totalprice < 80, " + " H AS H.totalprice > 80 " - + ") AS m"; + + ") AS m ORDER BY m.time"; tableResultSetEqualTest(sql, expectedHeader, retArray1, DATABASE_NAME); } @@ -713,7 +709,7 @@ public void testClassifierFunction() { + " A AS A.totalprice = 80, " + " L AS L.totalprice < 80, " + " H AS H.totalprice > 80 " - + ") AS m"; + + ") AS m ORDER BY m.time"; tableResultSetEqualTest(sql, expectedHeader, retArray1, DATABASE_NAME); } @@ -802,7 +798,7 @@ public void testRowPattern() { // pattern concatenation tableResultSetEqualTest( format( - sql, + sql + " ORDER BY m.time", "PATTERN (A B C) " + "DEFINE " + " B AS B.totalprice < PREV (B.totalprice), " @@ -814,7 +810,7 @@ public void testRowPattern() { // pattern alternation tableResultSetEqualTest( format( - sql, + sql + " ORDER BY m.time", "PATTERN (B | C | A) " + "DEFINE " + " B AS B.totalprice < PREV (B.totalprice), " @@ -826,7 +822,7 @@ public void testRowPattern() { // pattern permutation tableResultSetEqualTest( format( - sql, + sql + " ORDER BY m.time", "PATTERN (PERMUTE(B, C)) " + "DEFINE " + " B AS B.totalprice < PREV (B.totalprice), " @@ -838,7 +834,7 @@ public void testRowPattern() { // grouped pattern tableResultSetEqualTest( format( - sql, + sql + " ORDER BY m.time", "PATTERN (((A) (B (C)))) " + "DEFINE " + " B AS B.totalprice < PREV (B.totalprice), " @@ -905,35 +901,68 @@ public void testPatternQuantifier() { + " B AS B.totalprice <= PREV(B.totalprice) " + ") AS m"; - tableResultSetEqualTest(format(sql, "PATTERN (B*)"), expectedHeader, retArray1, DATABASE_NAME); + tableResultSetEqualTest( + format(sql + " ORDER BY m.time", "PATTERN (B*)"), expectedHeader, retArray1, DATABASE_NAME); - tableResultSetEqualTest(format(sql, "PATTERN (B*?)"), expectedHeader, retArray2, DATABASE_NAME); + tableResultSetEqualTest( + format(sql + " ORDER BY m.time", "PATTERN (B*?)"), + expectedHeader, + retArray2, + DATABASE_NAME); - tableResultSetEqualTest(format(sql, "PATTERN (B+)"), expectedHeader, retArray3, DATABASE_NAME); + tableResultSetEqualTest( + format(sql + " ORDER BY m.time", "PATTERN (B+)"), expectedHeader, retArray3, DATABASE_NAME); - tableResultSetEqualTest(format(sql, "PATTERN (B+?)"), expectedHeader, retArray4, DATABASE_NAME); + tableResultSetEqualTest( + format(sql + " ORDER BY m.time", "PATTERN (B+?)"), + expectedHeader, + retArray4, + DATABASE_NAME); - tableResultSetEqualTest(format(sql, "PATTERN (B?)"), expectedHeader, retArray5, DATABASE_NAME); + tableResultSetEqualTest( + format(sql + " ORDER BY m.time", "PATTERN (B?)"), expectedHeader, retArray5, DATABASE_NAME); - tableResultSetEqualTest(format(sql, "PATTERN (B??)"), expectedHeader, retArray2, DATABASE_NAME); + tableResultSetEqualTest( + format(sql + " ORDER BY m.time", "PATTERN (B??)"), + expectedHeader, + retArray2, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{,})"), expectedHeader, retArray1, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{,})"), + expectedHeader, + retArray1, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{,}?)"), expectedHeader, retArray2, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{,}?)"), + expectedHeader, + retArray2, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{1,})"), expectedHeader, retArray3, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{1,})"), + expectedHeader, + retArray3, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{1,}?)"), expectedHeader, retArray4, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{1,}?)"), + expectedHeader, + retArray4, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{2,})"), expectedHeader, retArray3, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{2,})"), + expectedHeader, + retArray3, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{2,}?)"), expectedHeader, retArray6, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{2,}?)"), + expectedHeader, + retArray6, + DATABASE_NAME); tableResultSetEqualTest( format(sql, "PATTERN (B{5,})"), expectedHeader, new String[] {}, DATABASE_NAME); @@ -942,21 +971,53 @@ public void testPatternQuantifier() { format(sql, "PATTERN (B{5,}?)"), expectedHeader, new String[] {}, DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{,1})"), expectedHeader, retArray5, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{,1})"), + expectedHeader, + retArray5, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{,1}?)"), expectedHeader, retArray2, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{,1}?)"), + expectedHeader, + retArray2, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{1,1})"), expectedHeader, retArray4, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{1,1})"), + expectedHeader, + retArray4, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{1,1}?)"), expectedHeader, retArray4, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{1,1}?)"), + expectedHeader, + retArray4, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{1})"), expectedHeader, retArray4, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{1})"), + expectedHeader, + retArray4, + DATABASE_NAME); tableResultSetEqualTest( - format(sql, "PATTERN (B{1}?)"), expectedHeader, retArray4, DATABASE_NAME); + format(sql + " ORDER BY m.time", "PATTERN (B{1}?)"), + expectedHeader, + retArray4, + DATABASE_NAME); + } + + private static void assertUnorderedTableResult( + String sql, String[] expectedHeader, String[] retArray) { + String header = String.join(",", expectedHeader) + ","; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, resultSet, header, Arrays.asList(retArray)); + } + } catch (SQLException e) { + fail(e.getMessage()); + } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunction3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunction3IT.java index 9922948550478..ff13d8fc154c4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunction3IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunction3IT.java @@ -346,7 +346,7 @@ public void testReplaceWindowWithRowNumber() { "2021-01-01T09:15:00.000Z,d2,4.0,2,", }; tableResultSetEqualTest( - "SELECT *, row_number() OVER (PARTITION BY device) AS rn FROM demo ORDER BY device, time", + "SELECT *, row_number() OVER (PARTITION BY device ORDER BY time) AS rn FROM demo ORDER BY device, time", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java index 2c8a822fb09de..533c284b2a707 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java @@ -41,6 +41,7 @@ import java.util.Comparator; import java.util.List; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -142,7 +143,7 @@ public void testHopFunction() { "2021-01-01T09:15:00.000Z,2021-01-01T09:25:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", }; tableResultSetEqualTest( - "SELECT * FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => 5m, SIZE => 10m) ORDER BY stock_id, time", + "SELECT * FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => 5m, SIZE => 10m) ORDER BY stock_id, time, window_start, window_end", expectedHeader, retArray, DATABASE_NAME); @@ -304,7 +305,7 @@ public void testVariationWithMultiTypeAndNullValue() { "3,2021-01-01T09:05:00.000Z,device1,3," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, long_val FROM variation(multi_type ORDER BY time, 'long_val', 1.0, false)", + "SELECT window_index, time, device_id, long_val FROM variation(multi_type ORDER BY time, 'long_val', 1.0, false) ORDER BY window_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -321,7 +322,7 @@ public void testVariationWithMultiTypeAndNullValue() { "1,2021-01-01T09:05:00.000Z,device1,3.0," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, float_val FROM variation(multi_type ORDER BY time, 'float_val', 1.0, true)", + "SELECT window_index, time, device_id, float_val FROM variation(multi_type ORDER BY time, 'float_val', 1.0, true) ORDER BY window_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -338,7 +339,7 @@ public void testVariationWithMultiTypeAndNullValue() { "0,2021-01-01T09:05:00.000Z,device1,3.0," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, double_val FROM variation(multi_type ORDER BY time, 'double_val', 1.0, true)", + "SELECT window_index, time, device_id, double_val FROM variation(multi_type ORDER BY time, 'double_val', 1.0, true) ORDER BY window_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -355,7 +356,7 @@ public void testVariationWithMultiTypeAndNullValue() { "1,2021-01-01T09:05:00.000Z,device1,false," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, bool_val FROM variation(multi_type ORDER BY time, 'bool_val', 0.0, true)", + "SELECT window_index, time, device_id, bool_val FROM variation(multi_type ORDER BY time, 'bool_val', 0.0, true) ORDER BY window_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -372,7 +373,7 @@ public void testVariationWithMultiTypeAndNullValue() { "1,2021-01-01T09:05:00.000Z,device1,2," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, str_val FROM variation(multi_type ORDER BY time, 'str_val', 0.0, true)", + "SELECT window_index, time, device_id, str_val FROM variation(multi_type ORDER BY time, 'str_val', 0.0, true) ORDER BY window_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -389,7 +390,7 @@ public void testVariationWithMultiTypeAndNullValue() { "4,2021-01-01T09:05:00.000Z,device1,0x02," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, blob_val FROM variation(multi_type ORDER BY time, 'blob_val', 0.0, false)", + "SELECT window_index, time, device_id, blob_val FROM variation(multi_type ORDER BY time, 'blob_val', 0.0, false) ORDER BY window_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -406,7 +407,7 @@ public void testVariationWithMultiTypeAndNullValue() { "3,2021-01-01T09:05:00.000Z,device1,2021-01-01T10:00:00.000Z," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, ts_val FROM variation(DATA=>multi_type ORDER BY time, COL=>'ts_val', ignore_null=>false)", + "SELECT window_index, time, device_id, ts_val FROM variation(DATA=>multi_type ORDER BY time, COL=>'ts_val', ignore_null=>false) ORDER BY window_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -423,7 +424,7 @@ public void testVariationWithMultiTypeAndNullValue() { "1,2021-01-01T09:05:00.000Z,device1,2021-01-02," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, date_val FROM variation(multi_type ORDER BY time, 'date_val', 0.0)", + "SELECT window_index, time, device_id, date_val FROM variation(multi_type ORDER BY time, 'date_val', 0.0) ORDER BY window_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -671,7 +672,7 @@ public void testCumulateFunction() { "2021-01-01T09:12:00.000Z,2021-01-01T09:24:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", }; tableResultSetEqualTest( - "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m) ORDER BY stock_id, time", + "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m) ORDER BY stock_id, time, window_start, window_end", expectedHeader, retArray, DATABASE_NAME); @@ -694,7 +695,7 @@ public void testCumulateFunction() { "2021-01-01T09:06:00.000Z,2021-01-01T09:18:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", }; tableResultSetEqualTest( - "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m, ORIGIN => 2021-01-01T09:06:00) ORDER BY stock_id, time", + "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m, ORIGIN => 2021-01-01T09:06:00) ORDER BY stock_id, time, window_start, window_end", expectedHeader, retArray, DATABASE_NAME); @@ -779,7 +780,7 @@ public void testPatternMatchFunction() { "2,0.6797687270429319,1970-01-01T00:00:00.052Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false)", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false) ORDER BY match_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -810,7 +811,7 @@ public void testPatternMatchFunction() { "1,0.6797687270429319,1970-01-01T00:00:00.052Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 1.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false)", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 1.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false) ORDER BY match_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -837,7 +838,7 @@ public void testPatternMatchFunction() { "1,0.6465473220616865,1970-01-01T00:00:00.021Z,1.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 10.0, height => 500.0, smooth_On_Pattern => false)", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 10.0, height => 500.0, smooth_On_Pattern => false) ORDER BY match_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -855,7 +856,7 @@ public void testPatternMatchFunction() { "0,4.637180787192477,1970-01-01T00:00:00.008Z,1.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 1000.0, height => 2.0, smooth_On_Pattern => false)", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 1000.0, height => 2.0, smooth_On_Pattern => false) ORDER BY match_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -908,7 +909,7 @@ public void testPatternMatchFunction() { "3,0.6797687270429319,1970-01-01T00:00:00.052Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,{2.0,3.0,2.0}+,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false)", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,{2.0,3.0,2.0}+,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false) ORDER BY match_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -955,7 +956,7 @@ public void testPatternMatchFunction() { "3,0.0,1970-01-01T00:00:00.052Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,1.0,{1.0,2.0,3.0,2.0}*,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false)", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,1.0,{1.0,2.0,3.0,2.0}*,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false) ORDER BY match_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -985,7 +986,7 @@ public void testPatternMatchFunction() { "3,1.7704361692956723,1970-01-01T00:00:00.046Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1,2.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => true)", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1,2.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => true) ORDER BY match_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -1013,7 +1014,7 @@ public void testPatternMatchFunction() { "2,0.09999999999999994,1970-01-01T00:00:00.052Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,1.2,1.0', smooth => 0.5, threshold => 1.0, width => 1000.0, height => 500.0, smooth_On_Pattern => true)", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,1.2,1.0', smooth => 0.5, threshold => 1.0, width => 1000.0, height => 500.0, smooth_On_Pattern => true) ORDER BY match_index, time", expectedHeader, retArray, DATABASE_NAME); @@ -1547,12 +1548,10 @@ public void testXCorrP0() { "TESL,29997.0,", "TESL,19890.0," }; - tableResultSetEqualTest( - "SELECT * FROM XCORR(DATA => bid PARTITION BY stock_id ORDER BY time, " - + "TIMECOL => 'time') ORDER BY stock_id", + assertUnorderedTableResult( + "SELECT * FROM XCORR(DATA => bid PARTITION BY stock_id ORDER BY time, TIMECOL => 'time')", expectedHeader, - retArray, - DATABASE_NAME); + retArray); } @Test @@ -1570,13 +1569,12 @@ public void testXCorrWithNullValues() { "device_1,120.0,", "device_1,480.0," }; - tableResultSetEqualTest( + assertUnorderedTableResult( "SELECT * FROM XCORR(DATA => " + "(SELECT time, device_id, s1, s2 FROM table1 WHERE device_id = 'device_1') " + "PARTITION BY device_id ORDER BY time, TIMECOL => 'time')", expectedHeader, - retArray, - DATABASE_NAME); + retArray); } @Test @@ -1594,14 +1592,12 @@ public void testXCorrWithMultiplePartitionColumns() { "F1,device_2,500.0,", "F2,device_1,900.0," }; - tableResultSetEqualTest( + assertUnorderedTableResult( "SELECT * FROM XCORR(DATA => " + "(SELECT time, factory_id, device_id, s1, s1 AS s2 FROM table5) " - + "PARTITION BY (factory_id, device_id) ORDER BY time, TIMECOL => 'time') " - + "ORDER BY factory_id, device_id", + + "PARTITION BY (factory_id, device_id) ORDER BY time, TIMECOL => 'time')", expectedHeader, - retArray, - DATABASE_NAME); + retArray); } @Test @@ -1706,4 +1702,18 @@ private static void tableResultSetEqualWithTolerance( fail(e.getMessage()); } } + + private static void assertUnorderedTableResult( + String sql, String[] expectedHeader, String[] retArray) { + String header = String.join(",", expectedHeader) + ","; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, resultSet, header, Arrays.asList(retArray)); + } + } catch (SQLException e) { + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java index 9db3acb13e098..5fd2ef8dd8ee7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java @@ -129,12 +129,12 @@ public void testMySplit() { "2,2,", "4,4,", }; tableResultSetEqualTest( - "select * from TABLE(SPLIT('1,2,4,5')) a(o1) join TABLE(SPLIT('2、3、4', '、')) b(o2) on a.o1=b.o2", + "select * from TABLE(SPLIT('1,2,4,5')) a(o1) join TABLE(SPLIT('2、3、4', '、')) b(o2) on a.o1=b.o2 ORDER BY o1", expectedHeader, retArray, DATABASE_NAME); tableResultSetEqualTest( - "select * from SPLIT('1,2,4,5') a(o1) join TABLE(SPLIT('2、3、4', '、')) b(o2) on a.o1=b.o2", + "select * from SPLIT('1,2,4,5') a(o1) join TABLE(SPLIT('2、3、4', '、')) b(o2) on a.o1=b.o2 ORDER BY o1", expectedHeader, retArray, DATABASE_NAME); @@ -329,10 +329,10 @@ public void testPassThroughPartitionColumn() { String[] expectedHeader = new String[] {"s1", "device_id"}; String[] retArray = new String[] { - "1,d0,", "null,d0,", "3,d0,", "4,d1,", + "1,d0,", "3,d0,", "null,d0,", "4,d1,", }; tableResultSetEqualTest( - "select * from MY_SELECT(vehicle PARTITION BY device_id, 's1') ORDER BY device_id", + "select * from MY_SELECT(vehicle PARTITION BY device_id, 's1') ORDER BY device_id, s1", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java index 08e9ccc8635ad..03166ce0fc9aa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java @@ -111,7 +111,8 @@ public void testDatetimeInputFormat() { } try (ResultSet resultSet = - statement.executeQuery("SELECT time, s1 FROM table1 where device_id='d1'")) { + statement.executeQuery( + "SELECT time, s1 FROM table1 where device_id='d1' ORDER BY time")) { Assert.assertNotNull(resultSet); int cnt = 0; while (resultSet.next()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java index 3a882b7118c47..89ed30f43c977 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java @@ -141,7 +141,7 @@ public void selectTest() { "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", }; tableResultSetEqualTest( - "select * from vehicle order by device_id", expectedHeader, retArray, DATABASE_NAME); + "select * from vehicle order by device_id, time", expectedHeader, retArray, DATABASE_NAME); } @Test @@ -208,7 +208,7 @@ public void selectWithDuplicatedPathsTest() { "1970-01-01T00:00:01.000Z,d1,888,888,null,", }; tableResultSetEqualTest( - "select time, device_id, s0,s0,s1 from vehicle where device_id = 'd0' or device_id = 'd1' order by device_id", + "select time, device_id, s0,s0,s1 from vehicle where device_id = 'd0' or device_id = 'd1' order by device_id, time", expectedHeader, retArray, DATABASE_NAME); @@ -272,7 +272,7 @@ public void selectDifferentSeriesWithValueFilterWithoutCacheTest() { "1999-12-31T16:00:00.000Z,d0,null,", }; tableResultSetEqualTest( - "select time, device_id, s0 from vehicle where s1 < 200 order by device_id", + "select time, device_id, s0 from vehicle where s1 < 200 order by device_id, time", expectedHeader, retArray, DATABASE_NAME); @@ -530,7 +530,7 @@ public void selectWithRegularExpressionTest() { "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", }; tableResultSetEqualTest( - "select * from vehicle where device_id like 'd%' order by device_id", + "select * from vehicle where device_id like 'd%' order by device_id, time", expectedHeader, retArray, DATABASE_NAME); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java index 5c9ade9530b52..db0a7364867aa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java @@ -228,7 +228,7 @@ public void overFlowTest() { } tableResultSetEqualTest( - "SELECT time, value FROM overflow", expectedHeader, retArray, DATABASE_NAME); + "SELECT time, value FROM overflow ORDER BY time", expectedHeader, retArray, DATABASE_NAME); } // ORDER BY DEVICE diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java index 2eaa96e5b85ee..2d64f67dd4dd6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java @@ -102,7 +102,7 @@ public void testNewTransformerIgnoreNull() { "1970-02-27T20:53:20.001Z,3.0,null," }; tableResultSetEqualTest( - "select time,diff(s1), diff(s2) from table1 where device_id='d1'", + "select time,diff(s1), diff(s2) from (select * from table1 where device_id='d1' order by time) order by time", expectedHeader, retArray, DATABASE_NAME); @@ -123,7 +123,7 @@ public void testNewTransformerRespectNull() { "1970-02-27T20:53:20.001Z,null,null," }; tableResultSetEqualTest( - "select time, Diff(s1, false), diff(s2, false) from table1 where device_id='d1'", + "select time, Diff(s1, false), diff(s2, false) from (select * from table1 where device_id='d1' order by time) order by time", expectedHeader, retArray, DATABASE_NAME); @@ -141,7 +141,7 @@ public void testCaseInSensitive() { "1970-01-01T00:00:00.005Z,5,5.0," }; tableResultSetEqualTest( - "select time, s1, s2 from table1 where device_id='d1' and diff(s1) between 1 and 2", + "select time, s1, s2 from (select * from table1 where device_id='d1' order by time) where diff(s1) between 1 and 2 order by time", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java index 8c0c74bf36a9f..37d63edde2223 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java @@ -99,7 +99,10 @@ public void testRound() { "2023-03-13T08:16:19.764Z,2.0," }; tableResultSetEqualTest( - "select time, round(s1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s1) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -112,7 +115,10 @@ public void testRound() { "2023-03-13T08:16:19.764Z,12345.0," }; tableResultSetEqualTest( - "select time, round(s2) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s2) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -125,7 +131,10 @@ public void testRound() { "2023-03-13T08:16:19.764Z,121.0," }; tableResultSetEqualTest( - "select time, round(s3) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s3) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -138,7 +147,10 @@ public void testRound() { "2023-03-13T08:16:19.764Z,10.0," }; tableResultSetEqualTest( - "select time, round(s4) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s4) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); } @Test @@ -155,7 +167,10 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,2.0," }; tableResultSetEqualTest( - "select time, round(s1, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s1, 1) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -168,7 +183,10 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,12345.0," }; tableResultSetEqualTest( - "select time, round(s2, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s2, 1) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -181,7 +199,10 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,120.5," }; tableResultSetEqualTest( - "select time, round(s3, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s3, 1) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -194,7 +215,10 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,120.51," }; tableResultSetEqualTest( - "select time, round(s3, 2) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s3, 2) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -207,7 +231,10 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,120.511," }; tableResultSetEqualTest( - "select time, round(s3, 3) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s3, 3) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -220,7 +247,10 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,120.5112," }; tableResultSetEqualTest( - "select time, round(s3, 4) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s3, 4) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -233,7 +263,10 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,10.14342," }; tableResultSetEqualTest( - "select time, round(s4, 5) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s4, 5) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -246,7 +279,10 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,10.1," }; tableResultSetEqualTest( - "select time, round(s4, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s4, 1) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); } @Test @@ -264,7 +300,10 @@ public void testRoundWithNegativePlaces() { "2023-03-13T08:16:19.764Z,0.0," }; tableResultSetEqualTest( - "select time, round(s1, -1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s1, -1) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -277,7 +316,10 @@ public void testRoundWithNegativePlaces() { "2023-03-13T08:16:19.764Z,12340.0," }; tableResultSetEqualTest( - "select time, round(s2, -1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s2, -1) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -290,7 +332,10 @@ public void testRoundWithNegativePlaces() { "2023-03-13T08:16:19.764Z,120.0," }; tableResultSetEqualTest( - "select time, round(s3,-1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s3,-1) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); intRetArray = new String[] { @@ -303,7 +348,10 @@ public void testRoundWithNegativePlaces() { "2023-03-13T08:16:19.764Z,10.0," }; tableResultSetEqualTest( - "select time, round(s4, -1) from table1", expectedHeader, intRetArray, DATABASE_NAME); + "select time, round(s4, -1) from table1 ORDER BY time", + expectedHeader, + intRetArray, + DATABASE_NAME); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java index 7697bc81649bd..59499aa5db0de 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTopKRuntimeFilterScanPathIT.java @@ -140,7 +140,8 @@ public void unionAllSiblingTopKOrderByTimeLimit() { tableResultSetEqualTest( "(SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd1' ORDER BY time DESC LIMIT 1)" + " UNION ALL" - + " (SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd3' ORDER BY time DESC LIMIT 1)", + + " (SELECT time, device_id, s1 FROM table1 WHERE device_id = 'd3' ORDER BY time DESC LIMIT 1)" + + " ORDER BY time, device_id, s1", expectedHeader, retArray, TABLE_DATABASE); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java index 4951fd6670a02..fe725976bd0c2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java @@ -31,12 +31,20 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -57,7 +65,7 @@ public static void tearDown() throws Exception { @Test public void testCorrelatedExistsSubqueryInWhereClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: exists and other filter @@ -65,31 +73,29 @@ public void testCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: only exists sql = "SELECT s1 FROM table3 t3 WHERE exists(SELECT s1 from table1 t1 WHERE t1.s1 = t3.s1)"; retArray = new String[] {"30,", "30,", "40,"}; - expectedHeader = new String[] {"s1"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "s1,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); // Test case: exists with distinct sql = "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: aggregation in exists @@ -97,12 +103,11 @@ public void testCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(min(%s) as INT32) as %s FROM table1 t1 WHERE exists(SELECT avg(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s)"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: aggregation with group by in exists(subquery returns empty result with having @@ -111,12 +116,11 @@ public void testCorrelatedExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE exists(SELECT count(*) from table3 t3 where t1.%s = t3.%s group by device_id having count(*) > 5)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: limit 1 in exists @@ -124,61 +128,60 @@ public void testCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s limit 1)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testNestedExistsSubquery() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: Nested exists sql = "select distinct s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 = t3.s1 and exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25))"; retArray = new String[] {"30,"}; - expectedHeader = new String[] {"s1"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "s1,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); // Test case: Nested exists with not(64 - 4 = 60 rows remain) sql = "select count(*) as cnt from table1 t1 where not exists(select s1 from table3 t3 where t1.s1 = t3.s1 and exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25))"; retArray = new String[] {"60,"}; - expectedHeader = new String[] {"cnt"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "cnt,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); } @Test public void testMultipleExistsSubquery() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: multiple exists sql = "select distinct s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 = t3.s1) and exists(select s1 from table2 t2 where t2.s1 = t1.s1 - 25)"; retArray = new String[] {"30,"}; - expectedHeader = new String[] {"s1"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "s1,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); // Test case: multiple exists with not sql = "select distinct s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 = t3.s1) and not exists(select s1 from table2 t2 where t2.s1 = t1.s1)"; retArray = new String[] {"30,", "40,"}; - expectedHeader = new String[] {"s1"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "s1,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); } @Test public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubquery() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: with InPredicate @@ -186,8 +189,8 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and %s in (select %s from table3)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -198,8 +201,7 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with InPredicate in exists @@ -207,8 +209,8 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s not in (select %s from table2 where %s is not null))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -220,8 +222,7 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with Scalar Subquery @@ -229,13 +230,12 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 > (select min(%s) from table3)"; retArray = new String[] {"40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with Scalar Subquery in exists @@ -243,13 +243,12 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and s1 = (select min(%s) from table3))"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with Quantified Comparison @@ -257,13 +256,12 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 != any(select %s from table2)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with Quantified Comparison in exists @@ -271,8 +269,8 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s = any (select %s from table3))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -283,15 +281,14 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testCorrelatedExistsSubqueryWithMultipleCorrelation() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: Multiple correlation in exists @@ -299,91 +296,89 @@ public void testCorrelatedExistsSubqueryWithMultipleCorrelation() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE t1.%s = t3.%s and t1.s1 = t3.s1 and t1.s2 = t3.s2)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } sql = "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and not exists(SELECT (%s) from table3 t3 WHERE t1.%s = t3.%s and t1.s1 = t3.s1 and t1.s2 = t3.s2)"; retArray = new String[] {"50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testCorrelatedExistsSubqueryInHavingClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: exists in having sql = "SELECT device_id, count(*) from table1 t1 group by device_id having exists(SELECT 1 from table3 t3 where t3.device_id = t1.device_id)"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] {"d01,5,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); } sql = "SELECT device_id, count(*) from table1 t1 group by device_id having exists(SELECT 1 from table3 t3 where t3.device_id != 'd01' and t3.device_id = t1.device_id)"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: not exists in having sql = "SELECT device_id, count(*) from table1 t1 group by device_id having count(*) = 5 and not exists(SELECT 1 from table3 t3 where t3.device_id = t1.device_id)"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] {"d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); } } @Test public void testCorrelatedExistsSubqueryInSelectClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: exists in Select clause sql = "select exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3 where exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25)"; retArray = new String[] {"true,", "true,"}; - expectedHeader = new String[] {"_col0"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "_col0,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); sql = "select exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3"; retArray = new String[] {"true,", "true,", "true,", "false,"}; - expectedHeader = new String[] {"_col0"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "_col0,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); sql = "select not exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3 where exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25)"; retArray = new String[] {"false,", "false,"}; - expectedHeader = new String[] {"_col0"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "_col0,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); sql = "select not exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3"; retArray = new String[] {"false,", "false,", "false,", "true,"}; - expectedHeader = new String[] {"_col0"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "_col0,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); } @Test @@ -428,4 +423,18 @@ public void testCorrelatedExistsLegalityCheck() { "Decorrelation for LIMIT with row count greater than 1 is not supported yet", DATABASE_NAME); } + + private static void assertUnorderedResult( + String sql, String expectedHeader, List expectedRows) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java index c4c41f2e196d7..5c233cdd6fbd8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java @@ -30,12 +30,20 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -57,80 +65,77 @@ public static void tearDown() throws Exception { @Test public void testCorrelatedQuantifiedComparisonInWhereClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; sql = "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= any(SELECT %s from table3 t3 where t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } sql = "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= some(SELECT %s from table3 t3 where t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } sql = "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= all(SELECT %s from table3 t3 where t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testQuantifiedComparisonInSelectClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; sql = "SELECT %s >= any(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.s1 = t3.s1) from table1 t1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } sql = "SELECT %s >= some(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.s1 = t3.s1) from table1 t1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } sql = "SELECT %s >= all(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.s1 = t3.s1) from table1 t1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"true,", "true,", "true,", "true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } } @@ -150,4 +155,18 @@ public void testCorrelatedQuantifiedComparisonLegalityCheck() { "Decorrelation for LIMIT with row count greater than 1 is not supported yet", DATABASE_NAME); } + + private static void assertUnorderedResult( + String sql, String expectedHeader, List expectedRows) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java index 071635f53f6c8..91a7e99f7752f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java @@ -31,12 +31,20 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -58,7 +66,7 @@ public static void tearDown() throws Exception { @Test public void testCorrelatedScalarSubqueryInWhereClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: Aggregation with correlated filter in scalar subquery @@ -66,13 +74,12 @@ public void testCorrelatedScalarSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT max(%s) from table3 t3 where t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: Non-Aggregation with correlated filter in scalar subquery @@ -80,8 +87,8 @@ public void testCorrelatedScalarSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT distinct %s from table3 t3 where t1.%s = t3.%s and %s > 30)"; retArray = new String[] {"40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -92,8 +99,7 @@ public void testCorrelatedScalarSubqueryInWhereClause() { measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: limit 1 in scalar subquery @@ -101,8 +107,8 @@ public void testCorrelatedScalarSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT %s from table3 t3 where t1.%s = t3.%s and %s > 30 limit 1)"; retArray = new String[] {"40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -113,51 +119,50 @@ public void testCorrelatedScalarSubqueryInWhereClause() { measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testNestedCorrelatedScalarSubquery() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: Nested exists sql = "select distinct s1 from table1 t1 where s1 >= (select max(s1) from table3 t3 where t1.s1 = t3.s1 and s1 = (select max(s1) from table1 t1_2 where t1_2.s1 = t3.s1))"; retArray = new String[] {"30,", "40,"}; - expectedHeader = new String[] {"s1"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "s1,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); } @Test public void testMultipleScalarSubquery() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: multiple scalar subquery sql = "select distinct s1 from table1 t1 where s1 = (select max(s1) from table3 t3 where t1.s1 = t3.s1) and s1 = (select min(s1) from table3 t3 where t1.s1 = t3.s1)"; retArray = new String[] {"30,", "40,"}; - expectedHeader = new String[] {"s1"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "s1,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); } @Test public void testCorrelatedScalarSubqueryInWhereClauseWithOtherCorrelatedSubquery() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: with Exists sql = "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and exists(select s1 from table3 t3 where t1.s1 = t3.s1)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -169,15 +174,14 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherCorrelatedSubquery measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubquery() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: with In Predicate @@ -185,8 +189,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and %s in (select %s from table3)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -198,8 +202,7 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with InPredicate in scalar subquery @@ -207,8 +210,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s not in (select %s from table2 where %s is not null))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -221,8 +224,7 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with Scalar Subquery @@ -230,8 +232,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 > (select min(%s) from table3)"; retArray = new String[] {"40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -242,8 +244,7 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with nested Scalar Subquery @@ -251,8 +252,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and s1 = (select min(%s) from table3))"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -263,8 +264,7 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with Quantified Comparison @@ -272,8 +272,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 != any(select %s from table2)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -284,8 +284,7 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with Quantified Comparison in scalar subquery @@ -293,8 +292,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s = any (select %s from table3))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -306,15 +305,14 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testCorrelatedScalarSubqueryWithMultipleCorrelation() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: Multiple correlation in exists @@ -322,50 +320,49 @@ public void testCorrelatedScalarSubqueryWithMultipleCorrelation() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE t1.%s = t3.%s and t1.s1 = t3.s1 and t1.s2 = t3.s2)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testCorrelatedScalarSubqueryInHavingClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: scalar subquery in having sql = "SELECT device_id, count(*) from table1 t1 group by device_id having count(*) + 35 = (SELECT max(s1) from table3 t3 where t3.device_id = t1.device_id)"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] {"d01,5,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); } } @Test public void testCorrelatedScalarSubqueryInSelectClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: exists in Select clause sql = "select s1 = (select max(s1) from table1 t1 where t1.s1 = t3.s1) from table3 t3 where exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25)"; retArray = new String[] {"true,", "true,"}; - expectedHeader = new String[] {"_col0"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "_col0,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); sql = "select (select max(s1) from table1 t1 where t1.s1 = t3.s1) from table3 t3"; retArray = new String[] {"30,", "30,", "40,", "null,"}; - expectedHeader = new String[] {"_col0"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = "_col0,"; + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); } @Test @@ -417,4 +414,18 @@ public void testCorrelatedScalarSubqueryLegalityCheck() { "701: Scalar sub-query has returned multiple rows", DATABASE_NAME); } + + private static void assertUnorderedResult( + String sql, String expectedHeader, List expectedRows) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java index 1f424347e0722..ea9a7235d527d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java @@ -30,11 +30,19 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; -import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -55,7 +63,7 @@ public static void tearDown() throws Exception { @Test public void testUnCorrelatedExistsSubqueryInWhereClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: aggregation in exist subquery @@ -63,9 +71,9 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + expectedHeader = measurement + ","; + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: group by in exist subquery @@ -73,9 +81,9 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1 group by device_id)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + expectedHeader = measurement + ","; + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: expression in select clause with exist subquery @@ -83,9 +91,9 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) + 1 as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1 group by device_id)"; retArray = new String[] {"31,", "41,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + expectedHeader = measurement + ","; + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: exists with other conjuncts @@ -93,12 +101,11 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE %s = 30 and exists(select max(s1) from table1)"; retArray = new String[] {"30,", "30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: false exist @@ -106,12 +113,11 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE %s = 30 and exists(select s1 from table1 where s1 < 0)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: limit in exist subquery @@ -119,12 +125,11 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select s1 from table1 limit 10)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: Nested exists @@ -132,12 +137,11 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select s1 from table1 where exists(select s1 from table2))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: Multiple exists @@ -145,19 +149,18 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select avg(s1) from table1 limit 1) and exists(select s1 from table1 where exists(select s1 from table2))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testUnCorrelatedNotExistsSubqueryInWhereClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: aggregation in exist subquery @@ -165,9 +168,9 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and not exists(select max(s1) from table1)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + expectedHeader = measurement + ","; + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: limit in exist subquery @@ -175,9 +178,9 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE not exists(select s1 from table1 limit 1)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + expectedHeader = measurement + ","; + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: subquery has output with not exists @@ -185,12 +188,11 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE %s = 30 and not exists(select s1 from table1 where s1 < 0)"; retArray = new String[] {"30,", "30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with distinct @@ -198,12 +200,11 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and not exists(select s1 from table1 where s1 < 0)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: Nested exists @@ -211,12 +212,11 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select s1 from table1 where not exists(select s1 from table1 where s1 < 0))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: Multiple exists @@ -224,19 +224,18 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and not exists(select s1 from table1 where s1 < 0) and exists(select s1 from table1 where not exists(select s1 from table1 where s1 < 0))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: with InPredicate @@ -244,9 +243,9 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(select max(s1) from table1) and s1 in (select s1 from table3)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + expectedHeader = measurement + ","; + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: with InPredicate in exists @@ -254,9 +253,9 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1 where s1 in (select s1 from table3))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + expectedHeader = measurement + ","; + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: with Scalar Subquery @@ -264,12 +263,11 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(select s1 from table1) and s1 = (select min(s1) from table3)"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with Scalar Subquery in exists @@ -277,12 +275,11 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s < 50 and exists(select s1 from table1 where s1 = (select min(s1) from table3))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with Quantified Comparison @@ -290,12 +287,11 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(select s1 from table1) and s1 = any(select s1 from table3)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: with Quantified Comparison in exists @@ -303,70 +299,69 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = 30 and exists(select s1 from table1 where s1 = any (select s1 from table3))"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testUnCorrelatedExistsSubqueryInHavingClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: exists in having sql = "SELECT device_id, count(*) from table1 group by device_id having exists(SELECT %s from table1) and count(*) = 5"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); } sql = "SELECT device_id, count(*) from table1 group by device_id having exists(SELECT %s from table1 where s1 < 0)"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: not exists in having sql = "SELECT device_id, count(*) from table1 group by device_id having not exists(SELECT %s from table1 where s1 < 0) and count(*) = 5"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); } sql = "SELECT device_id, count(*) from table1 group by device_id having not exists(SELECT %s from table1) and count(*) = 5"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); } } @Test public void testUnCorrelatedExistsSubqueryInSelectClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: exists in Select clause @@ -374,18 +369,32 @@ public void testUnCorrelatedExistsSubqueryInSelectClause() { "SELECT exists(select max(s1) from table1) as %s FROM table3 t3 WHERE %s is not null and device_id = 'd01'"; retArray = new String[] {"true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + expectedHeader = measurement + ","; + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } sql = "SELECT not exists(select max(s1) from table1) as %s FROM table3 t3 WHERE %s is not null and device_id = 'd01'"; retArray = new String[] {"false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + expectedHeader = measurement + ","; + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + } + } + + private static void assertUnorderedResult( + String sql, String expectedHeader, List expectedRows) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java index add410b3bee17..9144eb49a4a32 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -30,12 +30,20 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -56,7 +64,7 @@ public static void tearDown() throws Exception { @Test public void testInPredicateSubqueryInWhereClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: where s in (subquery) @@ -64,12 +72,11 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s not in (subquery) @@ -77,12 +84,11 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {"50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s in (subquery), subquery returns empty set @@ -90,12 +96,11 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s not in (subquery), subquery returns empty set. Should return all rows @@ -103,12 +108,11 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s in (subquery), subquery contains scalar subquery. @@ -116,12 +120,11 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT min(%s) from table3 WHERE device_id = 'd01')"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s in (subquery), subquery contains scalar subquery. @@ -129,13 +132,12 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01' and (%s) > (select avg(%s) from table3 where device_id = 'd01'))"; retArray = new String[] {"40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s in (subquery), subquery contains expression. @@ -143,117 +145,116 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s AS INT32) in (SELECT cast(((%s) + 30) AS INT32) from table3 WHERE device_id = 'd01')"; retArray = new String[] {"60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s in (subquery), s contains null value sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 in (SELECT s1 from table3)"; - expectedHeader = new String[] {"s1"}; + expectedHeader = "s1,"; retArray = new String[] {"30,"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); // Test case: where s not in (subquery), s contains null value, the resutl should be empty sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 not in (SELECT s1 from table3)"; - expectedHeader = new String[] {"s1"}; + expectedHeader = "s1,"; retArray = new String[] {}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); } @Test public void testInPredicateSubqueryInHavingClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: having s in (subquery) sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); // Test case: having s not in (subquery) sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 30 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01') and count(*) > 3"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); // Test case: having s in (subquery), subquery returns empty set sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd010')"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] {}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); // Test case: having s not in (subquery), subquery returns empty set, should return all rows sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd11') and count(*) > 3"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); } @Test public void testInPredicateSubqueryInSelectClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: select s in (subquery) sql = "SELECT %s in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: select s not in (subquery) sql = "SELECT %s not in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"false,", "false,", "true,", "true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: select s in (subquery), s contains null value. The result should also be null when // s is null. sql = "SELECT s1 in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"true,", "null,"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); // Test case: select s in (subquery), s contains null value. The result should also be null when // s is null. sql = "SELECT s1 not in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"false,", "null,"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); // Test case: select s in (subquery), s contains null value. 36 not in(30, 40, null) returns // null sql = "SELECT s1 not in (SELECT s1 from table3) from table1 where device_id = 'd02'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"null,", "false,", "null,"}; - tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); } @Test @@ -292,4 +293,18 @@ public void testInPredicateSubqueryLegalityCheck() { tableAssertTestFail( "select s1 from table1 where s1 in (select s1 from)", "mismatched input", DATABASE_NAME); } + + private static void assertUnorderedResult( + String sql, String expectedHeader, List expectedRows) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java index 11eb1feab9014..ff4a091ab99ed 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java @@ -30,12 +30,20 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -57,7 +65,7 @@ public static void tearDown() throws Exception { @Test public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: where s > any (subquery), s does not contain null value @@ -65,12 +73,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s > some (subquery), s does not contain null value @@ -78,12 +85,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s >= any (subquery), s does not contain null value @@ -91,12 +97,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s >= some (subquery), s does not contain null value @@ -104,12 +109,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s < any (subquery), s does not contain null value @@ -117,12 +121,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where < some (subquery), s does not contain null value @@ -130,12 +133,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s <= any (subquery), s does not contain null value @@ -143,12 +145,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s <= some (subquery), s does not contain null value @@ -156,12 +157,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s = any (subquery), s does not contain null value @@ -169,12 +169,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s = some (subquery), s does not contain null value @@ -182,12 +181,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s != any (subquery), s does not contain null value @@ -195,12 +193,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s != some (subquery), s does not contain null value @@ -208,19 +205,18 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testAllComparisonInWhereClauseWithoutNull() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: where s > all (subquery), s does not contain null value @@ -228,12 +224,11 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {"50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s >= all (subquery), s does not contain null value @@ -241,12 +236,11 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {"40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s < all (subquery), s does not contain null value @@ -254,12 +248,11 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s <= all (subquery), s does not contain null value @@ -267,12 +260,11 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s = all (subquery), s does not contain null value @@ -280,12 +272,11 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s != all (subquery), s does not contain null value @@ -293,19 +284,18 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {"50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testAnyAndSomeComparisonInWhereClauseWithNull() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: where s1 > any (subquery), s1 in table3 contains null value @@ -313,12 +303,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > any (SELECT s1 FROM table3)"; retArray = new String[] {"40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s1 > some (subquery), s1 in table3 contains null value @@ -326,12 +315,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > some (SELECT s1 FROM table3)"; retArray = new String[] {"40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s1 >= any (subquery), s1 in table3 contains null value @@ -339,12 +327,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= any (SELECT s1 FROM table3)"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s1 >= some (subquery), s1 in table3 contains null value @@ -352,12 +339,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= some (SELECT s1 FROM table3)"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s1 < any (subquery), s1 in table3 contains null value @@ -365,12 +351,11 @@ public void testAnyAndSomeComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < any (SELECT s1 FROM table3)"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s1 < some (subquery), s1 in table3 contains null value @@ -378,19 +363,18 @@ public void testAnyAndSomeComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < some (SELECT s1 FROM table3)"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testAllComparisonInWhereClauseWithNull() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: where s1 > all (subquery), s1 in table3 contains null value. @@ -398,12 +382,11 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s1 >= all (subquery), s1 in table3 contains null value. @@ -411,12 +394,11 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s1 < all (subquery), s1 in table3 contains null value. @@ -424,12 +406,11 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s1 <= all (subquery), s1 in table3 contains null value. @@ -437,12 +418,11 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s1 = all (subquery), s1 in table3 contains null value. @@ -450,12 +430,11 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: where s1 != all (subquery), s1 in table3 contains null value. @@ -463,191 +442,184 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s as INT32) != all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testQuantifiedComparisonInWhereWithExpression() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; sql = "SELECT cast(%s + 10 AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s + 10 > any (SELECT %s + 10 FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"50,", "60,", "70,", "80,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } sql = "SELECT cast(%s + 10 AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s + 10 > some (SELECT %s + 10 FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"50,", "60,", "70,", "80,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } sql = "SELECT cast(%s + 10 AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s + 10 >= all (SELECT %s + 10 FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"80,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testQuantifiedComparisonInHavingClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: having s >= any(subquery) sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 >= any(SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: having s >= some(subquery) sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 >= some(SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: having s >= all(subquery) sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 35 >= all(SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } public void testQuantifiedComparisonInSelectClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: select s > any(subquery) sql = "SELECT %s > any(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"false,", "true,", "true,", "true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: select s > some(subquery) sql = "SELECT %s > some(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"false,", "true,", "true,", "true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: select s > all(subquery) sql = "SELECT %s > all(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"false,", "false,", "false,", "false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: select s < any(subquery), subquery contains null value sql = "SELECT %s < any(SELECT (%s) from table3) from table1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"null,", "null,", "null,", "null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: select s < some(subquery), subquery contains null value sql = "SELECT %s < some(SELECT (%s) from table3) from table1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"null,", "null,", "null,", "null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: select s <= any(subquery), subquery contains null value sql = "SELECT %s <= any(SELECT (%s) from table3) from table1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"true,", "null,", "null,", "null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: select s <= some(subquery), subquery contains null value sql = "SELECT %s <= some(SELECT (%s) from table3) from table1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"true,", "null,", "null,", "null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: select s != all(subquery), subquery contains null value sql = "SELECT %s != all(SELECT (%s) from table3) from table1 where device_id = 'd01'"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"false,", "false,", "null,", "null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } // Test case: select s != all(subquery), subquery result contains null value and s not in @@ -655,11 +627,11 @@ public void testQuantifiedComparisonInSelectClause() { // value result set sql = "SELECT %s != all(SELECT (%s) from table3 where device_id = 'd_null') from table1 where device_id = 'd02' and %s != 30"; - expectedHeader = new String[] {"_col0"}; + expectedHeader = "_col0,"; retArray = new String[] {"null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( - String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + assertUnorderedResult( + String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); } } @@ -671,4 +643,18 @@ public void testQuantifiedComparisonLegalityCheck() { "mismatched input", DATABASE_NAME); } + + private static void assertUnorderedResult( + String sql, String expectedHeader, List expectedRows) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java index f6a2cdff48b3c..efa8327eafb3b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java @@ -30,12 +30,20 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; + +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; +import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -58,7 +66,7 @@ public static void tearDown() throws Exception { @Test public void testScalarSubqueryAfterComparisonInOneTable() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: s equals to the maximum value of s in table1 @@ -66,12 +74,11 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 WHERE device_id = 'd01')"; retArray = new String[] {"70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: s not equals to the maximum value of s in table1 @@ -79,12 +86,11 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != ((SELECT max(%s) FROM table1 WHERE device_id = 'd01'))"; retArray = new String[] {"30,", "40,", "50,", "60,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: s greater than the average value of s in table1 @@ -92,12 +98,11 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= ((SELECT AVG(%s) FROM table1 WHERE device_id = 'd01'))"; retArray = new String[] {"50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: s greater than the max value of s in table1 @@ -105,12 +110,11 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > ((SELECT max(%s) FROM table1 WHERE device_id = 'd01'))"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: s is less than the maximum value of s in table1 and greater than the minimum value @@ -119,13 +123,12 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < (SELECT max(%s) from table1 WHERE device_id = 'd01') and %s > (SELECT min(%s) from table1 WHERE device_id = 'd01') "; retArray = new String[] {"40,", "50,", "60,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: s greater than the avg value of s in table1 and s5 = true @@ -133,12 +136,11 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > ((SELECT avg(%s) FROM table1 WHERE device_id = 'd01' and s5 = true))"; retArray = new String[] {"60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: s greater than the count value of s in table1 @@ -146,12 +148,11 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > (SELECT count(%s) FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: s less than the sum value of s in table1 @@ -159,12 +160,11 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < (SELECT sum(%s) FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: subquery is not aggregation but returns exactly one row @@ -172,20 +172,19 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT %s FROM table1 WHERE device_id = 'd01' and %s = 30)"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testScalarSubqueryAfterComparisonInDifferentTables() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: s greater than the count value of s in table2 @@ -193,12 +192,11 @@ public void testScalarSubqueryAfterComparisonInDifferentTables() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > (SELECT count(%s) from table2)"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: s less than the max value of s in table2 * the count value of s in table2 * 10 @@ -206,19 +204,18 @@ public void testScalarSubqueryAfterComparisonInDifferentTables() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < ((SELECT max(%s) from table2) * (SELECT count(%s) from table2)) * 10"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @Test public void testNestedScalarSubqueryAfterComparison() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: nested scalar subquery in where clause @@ -226,13 +223,12 @@ public void testNestedScalarSubqueryAfterComparison() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 where %s = (SELECT max(%s) from table1 WHERE device_id = 'd01'))"; retArray = new String[] {"70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: nested scalar subquery with table subquery @@ -240,8 +236,8 @@ public void testNestedScalarSubqueryAfterComparison() { "SELECT %s from (SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 where %s = (SELECT max(%s) from table1 WHERE device_id = 'd01')))"; retArray = new String[] {"70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format( sql, measurement, @@ -252,8 +248,7 @@ public void testNestedScalarSubqueryAfterComparison() { measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @@ -293,7 +288,7 @@ public void testScalarSubqueryAfterComparisonLegalityCheck() { @Test public void testScalarSubqueryInSelectClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: select scalar subquery as one constant column @@ -301,12 +296,11 @@ public void testScalarSubqueryInSelectClause() { "SELECT cast((SELECT max(%s) from table1 where device_id = 'd01') AS INT32) as %s from table1 where device_id = 'd01'"; retArray = new String[] {"70,", "70,", "70,", "70,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: scalar subquery in arithmetic expression and function expression @@ -314,24 +308,22 @@ public void testScalarSubqueryInSelectClause() { "SELECT cast((s1 + (SELECT max(%s) from table1 where device_id = 'd01')) AS INT32) as %s from table1 where device_id = 'd01'"; retArray = new String[] {"100,", "110,", "120,", "130,", "140,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } sql = "SELECT cast(((SELECT max(%s) from table1 where device_id = 'd01') + (SELECT max(%s) from table1 where device_id = 'd01')) AS INT32) as %s from table1 where device_id = 'd01'"; retArray = new String[] {"140,", "140,", "140,", "140,", "140,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: scalar subquery in udf expression @@ -339,24 +331,22 @@ public void testScalarSubqueryInSelectClause() { "SELECT cast(floor(sin((SELECT max(%s) from table1 where device_id = 'd01'))) AS INT32) as %s from table1 where device_id = 'd01'"; retArray = new String[] {"0,", "0,", "0,", "0,", "0,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } // Test case: scalar subquery in aggregation function sql = "SELECT count((select count(s1) from table1)) as %s from table1"; retArray = new String[] {"64,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = new String[] {measurement}; - tableResultSetEqualTest( + expectedHeader = measurement + ","; + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } @@ -366,24 +356,38 @@ public void testScalarSubqueryInSelectClause() { @Test public void testScalarSubqueryInHavingClause() { String sql; - String[] expectedHeader; + String expectedHeader; String[] retArray; // Test case: scalar subquery in having clause sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) > 3 + (SELECT count(*) from table2 where device_id = 'd01')"; - expectedHeader = new String[] {"device_id", "_col1"}; + expectedHeader = "device_id,_col1,"; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; for (String measurement : NUMERIC_MEASUREMENTS) { - tableResultSetEqualTest( + assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - retArray, - DATABASE_NAME); + Arrays.asList(retArray)); } } + // endregion + + private static void assertUnorderedResult( + String sql, String expectedHeader, List expectedRows) { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java index ca92a1e1de6b6..7d9b4be5a93aa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java @@ -162,7 +162,7 @@ public void selectTest() { "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", }; tableResultSetEqualTest( - "select * from vehicle order by device_id", expectedHeader, retArray, DATABASE_NAME); + "select * from vehicle order by device_id, time", expectedHeader, retArray, DATABASE_NAME); } @Test @@ -224,7 +224,7 @@ public void selectWithDuplicatedPathsTest() { "1970-01-01T00:00:01.000Z,d1,888,888,null,", }; tableResultSetEqualTest( - "select time, device_id, s0,s0,s1 from vehicle where device_id = 'd0' or device_id = 'd1' order by device_id", + "select time, device_id, s0,s0,s1 from vehicle where device_id = 'd0' or device_id = 'd1' order by device_id, time", expectedHeader, retArray, DATABASE_NAME); @@ -289,7 +289,7 @@ public void selectDifferentSeriesWithValueFilterWithoutCacheTest() { "1999-12-31T16:00:00.000Z,d0,null,", }; tableResultSetEqualTest( - "select time, device_id, s0 from vehicle where s1 < 200 order by device_id", + "select time, device_id, s0 from vehicle where s1 < 200 order by device_id, time", expectedHeader, retArray, DATABASE_NAME); @@ -547,7 +547,7 @@ public void selectWithRegularExpressionTest() { "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", }; tableResultSetEqualTest( - "select * from vehicle where device_id like 'd%' order by device_id", + "select * from vehicle where device_id like 'd%' order by device_id, time", expectedHeader, retArray, DATABASE_NAME); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java index 8dbf6b892f01e..e4d99ec0a442a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java @@ -249,7 +249,7 @@ public void overFlowTest() { } tableResultSetEqualTest( - "SELECT time, value FROM overflow", expectedHeader, retArray, DATABASE_NAME); + "SELECT time, value FROM overflow ORDER BY time", expectedHeader, retArray, DATABASE_NAME); } // ORDER BY DEVICE diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java index e3c01f5e6d4ce..64ed2e54f51ad 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java @@ -581,7 +581,7 @@ public void countIfTest() { "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d16,0,", }; tableResultSetEqualTest( - "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id", + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id ORDER BY device_id, 1", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java index dbe22b168f1fd..b451beb6aef6d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java @@ -581,7 +581,7 @@ public void countIfTest() { "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d16,0,", }; tableResultSetEqualTest( - "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id", + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id ORDER BY device_id, 1", expectedHeader, retArray, DATABASE_NAME); diff --git a/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java b/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java index 33976ed731f22..1b1c57fe8abc3 100644 --- a/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java +++ b/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java @@ -581,8 +581,4 @@ private CalcMessages() {} public static final String EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_ARG_ARGUMENTS_BUT_GOT_ARG_D6249DD2 = "Aggregate function [%s] requires %d arguments, but got %d"; - - public static final String - EXCEPTION_READY_INPUT_CONSUMPTION_MUST_BE_ENABLED_BEFORE_COLLECT_EXECUTION_STARTS_5FAF2DC4 = - "Ready input consumption must be enabled before Collect execution starts"; } diff --git a/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java b/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java index 19a56413d8b0f..06ac8d62cbbfd 100644 --- a/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java +++ b/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java @@ -558,7 +558,4 @@ private CalcMessages() {} public static final String EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_ARG_ARGUMENTS_BUT_GOT_ARG_D6249DD2 = "聚合函数 [%s] 需要 %d 个参数，但实际传入 %d 个"; - public static final String - EXCEPTION_READY_INPUT_CONSUMPTION_MUST_BE_ENABLED_BEFORE_COLLECT_EXECUTION_STARTS_5FAF2DC4 = - "必须在 Collect 开始执行前启用就绪输入消费"; } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java index c1ef2edb1b619..72058f3cfd3c1 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AnyChildBlocked.java @@ -185,10 +185,6 @@ void invalidateChild(int childId) { } } - void finishChild(int childId) { - invalidateChild(childId); - } - void close() { WaitRound round; synchronized (lock) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java index e94139ff0cdc2..efcc56677856f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java @@ -21,7 +21,6 @@ import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; import org.apache.iotdb.calc.execution.operator.Operator; -import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.commons.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.commons.utils.TestOnly; @@ -31,20 +30,20 @@ import java.util.List; -import static com.google.common.base.Preconditions.checkState; import static com.google.common.util.concurrent.Futures.getDone; import static com.google.common.util.concurrent.Futures.nonCancellationPropagating; +/** + * Consumes whichever direct child is ready first. Output rows are ordered per child but interleaved + * across children, so the plan must not rely on this operator for any cross-child ordering. + */ public class CollectOperator implements ProcessOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(CollectOperator.class); private final CommonOperatorContext operatorContext; protected final List children; - private AnyChildBlocked anyChildBlocked; - private boolean inited; - private boolean executionStarted; - protected int currentIndex; + private final AnyChildBlocked anyChildBlocked; private ListenableFuture selectionFuture; private ListenableFuture exposedFuture; private int remainingChildren; @@ -53,41 +52,17 @@ public class CollectOperator implements ProcessOperator { public CollectOperator(CommonOperatorContext operatorContext, List children) { this.operatorContext = operatorContext; this.children = children; + this.anyChildBlocked = new AnyChildBlocked(children); this.remainingChildren = children.size(); } - /** - * Enable only after the complete table FI has passed eligibility checks, before drivers start. - */ - public void enableReadyInputConsumption() { - if (anyChildBlocked == null) { - checkState( - !executionStarted, - CalcMessages - .EXCEPTION_READY_INPUT_CONSUMPTION_MUST_BE_ENABLED_BEFORE_COLLECT_EXECUTION_STARTS_5FAF2DC4); - anyChildBlocked = new AnyChildBlocked(children); - } - } - @Override public boolean hasNext() { - return anyChildBlocked == null - ? currentIndex < children.size() - : !closed && remainingChildren > 0; + return !closed && remainingChildren > 0; } @Override public TsBlock next() throws Exception { - executionStarted = true; - if (anyChildBlocked == null) { - if (children.get(currentIndex).hasNextWithTimer()) { - TsBlock block = children.get(currentIndex).nextWithTimer(); - return block == null ? null : transformBlock(currentIndex, block); - } - closeCurrentChild(currentIndex); - currentIndex++; - return null; - } anyChildBlocked.checkFailure(); if (!hasNext()) { return null; @@ -127,17 +102,10 @@ protected TsBlock transformBlock(int childId, TsBlock block) { } protected void closeCurrentChild(int index) throws Exception { - if (anyChildBlocked == null) { - children.get(index).close(); - children.set(index, null); - return; - } Operator child = children.get(index); if (child == null) { return; } - // Retire the record even if closing its subtree throws. - anyChildBlocked.finishChild(index); children.set(index, null); remainingChildren--; child.close(); @@ -145,16 +113,6 @@ protected void closeCurrentChild(int index) throws Exception { @Override public ListenableFuture isBlocked() { - executionStarted = true; - if (anyChildBlocked == null) { - if (!inited) { - inited = true; - for (Operator child : children) { - child.isBlocked(); - } - } - return currentIndex >= children.size() ? NOT_BLOCKED : children.get(currentIndex).isBlocked(); - } if (!hasNext()) { return NOT_BLOCKED; } @@ -179,15 +137,6 @@ public CommonOperatorContext getOperatorContext() { @Override public void close() throws Exception { - executionStarted = true; - if (anyChildBlocked == null) { - for (int i = currentIndex; i < children.size(); i++) { - if (children.get(i) != null) { - closeCurrentChild(i); - } - } - return; - } if (closed) { return; } @@ -214,13 +163,6 @@ public void close() throws Exception { @Override public long calculateMaxPeekMemory() { - if (anyChildBlocked == null) { - long maxPeekMemory = 0; - for (Operator child : children) { - maxPeekMemory = Math.max(maxPeekMemory, child.calculateMaxPeekMemoryWithCounter()); - } - return maxPeekMemory; - } long retained = calculateRetainedSizeAfterCallingNext(); long peak = retained; for (Operator child : children) { @@ -249,9 +191,6 @@ public long calculateMaxReturnSize() { @Override public long calculateRetainedSizeAfterCallingNext() { - if (anyChildBlocked == null) { - return 0; - } long retained = 0; for (Operator child : children) { if (child != null) { @@ -269,7 +208,7 @@ public List getChildren() { @Override public long ramBytesUsed() { return INSTANCE_SIZE - + (anyChildBlocked == null ? 0 : anyChildBlocked.ramBytesUsed()) + + anyChildBlocked.ramBytesUsed() + children.stream() .mapToLong(MemoryEstimationHelper::getEstimatedSizeOfAccountableObject) .sum() diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java index ac05610712aaa..00483f30b98bf 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java @@ -55,7 +55,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -74,7 +73,6 @@ private static TsBlock block(int... values) { private static CollectOperator collect(Operator... children) { CollectOperator collect = new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(children))); - collect.enableReadyInputConsumption(); return collect; } @@ -104,7 +102,6 @@ public void testThousandChildrenAndOneListenerPerPendingPeriod() throws Exceptio children.add(new Child(block(i))); } CollectOperator collect = new CollectOperator(new TestContext(), children); - collect.enableReadyInputConsumption(); Child ready = (Child) children.get(count - 1); for (int round = 0; round < 5; round++) { ListenableFuture wait = collect.isBlocked(); @@ -138,7 +135,6 @@ public void testAllReadyChildrenHaveLinearSelectionCost() throws Exception { } CountingList countedChildren = new CountingList(children); CollectOperator collect = new CollectOperator(new TestContext(), countedChildren); - collect.enableReadyInputConsumption(); try { for (int round = 0; round < rounds; round++) { assertTrue(collect.isBlocked().isDone()); @@ -184,7 +180,6 @@ public void testClosedChildrenDoNotIncreaseRemainingChildSelectionCost() throws } CountingList countedChildren = new CountingList(children); CollectOperator collect = new CollectOperator(new TestContext(), countedChildren); - collect.enableReadyInputConsumption(); try { // Retire the first 999 children and consume one block from the remaining child. for (int i = 0; i < childCount; i++) { @@ -385,160 +380,6 @@ public void testCloseWakesWaitAndClosesEveryChildDespiteException() throws Excep assertEquals(1, b.closes); } - @Test - public void testDefaultCollectRetainsSequentialConsumption() throws Exception { - Child a = new Child(block(1)); - Child b = new Child(block(2)); - b.ready(); - CollectOperator collect = - new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(a, b))); - assertFalse(collect.isBlocked().isDone()); - a.ready(); - assertSame(a.output, collect.next()); - a.hasNext = false; - a.finished = true; - assertNull(collect.next()); - assertSame(b.output, collect.next()); - assertEquals(0, collect.calculateRetainedSizeAfterCallingNext()); - collect.close(); - assertEquals(1, a.closes); - assertEquals(1, b.closes); - } - - @Test - public void testDefaultMappingCollectRetainsSequentialConsumption() throws Exception { - Child a = new Child(block(10, 11)); - Child b = new Child(block(20, 21)); - b.ready(); - List> mappings = new ArrayList<>(); - mappings.add(Collections.singletonList(0)); - mappings.add(Collections.singletonList(1)); - MappingCollectOperator collect = - new MappingCollectOperator( - new TestContext(), new ArrayList<>(Arrays.asList(a, b)), mappings); - assertFalse(collect.isBlocked().isDone()); - a.ready(); - assertEquals(10, collect.next().getColumn(0).getInt(0)); - a.hasNext = false; - a.finished = true; - assertNull(collect.next()); - assertEquals(21, collect.next().getColumn(0).getInt(0)); - collect.close(); - } - - @Test - public void testLegacyInitializesOtherInputsOnceAndFinishesAtLastChild() throws Exception { - Child a = new Child(block(1)); - Child b = new Child(block(2)); - b.ready(); - CollectOperator collect = - new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(a, b))); - for (int i = 0; i < 10; i++) { - assertFalse(collect.isBlocked().isDone()); - } - assertEquals(11, a.waitCalls); - assertEquals(1, b.waitCalls); - assertEquals(0, b.advances); - a.ready(); - assertSame(a.output, collect.next()); - a.hasNext = false; - assertNull(collect.next()); - assertTrue(collect.hasNext()); - assertFalse(collect.isFinished()); - for (int i = 0; i < 10; i++) { - assertTrue(collect.isBlocked().isDone()); - } - assertEquals(11, a.waitCalls); - assertEquals(11, b.waitCalls); - assertSame(b.output, collect.next()); - b.hasNext = false; - assertNull(collect.next()); - assertFalse(collect.hasNext()); - assertTrue(collect.isFinished()); - assertTrue(collect.isBlocked().isDone()); - collect.close(); - assertEquals(1, a.closes); - assertEquals(1, b.closes); - } - - @Test - public void testLegacyMemoryUsesLargestChildPeak() throws Exception { - Child a = new Child(block(1)); - Child b = new Child(block(2)); - a.peak = 100; - a.retained = 30; - a.returnSize = 10; - b.peak = 80; - b.retained = 50; - b.returnSize = 20; - CollectOperator collect = - new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(a, b))); - assertEquals(100, collect.calculateMaxPeekMemory()); - assertEquals(0, collect.calculateRetainedSizeAfterCallingNext()); - assertEquals(20, collect.calculateMaxReturnSize()); - collect.close(); - } - - @Test - public void testCannotEnableReadyConsumptionAfterLegacyInputIsRetired() throws Exception { - Child a = new Child(block(1)); - Child b = new Child(block(2)); - a.hasNext = false; - b.ready(); - CollectOperator collect = - new CollectOperator(new TestContext(), new ArrayList<>(Arrays.asList(a, b))); - assertNull(collect.next()); - assertThrows(IllegalStateException.class, collect::enableReadyInputConsumption); - assertSame(b.output, collect.next()); - b.hasNext = false; - assertNull(collect.next()); - assertTrue(collect.isFinished()); - } - - @Test - public void testCannotEnableReadyConsumptionAfterLegacyReadinessObservation() throws Exception { - Child child = new Child(block(1)); - CollectOperator collect = - new CollectOperator(new TestContext(), new ArrayList<>(Collections.singletonList(child))); - assertFalse(collect.isBlocked().isDone()); - assertThrows(IllegalStateException.class, collect::enableReadyInputConsumption); - collect.close(); - } - - @Test - public void testCannotEnableReadyConsumptionAfterLegacyNextWithoutIsBlocked() throws Exception { - Child child = new Child(block(1)); - child.ready(); - CollectOperator collect = - new CollectOperator(new TestContext(), new ArrayList<>(Collections.singletonList(child))); - assertSame(child.output, collect.next()); - assertThrows(IllegalStateException.class, collect::enableReadyInputConsumption); - collect.close(); - } - - @Test - public void testCannotEnableReadyConsumptionAfterLegacyClose() throws Exception { - Child child = new Child(block(1)); - CollectOperator collect = - new CollectOperator(new TestContext(), new ArrayList<>(Collections.singletonList(child))); - collect.close(); - assertThrows(IllegalStateException.class, collect::enableReadyInputConsumption); - assertEquals(1, child.closes); - } - - @Test - public void testEnablingReadyConsumptionIsIdempotent() throws Exception { - Child child = new Child(block(1)); - child.ready(); - CollectOperator collect = collect(child); - collect.enableReadyInputConsumption(); - ListenableFuture selection = collect.isBlocked(); - collect.enableReadyInputConsumption(); - assertSame(selection, collect.isBlocked()); - assertSame(child.output, collect.next()); - collect.close(); - } - @Test public void testMappingUsesActualChildIdentity() throws Exception { Child a = new Child(block(10, 11)); @@ -550,7 +391,6 @@ public void testMappingUsesActualChildIdentity() throws Exception { MappingCollectOperator collect = new MappingCollectOperator( new TestContext(), new ArrayList<>(Arrays.asList(a, b)), mappings); - collect.enableReadyInputConsumption(); assertTrue(collect.isBlocked().isDone()); assertEquals(21, collect.next().getColumn(0).getInt(0)); a.ready(); @@ -635,7 +475,6 @@ public void testConcurrentCompletionsConsumeEveryChildExactlyOnce() throws Excep List children = new ArrayList<>(); for (int i = 0; i < count; i++) children.add(new Child(block(i))); CollectOperator collect = new CollectOperator(new TestContext(), new ArrayList<>(children)); - collect.enableReadyInputConsumption(); ListenableFuture pending = collect.isBlocked(); assertFalse(pending.isDone()); ExecutorService executor = Executors.newFixedThreadPool(count); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 4b4ca52b8558f..2fae9aa537081 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -22,7 +22,6 @@ import org.apache.iotdb.calc.execution.filter.TopKRuntimeFilter; import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; import org.apache.iotdb.calc.execution.operator.Operator; -import org.apache.iotdb.calc.execution.operator.process.CollectOperator; import org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator; import org.apache.iotdb.calc.execution.operator.process.LimitOperator; import org.apache.iotdb.calc.execution.operator.process.OffsetOperator; @@ -48,9 +47,7 @@ import org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.commons.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.AggregationNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral; @@ -304,20 +301,6 @@ protected OperatorContext addOperatorContext( .addOperatorContext(context.getNextOperatorId(), planNodeId, operatorType); } - @Override - public Operator visitCollect(CollectNode node, LocalExecutionPlanContext context) { - CollectOperator collect = (CollectOperator) super.visitCollect(node, context); - context.addCollectOperator(collect); - return collect; - } - - @Override - public Operator visitUnion(UnionNode node, LocalExecutionPlanContext context) { - CollectOperator collect = (CollectOperator) super.visitUnion(node, context); - context.addCollectOperator(collect); - return collect; - } - @Override public Operator visitCteScan(CteScanNode node, LocalExecutionPlanContext context) { OperatorContext operatorContext = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java index 055bc4b19f98f..ec93a08690c1e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContext.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.queryengine.plan.planner; import org.apache.iotdb.calc.execution.operator.Operator; -import org.apache.iotdb.calc.execution.operator.process.CollectOperator; import org.apache.iotdb.calc.plan.planner.ITableOperatorGeneratorContext; import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.calc.plan.planner.memory.PipelineMemoryEstimator; @@ -92,7 +91,6 @@ public class LocalExecutionPlanContext implements ITableOperatorGeneratorContext private int exchangeSumNum = 0; private final long fragmentExchangeMaxBytes; private Map legacySinkLimits = new IdentityHashMap<>(); - private List collectOperators = new ArrayList<>(); private List cachedDataTypes; @@ -135,7 +133,6 @@ public LocalExecutionPlanContext( public LocalExecutionPlanContext(LocalExecutionPlanContext parentContext) { this.fragmentExchangeMaxBytes = parentContext.fragmentExchangeMaxBytes; this.legacySinkLimits = parentContext.legacySinkLimits; - this.collectOperators = parentContext.collectOperators; this.nextOperatorId = parentContext.nextOperatorId; this.typeProvider = parentContext.typeProvider; this.allSensorsMap = parentContext.allSensorsMap; @@ -279,10 +276,6 @@ public void addExchangeOperator(ExchangeOperator exchangeOperator) { this.exchangeOperatorList.add(exchangeOperator); } - public void addCollectOperator(CollectOperator collectOperator) { - collectOperators.add(collectOperator); - } - public void setMaxBytesOneHandleCanReserve() { long maxBytesOneHandleCanReserve = getMaxBytesOneHandleCanReserve(); LOGGER.debug( @@ -312,7 +305,7 @@ public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { long maxBlockBytes = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); boolean sharedBudget = (inputMaxBytes > 0 || outputMaxBytes > 0) - && SharedExchangeMemoryEligibility.isEligible(plan) + && SharedExchangeBudgetEligibility.isEligible(plan) // Only the root pipeline may exist. Child factories are registered during generation; // the root factory is registered after its memory has been estimated below. && pipelineDriverFactories.isEmpty() @@ -326,7 +319,6 @@ public void setExchangeMemoryLimits(PlanNode plan, MemoryPool memoryPool) { memoryPool.registerFragmentInstanceSharedBudget( instanceContext.getId().getQueryId().getId(), instanceContext.getId().getFragmentInstanceId()); - collectOperators.forEach(CollectOperator::enableReadyInputConsumption); LOGGER.debug( DataNodeQueryMessages.LOG_SHARED_EXCHANGE_BUDGET_FOR_FI_ARG_INPUT_ARG_OUTPUT_ARG_98AE1109, instanceContext.getId(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeBudgetEligibility.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeBudgetEligibility.java new file mode 100644 index 0000000000000..5c9d78291dd5f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeBudgetEligibility.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.queryengine.plan.planner; + +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +/** + * Decides whether a fragment may share its exchange budget across INPUT and OUTPUT sides. Each + * fragment contains at most one multi-child level: either a Collect/Union fan-in, whose ready-first + * consumption can borrow idle capacity from any child, or an operator that waits for all of its + * inputs before advancing, which a shared budget could starve. Ordering plays no part in this + * decision; consumers that require ordered input are handled by the planner. + */ +public final class SharedExchangeBudgetEligibility { + private SharedExchangeBudgetEligibility() {} + + public static boolean isEligible(PlanNode root) { + boolean hasCollect = false; + int multiChildNodeCount = 0; + Deque nodes = new ArrayDeque<>(); + nodes.push(root); + while (!nodes.isEmpty()) { + PlanNode node = nodes.pop(); + if (node instanceof ExchangeNode) { + // An upstream plan, if still attached, belongs to a different FI. + continue; + } + // TopK's input ordering assumptions remain unchanged, including its single-child form. + if (node instanceof TopKNode) { + return false; + } + boolean anyReady = node instanceof CollectNode || node instanceof UnionNode; + hasCollect |= anyReady; + List children = node.getChildren(); + if (children.size() > 1) { + // A fragment is expected to contain at most one multi-child level. Keep the fallback + // conservative if a future plan shape violates that invariant. + if (++multiChildNodeCount > 1 || !anyReady) { + return false; + } + } + nodes.addAll(children); + } + return hasCollect; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java deleted file mode 100644 index e66f1719bec61..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/SharedExchangeMemoryEligibility.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.queryengine.plan.planner; - -import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.AggregationNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.FillNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.FilterNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.LimitNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.OffsetNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.OutputNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.ProjectNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.TopKNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.UnionNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CopyToNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; - -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.List; -import java.util.Locale; -import java.util.Set; - -/** Select shared budgets only for FIs whose fan-in can consume any ready input. */ -public final class SharedExchangeMemoryEligibility { - private static final Set UNORDERED_AGGREGATIONS = - Set.of("count", "sum", "avg", "min", "max"); - - private SharedExchangeMemoryEligibility() {} - - public static boolean isEligible(PlanNode root) { - boolean hasCollect = false; - Deque nodes = new ArrayDeque<>(); - nodes.push(root); - while (!nodes.isEmpty()) { - PlanNode node = nodes.pop(); - if (node instanceof ExchangeNode) { - // An upstream plan, if still attached, belongs to a different FI. - continue; - } - // TopK's input ordering assumptions remain unchanged, including its single-child form. - if (node instanceof TopKNode) { - return false; - } - // Core relational nodes need not implement the serialization-only PlanNode.getType(). - boolean anyReady = node instanceof CollectNode || node instanceof UnionNode; - hasCollect |= anyReady; - if (!canConsumeUnordered(node)) { - return false; - } - List children = node.getChildren(); - // Unknown multi-input operators are not implicitly safe. In particular, sequential sinks - // and joins can wait for an input while another input holds the entire shared budget. - if (!anyReady && children.size() > 1) { - return false; - } - nodes.addAll(children); - } - return hasCollect; - } - - private static boolean canConsumeUnordered(PlanNode node) { - if (node instanceof AggregationNode) { - AggregationNode aggregation = (AggregationNode) node; - return aggregation.getPreGroupedSymbols().isEmpty() - && aggregation.getAggregations().values().stream() - .allMatch( - function -> - function.getOrderingScheme().isEmpty() - && !function.isInputOrderedByTimeAscending() - && UNORDERED_AGGREGATIONS.contains( - function - .getResolvedFunction() - .getSignature() - .getName() - .toLowerCase(Locale.ROOT))); - } - // Do not infer safety from the number of children. Windows, streaming grouping and - // TopK can depend on order even when they have only one input. Unknown - // consumers conservatively retain sequential consumption within this FI. - return node instanceof CollectNode - || node instanceof UnionNode - || node instanceof OutputNode - || node instanceof ProjectNode - || node instanceof FilterNode - || node instanceof IdentitySinkNode - || node instanceof ExchangeNode - || node instanceof FillNode - || node instanceof CopyToNode - || node instanceof LimitNode - || node instanceof OffsetNode - || node.getClass() == SortNode.class - || node.getChildren().isEmpty(); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index d4bab67db0616..1ead17632aaf1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -689,6 +689,12 @@ public List visitGroup(GroupNode node, PlanContext context) { for (PlanNode child : childrenNodes) { if (canSortEliminated(node.getOrderingScheme(), nodeOrderingMap.get(child.getPlanNodeId()))) { result.add(child); + } else if (groupingInvalidatedByCollect( + child, node.getOrderingScheme().getOrderBy().subList(0, node.getPartitionKeyCount()))) { + SortNode sort = + new SortNode(queryId.genPlanNodeId(), child, node.getOrderingScheme(), false, false); + nodeOrderingMap.put(sort.getPlanNodeId(), sort.getOrderingScheme()); + result.add(sort); } else { GroupNode subGroupNode = new GroupNode( @@ -1100,7 +1106,7 @@ public List visitExternalTsFileAggregationScan( if (needFinalAggregation) { OrderingScheme childOrdering = nodeOrderingMap.get(result.get(0).getPlanNodeId()); finalAggregation.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, result)); - return Collections.singletonList(finalAggregation); + return Collections.singletonList(validateAggregationGrouping(finalAggregation)); } return result; } @@ -2118,7 +2124,7 @@ public List visitAggregation(AggregationNode node, PlanContext context if (physicalAggregation.isStreamable() && childOrdering != null) { nodeOrderingMap.put(planNodeId, expectedOrderingSchema); } - return aggregationNode; + return validateAggregationGrouping(aggregationNode); }) .collect(Collectors.toList()); @@ -2128,10 +2134,10 @@ public List visitAggregation(AggregationNode node, PlanContext context new CollectNode(queryId.genPlanNodeId(), aggregationNodes.get(0).getOutputSymbols()); collectNode.setChildren(aggregationNodes); splitResult.left.setChild(collectNode); - return Collections.singletonList(splitResult.left); + return Collections.singletonList(validateAggregationGrouping(splitResult.left)); } - return Collections.singletonList(physicalAggregation); + return Collections.singletonList(validateAggregationGrouping(physicalAggregation)); } // We cannot do multi-stage Aggregate if any aggregation-function is distinct. @@ -2144,7 +2150,9 @@ public List visitAggregation(AggregationNode node, PlanContext context nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()), childrenNodes); node.setChild(physicalChild); return Collections.singletonList( - withRateFunctionInputOrdering(node, nodeOrderingMap.get(physicalChild.getPlanNodeId()))); + validateAggregationGrouping( + withRateFunctionInputOrdering( + node, nodeOrderingMap.get(physicalChild.getPlanNodeId())))); } Pair splitResult = split(node, symbolAllocator, queryId); AggregationNode intermediate = splitResult.right; @@ -2167,13 +2175,73 @@ public List visitAggregation(AggregationNode node, PlanContext context if (node.isStreamable() && childOrdering != null) { nodeOrderingMap.put(planNodeId, expectedOrderingSchema); } - return aggregationNode; + return validateAggregationGrouping(aggregationNode); }) .collect(Collectors.toList()); splitResult.left.setChild( mergeChildrenViaCollectOrMergeSort( nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()), childrenNodes)); - return Collections.singletonList(splitResult.left); + return Collections.singletonList(validateAggregationGrouping(splitResult.left)); + } + + private AggregationNode validateAggregationGrouping(AggregationNode node) { + if (!node.getPreGroupedSymbols().isEmpty() + && groupingInvalidatedByCollect(node.getChild(), node.getPreGroupedSymbols())) { + node.setPreGroupedSymbols(ImmutableList.of()); + nodeOrderingMap.remove(node.getPlanNodeId()); + } + return node; + } + + /** Whether an unordered fan-in invalidates a previously established grouping property. */ + private boolean groupingInvalidatedByCollect(PlanNode node, List grouping) { + if (grouping.isEmpty()) { + return false; + } + if ((node instanceof CollectNode || node instanceof UnionNode) + && node.getChildren().size() > 1) { + return true; + } + OrderingScheme ordering = nodeOrderingMap.get(node.getPlanNodeId()); + if (ordering != null && prefixMatched(ordering, grouping)) { + return false; + } + // A completed aggregation emits each full grouping key once, even when it hashes its input. + if (node instanceof AggregationNode) { + AggregationNode aggregation = (AggregationNode) node; + if (!aggregation.getStep().isOutputPartial() + && aggregation.getGroupingSetCount() == 1 + && grouping.containsAll(aggregation.getGroupingKeys())) { + return false; + } + } + if (node instanceof ProjectNode) { + ProjectNode project = (ProjectNode) node; + List inputs = new ArrayList<>(); + for (Symbol symbol : grouping) { + Expression expression = project.getAssignments().get(symbol); + if (!(expression instanceof SymbolReference)) { + // A computed key cannot inherit a branch-local continuity proof. + return containsUnorderedCollect(project.getChild()); + } + inputs.add(Symbol.from(expression)); + } + return groupingInvalidatedByCollect(project.getChild(), inputs); + } + for (PlanNode child : node.getChildren()) { + if (groupingInvalidatedByCollect(child, grouping)) { + return true; + } + } + return false; + } + + private boolean containsUnorderedCollect(PlanNode node) { + if ((node instanceof CollectNode || node instanceof UnionNode) + && node.getChildren().size() > 1) { + return true; + } + return node.getChildren().stream().anyMatch(this::containsUnorderedCollect); } private static AggregationNode withRateFunctionInputOrdering( @@ -2301,7 +2369,8 @@ public List visitAggregationTableScan( throw new IllegalStateException( DataNodeQueryMessages.LIST_PLANNODE_SIZE_SHOULD_1_BUT_NOW_IS); } - resultTableScanNodeList = Collections.singletonList(finalAggregation); + resultTableScanNodeList = + Collections.singletonList(validateAggregationGrouping(finalAggregation)); } return resultTableScanNodeList; @@ -2460,7 +2529,7 @@ private List constructSpilledAggregationTableScanByRegionReplicaSet( mergeChildrenViaCollectOrMergeSort( nodeOrderingMap.get(result.get(0).getPlanNodeId()), result)); } - return Collections.singletonList(finalAggregation); + return Collections.singletonList(validateAggregationGrouping(finalAggregation)); } return result; } @@ -2677,7 +2746,8 @@ public List visitAggregationTreeDeviceViewScan( throw new IllegalStateException( DataNodeQueryMessages.LIST_PLANNODE_SIZE_SHOULD_1_BUT_NOW_IS); } - resultTableScanNodeList = Collections.singletonList(finalAggregation); + resultTableScanNodeList = + Collections.singletonList(validateAggregationGrouping(finalAggregation)); } return resultTableScanNodeList; @@ -2863,7 +2933,7 @@ private List constructSpilledAggregationTreeDeviceViewScanByRegionRepl mergeChildrenViaCollectOrMergeSort( nodeOrderingMap.get(result.get(0).getPlanNodeId()), result)); } - return Collections.singletonList(finalAggregation); + return Collections.singletonList(validateAggregationGrouping(finalAggregation)); } return result; } @@ -4007,7 +4077,8 @@ public NodeDistribution getNodeDistribution(PlanNodeId nodeId) { public void clearExpectedOrderingScheme() { expectedOrderingScheme = null; hasSortProperty = false; - // Grouping pushdown requires an ordering; a global window clears that requirement. + // Grouping pushdown depends on this ordering. A boundary such as a global window + // must not pass an outer Group's partitioning request to its scans. pushDownGrouping = false; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java index e00322050fdfe..d010fc4a3275f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -386,7 +386,7 @@ public void testTreePlanningRetainsLegacySourceQuota() { context.setMaxBytesOneHandleCanReserve(); verify(source).setMaxBytesCanReserve(256); assertFalse( - SharedExchangeMemoryEligibility.isEligible( + SharedExchangeBudgetEligibility.isEligible( node( PlanNodeType.TREE_COLLECT, node(PlanNodeType.EXCHANGE), @@ -432,13 +432,13 @@ public void testSingleChildTopKRetainsLegacyBudgetAndConsumption() { PlanNode topK = node(PlanNodeType.TABLE_TOPK_NODE, child); context.setExchangeMemoryLimits(topK, pool); verify(source).setMaxBytesCanReserve(256); - assertFalse(collect.isBlocked().isDone()); + assertTrue(collect.isBlocked().isDone()); assertAdmissionIsShared(false); assertFalse( - SharedExchangeMemoryEligibility.isEligible( + SharedExchangeBudgetEligibility.isEligible( node(PlanNodeType.TABLE_COLLECT_NODE, topK, collect()))); assertTrue( - SharedExchangeMemoryEligibility.isEligible( + SharedExchangeBudgetEligibility.isEligible( node( PlanNodeType.TABLE_COLLECT_NODE, node(PlanNodeType.TABLE_EXCHANGE_NODE, topK), @@ -453,7 +453,7 @@ public void testMultiplePipelinesRetainSequentialCollectConsumption() { child.addPipelineDriverFactory(mock(Operator.class), child.getDriverContext(), 0); addSource(context); context.setExchangeMemoryLimits(plan, pool); - assertFalse(collect.isBlocked().isDone()); + assertTrue(collect.isBlocked().isDone()); assertAdmissionIsShared(false); } @@ -472,10 +472,10 @@ public void testNestedDependencyDisablesSharingForEntireFragment() { node(PlanNodeType.TABLE_EXCHANGE_NODE), node(PlanNodeType.TABLE_EXCHANGE_NODE)); assertFalse( - SharedExchangeMemoryEligibility.isEligible( + SharedExchangeBudgetEligibility.isEligible( node(PlanNodeType.TABLE_COLLECT_NODE, dependent, collect()))); assertFalse( - SharedExchangeMemoryEligibility.isEligible(node(dependency, collect(), collect()))); + SharedExchangeBudgetEligibility.isEligible(node(dependency, collect(), collect()))); } ISourceHandle source = addSource(context); ISinkHandle sink = mock(ISinkHandle.class); @@ -500,10 +500,13 @@ public void testNestedDependencyDisablesSharingForEntireFragment() { @Test public void testSequentialAndUnknownMultiInputOperatorsDisableSharing() { assertFalse( - SharedExchangeMemoryEligibility.isEligible( + SharedExchangeBudgetEligibility.isEligible( + node(PlanNodeType.TABLE_COLLECT_NODE, collect(), collect()))); + assertFalse( + SharedExchangeBudgetEligibility.isEligible( node(PlanNodeType.IDENTITY_SINK, collect(), collect()))); assertFalse( - SharedExchangeMemoryEligibility.isEligible( + SharedExchangeBudgetEligibility.isEligible( node(PlanNodeType.SCHEMA_QUERY_MERGE, collect(), collect()))); } @@ -511,7 +514,7 @@ public void testSequentialAndUnknownMultiInputOperatorsDisableSharing() { public void testExchangeBoundaryDoesNotInspectUpstreamFragment() { PlanNode remote = node(PlanNodeType.TABLE_MERGESORT_NODE, collect(), collect()); assertTrue( - SharedExchangeMemoryEligibility.isEligible( + SharedExchangeBudgetEligibility.isEligible( node( PlanNodeType.TABLE_COLLECT_NODE, node(PlanNodeType.TABLE_EXCHANGE_NODE, remote), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java index 31c562199fd63..ea9323fdfdbd3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerBudgetTest.java @@ -185,8 +185,8 @@ public void testNullSessionRetainsTreeGeneratorAndIndependentInputQuotas() throw planner.plan(plan, new TypeProvider(), instance, new DataNodeQueryContext(1)); assertEquals(1, factories.size()); CollectOperator collect = (CollectOperator) factories.get(0).getOperation(); - assertEquals(100 + collect.ramBytesUsed(), freeBefore - planner.getFreeMemoryForOperators()); - assertFalse(collect.isBlocked().isDone()); + assertEquals(150 + collect.ramBytesUsed(), freeBefore - planner.getFreeMemoryForOperators()); + assertTrue(collect.isBlocked().isDone()); assertEquals(0, instance.getInputMaxBytes()); verify(firstHandle).setMaxBytesCanReserve(512); verify(secondHandle).setMaxBytesCanReserve(512); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java index 28663c2c0a7d9..0bf8e170a6f98 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java @@ -24,10 +24,8 @@ import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GroupNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; -import org.apache.iotdb.db.queryengine.plan.planner.SharedExchangeMemoryEligibility; -import org.apache.iotdb.db.queryengine.plan.planner.plan.DistributedQueryPlan; -import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.StreamSortNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.WindowNode; import org.junit.Test; @@ -39,106 +37,138 @@ import static org.junit.Assert.assertTrue; public class CollectInputPropertyTest { - private static DistributedQueryPlan plan(String sql) { + private static List plans(String sql) { PlanTester tester = new PlanTester(); tester.createPlan(sql); - return tester.getDistributedPlan(); + List result = new ArrayList<>(); + for (int i = 0; i < tester.getDistributedPlan().getFragments().size(); i++) { + result.add(tester.getFragmentPlan(i)); + } + return result; } private static List nodes(PlanNode node, Class type) { List result = new ArrayList<>(); if (type.isInstance(node)) result.add(type.cast(node)); - if (!(node instanceof ExchangeNode)) - for (PlanNode child : node.getChildren()) result.addAll(nodes(child, type)); + for (PlanNode child : node.getChildren()) result.addAll(nodes(child, type)); return result; } - private static void assertSequential(DistributedQueryPlan plan) { - assertTrue( - plan.getInstances().stream() - .anyMatch( - fi -> !nodes(fi.getFragment().getPlanNodeTree(), CollectNode.class).isEmpty())); - for (FragmentInstance fi : plan.getInstances()) - assertFalse( - fi.getId().toString(), - SharedExchangeMemoryEligibility.isEligible(fi.getFragment().getPlanNodeTree())); + private static List nodes(List plans, Class type) { + List result = new ArrayList<>(); + for (PlanNode plan : plans) { + result.addAll(nodes(plan, type)); + } + return result; } @Test - public void testTopKWindowKeepsGroupInsteadOfFullSort() { - DistributedQueryPlan query = - plan( - "SELECT tag1, tag2, tag3, s1, SUM(s1) OVER (PARTITION BY tag1, tag2, tag3 ORDER BY s1) rs " - + "FROM (SELECT tag1, tag2, tag3, s1, ROW_NUMBER() OVER () n FROM table1) t " - + "WHERE n > 0 ORDER BY rs DESC LIMIT 10"); - assertSequential(query); - PlanNode root = query.getInstances().get(0).getFragment().getPlanNodeTree(); - assertEquals(1, nodes(root, GroupNode.class).size()); - assertTrue( - nodes(root, SortNode.class).stream().noneMatch(sort -> sort.getClass() == SortNode.class)); + public void testAggregationAboveGlobalWindowHashesInterleavedGroups() { + for (String aggregate : new String[] {"SUM(s1)", "COUNT(DISTINCT s1)"}) { + List fragments = + plans( + "SELECT tag1, tag2, tag3, " + + aggregate + + " FROM (" + + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " + + "FROM table1 WHERE tag1 != 'shenzhen') t WHERE n > 0 GROUP BY tag1, tag2, tag3"); + PlanNode root = fragments.get(0); + List aggregations = nodes(fragments, AggregationNode.class); + assertFalse(aggregations.isEmpty()); + assertFalse(nodes(fragments, CollectNode.class).isEmpty()); + for (AggregationNode aggregation : aggregations) { + if (!nodes(aggregation.getChild(), CollectNode.class).isEmpty()) { + assertTrue(aggregation.getPreGroupedSymbols().isEmpty()); + assertFalse(aggregation.isStreamable()); + } + } + assertTrue( + "Hash aggregation should not add a grouping sort", nodes(root, SortNode.class).isEmpty()); + } } @Test - public void testTopKAggregationKeepsStreamingGroups() { - DistributedQueryPlan query = - plan( - "SELECT tag1, tag2, tag3, SUM(s1) AS s FROM (SELECT tag1, tag2, tag3, s1, " - + "COUNT(*) OVER () AS n FROM table1 WHERE tag1 != 'shenzhen') t " - + "WHERE n > 0 GROUP BY tag1, tag2, tag3 ORDER BY s DESC LIMIT 10"); - assertSequential(query); - List aggregations = - nodes(query.getInstances().get(0).getFragment().getPlanNodeTree(), AggregationNode.class); - assertEquals(1, aggregations.size()); - assertEquals(3, aggregations.get(0).getPreGroupedSymbols().size()); - assertTrue(aggregations.get(0).isStreamable()); + public void testPartialPreGroupingAlsoFallsBackToHash() { + List fragments = + plans( + "SELECT tag1, tag2, tag3, s1, COUNT(*) FROM (" + + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " + + "FROM table1 WHERE tag1 != 'shenzhen') t WHERE n > 0 GROUP BY tag1, tag2, tag3, s1"); + List aggregations = nodes(fragments, AggregationNode.class); + assertFalse(aggregations.isEmpty()); + assertFalse(nodes(fragments, CollectNode.class).isEmpty()); + assertTrue(aggregations.get(0).getPreGroupedSymbols().isEmpty()); } @Test - public void testNestedWindowKeepsSequentialGroupingPlan() { - DistributedQueryPlan query = - plan( - "SELECT tag1, tag2, tag3, SUM(s1) OVER (PARTITION BY tag1, tag2, tag3 ORDER BY s1) AS rs " + public void testNestedWindowRestoresPartitionContiguityWithSort() { + List fragments = + plans( + "SELECT tag1, tag2, tag3, SUM(s1) OVER (" + + "PARTITION BY tag1, tag2, tag3 ORDER BY s1) AS running_sum " + "FROM (SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n FROM table1) t WHERE n > 0"); - assertSequential(query); - assertFalse( - nodes(query.getInstances().get(0).getFragment().getPlanNodeTree(), GroupNode.class) - .isEmpty()); + PlanNode root = fragments.get(0); + assertFalse(nodes(fragments, CollectNode.class).isEmpty()); + assertTrue(nodes(root, GroupNode.class).isEmpty()); + List windows = nodes(root, WindowNode.class); + assertEquals(2, windows.size()); + WindowNode outer = + windows.stream() + .filter(w -> !w.getSpecification().getPartitionBy().isEmpty()) + .findFirst() + .get(); + assertFalse(nodes(outer.getChild(), SortNode.class).isEmpty()); } @Test - public void testUnorderedScansProjectionsAggregationsAndUnionRetainReadyMode() { - for (String sql : - new String[] { - "SELECT * FROM table1", - "SELECT time, s1 + 1 AS x FROM table1", - "SELECT time, diff(s1) AS d FROM table1", - "SELECT time, abs(diff(s1)) AS d FROM table1 " - + "WHERE tag1='shenzhen' AND tag2='B2' AND tag3='ZZ'", - "SELECT time, s1 FROM table1 WHERE diff(s1) > 0", - "SELECT COUNT(*), SUM(s1), MIN(s1), MAX(s1) FROM table1", - "SELECT COUNT(DISTINCT s1) FROM table1", - "SELECT s1 FROM table1 UNION ALL SELECT s1 FROM table2" - }) { - DistributedQueryPlan query = plan(sql); - assertTrue( - sql, - query.getInstances().stream() - .anyMatch( - fi -> - SharedExchangeMemoryEligibility.isEligible( - fi.getFragment().getPlanNodeTree()))); + public void testStreamSortAfterGlobalWindowUsesFullSort() { + List fragments = + plans( + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " + + "FROM table1 ORDER BY tag1, tag2, tag3, s1"); + PlanNode root = fragments.get(0); + assertFalse(nodes(fragments, CollectNode.class).isEmpty()); + assertFalse(nodes(root, SortNode.class).isEmpty()); + assertTrue(nodes(root, StreamSortNode.class).isEmpty()); + } + + @Test + public void testConstantFillDoesNotRemoveRestoredSort() { + List fragments = + plans( + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n FROM table1 " + + "FILL METHOD CONSTANT 0 ORDER BY tag1, tag2, tag3"); + PlanNode root = fragments.get(0); + assertFalse(nodes(fragments, CollectNode.class).isEmpty()); + assertFalse(nodes(root, SortNode.class).isEmpty()); + assertTrue(nodes(root, StreamSortNode.class).isEmpty()); + } + + @Test + public void testUnionFinalAggregationDoesNotInheritBranchGrouping() { + List fragments = + plans( + "SELECT tag1, tag2, tag3, SUM(s1) FROM (" + + "SELECT tag1, tag2, tag3, s1 FROM table1 UNION ALL " + + "SELECT tag1, tag2, tag3, s1 FROM table2) u GROUP BY tag1, tag2, tag3"); + List aggregations = nodes(fragments, AggregationNode.class); + assertFalse(aggregations.isEmpty()); + boolean foundFinalCollect = false; + for (AggregationNode aggregation : aggregations) { + if (aggregation.getChild() instanceof CollectNode) { + foundFinalCollect = true; + assertTrue(aggregation.getPreGroupedSymbols().isEmpty()); + } } + assertTrue(foundFinalCollect); } @Test public void testOrderedScanBranchesKeepStreamingAggregation() { - DistributedQueryPlan query = - plan("SELECT tag1, tag2, tag3, SUM(s1 + 1) FROM table1 GROUP BY tag1, tag2, tag3"); - assertTrue( - query.getInstances().stream() - .anyMatch( - fi -> - nodes(fi.getFragment().getPlanNodeTree(), AggregationNode.class).stream() - .anyMatch(AggregationNode::isStreamable))); + List fragments = + plans("SELECT tag1, tag2, tag3, SUM(s1 + 1) FROM table1 GROUP BY tag1, tag2, tag3"); + List aggregations = nodes(fragments, AggregationNode.class); + assertFalse(aggregations.isEmpty()); + assertTrue(aggregations.stream().anyMatch(AggregationNode::isStreamable)); } } From 26d05c476d2246fce922432e5d1be7709faa9c33 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 21 Oct 2026 00:12:05 +0800 Subject: [PATCH 21/29] Fix nested ready Collect memory estimation --- .../iotdb/calc/execution/operator/Operator.java | 11 +++++++++++ .../operator/process/AbstractSortOperator.java | 7 ++++++- .../execution/operator/process/CollectOperator.java | 9 +++++++++ .../operator/process/PatternRecognitionOperator.java | 7 ++++++- .../operator/process/CollectOperatorTest.java | 1 + 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/Operator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/Operator.java index a2c937e0bb7c3..72d312b21655c 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/Operator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/Operator.java @@ -123,6 +123,17 @@ default long calculateMaxPeekMemoryWithCounter() { */ long calculateMaxPeekMemory(); + /** + * Returns the retained memory that a parent operator still needs to add to its peak estimate. + * + *

Most operators expose all of their retained input state separately, so the default is the + * complete retained size. Operators whose peak estimate already includes that state can override + * this method to avoid counting it again in a parent operator. + */ + default long calculateRetainedSizeForParentMaxPeekMemory() { + return calculateRetainedSizeAfterCallingNext(); + } + /** return estimated max memory footprint for returned TsBlock when calling operator.next(). */ long calculateMaxReturnSize(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AbstractSortOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AbstractSortOperator.java index c9f73f8493747..c4864ef3d147b 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AbstractSortOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AbstractSortOperator.java @@ -338,10 +338,15 @@ public boolean isFinished() throws Exception { @Override public long calculateMaxPeekMemory() { return inputOperator.calculateMaxPeekMemoryWithCounter() - + inputOperator.calculateRetainedSizeAfterCallingNext() + + inputOperator.calculateRetainedSizeForParentMaxPeekMemory() + sortBufferManager.getSortBufferSize(); } + @Override + public long calculateRetainedSizeForParentMaxPeekMemory() { + return 0; + } + @Override public long calculateMaxReturnSize() { return maxReturnSize; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java index efcc56677856f..3492e6fc9f2e8 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java @@ -178,6 +178,15 @@ public long calculateMaxPeekMemory() { return peak; } + /** + * Collect's peak includes the retained state of every live child. A parent must not add that + * state once more when it estimates its own peak. + */ + @Override + public long calculateRetainedSizeForParentMaxPeekMemory() { + return 0; + } + @Override public long calculateMaxReturnSize() { long maxReturnSize = 0; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/PatternRecognitionOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/PatternRecognitionOperator.java index 28c184eb17d8b..c4daacfef66f5 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/PatternRecognitionOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/PatternRecognitionOperator.java @@ -437,7 +437,12 @@ public long calculateMaxPeekMemory() { long maxPeekMemoryFromCurrent = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); return Math.max(maxPeekMemoryFromInput, maxPeekMemoryFromCurrent) - + child.calculateRetainedSizeAfterCallingNext(); + + child.calculateRetainedSizeForParentMaxPeekMemory(); + } + + @Override + public long calculateRetainedSizeForParentMaxPeekMemory() { + return 0; } @Override diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java index 00483f30b98bf..522666aa64348 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java @@ -413,6 +413,7 @@ public void testMemoryIncludesAllLiveRetainedStates() throws Exception { CollectOperator collect = collect(a, b); assertEquals(80, collect.calculateRetainedSizeAfterCallingNext()); assertEquals(150, collect.calculateMaxPeekMemory()); + assertEquals(0, collect.calculateRetainedSizeForParentMaxPeekMemory()); assertEquals(20, collect.calculateMaxReturnSize()); assertTrue(collect.ramBytesUsed() > 0); a.hasNext = false; From 1c96e7be4655d46db543808b471b0679cbcb2ce3 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 21 Oct 2026 11:44:00 +0800 Subject: [PATCH 22/29] Revert "Fix nested ready Collect memory estimation" In executed distributed plans every Collect child is an ExchangeOperator whose retained size is zero, so the parent-retained contract split never changes a production estimate. Restore the upstream formulas and drop the added interface method. --- .../iotdb/calc/execution/operator/Operator.java | 11 ----------- .../operator/process/AbstractSortOperator.java | 7 +------ .../execution/operator/process/CollectOperator.java | 9 --------- .../operator/process/PatternRecognitionOperator.java | 7 +------ .../operator/process/CollectOperatorTest.java | 1 - 5 files changed, 2 insertions(+), 33 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/Operator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/Operator.java index 72d312b21655c..a2c937e0bb7c3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/Operator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/Operator.java @@ -123,17 +123,6 @@ default long calculateMaxPeekMemoryWithCounter() { */ long calculateMaxPeekMemory(); - /** - * Returns the retained memory that a parent operator still needs to add to its peak estimate. - * - *

Most operators expose all of their retained input state separately, so the default is the - * complete retained size. Operators whose peak estimate already includes that state can override - * this method to avoid counting it again in a parent operator. - */ - default long calculateRetainedSizeForParentMaxPeekMemory() { - return calculateRetainedSizeAfterCallingNext(); - } - /** return estimated max memory footprint for returned TsBlock when calling operator.next(). */ long calculateMaxReturnSize(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AbstractSortOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AbstractSortOperator.java index c4864ef3d147b..c9f73f8493747 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AbstractSortOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/AbstractSortOperator.java @@ -338,15 +338,10 @@ public boolean isFinished() throws Exception { @Override public long calculateMaxPeekMemory() { return inputOperator.calculateMaxPeekMemoryWithCounter() - + inputOperator.calculateRetainedSizeForParentMaxPeekMemory() + + inputOperator.calculateRetainedSizeAfterCallingNext() + sortBufferManager.getSortBufferSize(); } - @Override - public long calculateRetainedSizeForParentMaxPeekMemory() { - return 0; - } - @Override public long calculateMaxReturnSize() { return maxReturnSize; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java index 3492e6fc9f2e8..efcc56677856f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java @@ -178,15 +178,6 @@ public long calculateMaxPeekMemory() { return peak; } - /** - * Collect's peak includes the retained state of every live child. A parent must not add that - * state once more when it estimates its own peak. - */ - @Override - public long calculateRetainedSizeForParentMaxPeekMemory() { - return 0; - } - @Override public long calculateMaxReturnSize() { long maxReturnSize = 0; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/PatternRecognitionOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/PatternRecognitionOperator.java index c4daacfef66f5..28c184eb17d8b 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/PatternRecognitionOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/PatternRecognitionOperator.java @@ -437,12 +437,7 @@ public long calculateMaxPeekMemory() { long maxPeekMemoryFromCurrent = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); return Math.max(maxPeekMemoryFromInput, maxPeekMemoryFromCurrent) - + child.calculateRetainedSizeForParentMaxPeekMemory(); - } - - @Override - public long calculateRetainedSizeForParentMaxPeekMemory() { - return 0; + + child.calculateRetainedSizeAfterCallingNext(); } @Override diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java index 522666aa64348..00483f30b98bf 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/CollectOperatorTest.java @@ -413,7 +413,6 @@ public void testMemoryIncludesAllLiveRetainedStates() throws Exception { CollectOperator collect = collect(a, b); assertEquals(80, collect.calculateRetainedSizeAfterCallingNext()); assertEquals(150, collect.calculateMaxPeekMemory()); - assertEquals(0, collect.calculateRetainedSizeForParentMaxPeekMemory()); assertEquals(20, collect.calculateMaxReturnSize()); assertTrue(collect.ramBytesUsed() > 0); a.hasNext = false; From 0ec1d7a3d698ccb023b8ed3de122f6f924b2983f Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 21 Oct 2026 11:51:06 +0800 Subject: [PATCH 23/29] Document ready-first Collect peak estimation --- .../calc/execution/operator/process/CollectOperator.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java index efcc56677856f..d0d9b396d76c0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/CollectOperator.java @@ -161,6 +161,14 @@ public void close() throws Exception { } } + /** + * Every live child holds its retained state while one of them is advanced from its retained size + * to its peak, so the instant peak is {@code retained + max(child.peak - child.retained)}. + * + *

In executed distributed plans every child is an ExchangeOperator whose retained size is + * zero, so this degenerates to {@code max(child.peak)} and no parent peak formula counts any + * child state twice. + */ @Override public long calculateMaxPeekMemory() { long retained = calculateRetainedSizeAfterCallingNext(); From a281cf2d146050434e4a82698888c9138ca5b1a2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 21 Oct 2026 11:58:40 +0800 Subject: [PATCH 24/29] Use unordered assertions in pattern ITs instead of ORDER BY A final ORDER BY creates an outer full Sort whose static sort-buffer reservation can exhaust the fragment instance's operators memory block and trigger error 719 when fragment-instance memory releases are delayed under the tests' tight memory configuration. --- .../it/db/it/IoTDBPatternAggregationIT.java | 77 +++-- .../it/db/it/IoTDBPatternRecognitionIT.java | 277 +++++++----------- 2 files changed, 153 insertions(+), 201 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java index 850d3480bb068..79f3531a6d341 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java @@ -31,8 +31,12 @@ import org.junit.runner.RunWith; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; +import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -205,7 +209,7 @@ public void test1() { "2025-01-01T00:06:00.000Z,2,2,3,20.0,10.0,30.0,60.0,15.0,20.0,20.0,", "2025-01-01T00:07:00.000Z,2,3,3,30.0,10.0,60.0,60.0,20.0,20.0,30.0,", }; - tableResultSetEqualTest( + assertUnorderedTableResult( "SELECT m.time, m.match, m.count1, m.count2, m.max, m.min, m.sum1, m.sum2, m.avg1, m.avg2, m.totalprice " + "FROM t1 " + "MATCH_RECOGNIZE ( " @@ -227,10 +231,9 @@ public void test1() { + " B AS B.totalprice = 20, " + " C AS C.totalprice = 30, " + " D AS D.totalprice = 40 " - + ") AS m ORDER BY m.time ", + + ") AS m ", expectedHeader, - retArray, - DATABASE_NAME); + retArray); } @Test @@ -259,7 +262,7 @@ public void test2() { "2025-01-01T00:06:00.000Z,2,B,2,2,2,0,1,1,1,20.0,", "2025-01-01T00:07:00.000Z,2,C,3,3,3,1,1,1,1,30.0,", }; - tableResultSetEqualTest( + assertUnorderedTableResult( "SELECT m.time, m.match, m.label, m.count_0, m.count_1, m.count_2, m.count_c, m.final_count_c, m.count_u, m.final_count_u, m.totalprice " + "FROM t1 " + "MATCH_RECOGNIZE ( " @@ -282,10 +285,9 @@ public void test2() { + " B AS B.totalprice = 20, " + " C AS C.totalprice = 30, " + " D AS D.totalprice = 40 " - + ") AS m ORDER BY m.time ", + + ") AS m ", expectedHeader, - retArray, - DATABASE_NAME); + retArray); } @Test @@ -312,7 +314,7 @@ public void test3() { "2025-01-01T00:06:00.000Z,2,B,30.0,0.0,30.0,20.0,20.0,20.0,", "2025-01-01T00:07:00.000Z,2,C,60.0,30.0,30.0,20.0,20.0,30.0,", }; - tableResultSetEqualTest( + assertUnorderedTableResult( "SELECT m.time, m.match, m.label, m.sum, m.sum_c, m.final_sum_c, m.sum_u, m.final_sum_u, m.totalprice " + "FROM t1 " + "MATCH_RECOGNIZE ( " @@ -333,10 +335,9 @@ public void test3() { + " B AS B.totalprice = 20, " + " C AS C.totalprice = 30, " + " D AS D.totalprice = 40 " - + ") AS m ORDER BY m.time ", + + ") AS m ", expectedHeader, - retArray, - DATABASE_NAME); + retArray); } @Test @@ -363,7 +364,7 @@ public void test4() { "2025-01-01T00:06:00.000Z,2,B,15.0,0.0,30.0,20.0,20.0,20.0,", "2025-01-01T00:07:00.000Z,2,C,20.0,30.0,30.0,20.0,20.0,30.0,", }; - tableResultSetEqualTest( + assertUnorderedTableResult( "SELECT m.time, m.match, m.label, m.avg, m.avg_c, m.final_avg_c, m.avg_u, m.final_avg_u, m.totalprice " + "FROM t1 " + "MATCH_RECOGNIZE ( " @@ -384,10 +385,9 @@ public void test4() { + " B AS B.totalprice = 20, " + " C AS C.totalprice = 30, " + " D AS D.totalprice = 40 " - + ") AS m ORDER BY m.time ", + + ") AS m ", expectedHeader, - retArray, - DATABASE_NAME); + retArray); } @Test @@ -406,7 +406,7 @@ public void test5() { "2025-01-01T00:06:00.000Z,2,2025-01-01T00:05:00.000Z,2025-01-01T00:06:00.000Z,2025-01-01T00:06:00.000Z,2025-01-01T00:05:00.000Z,10.0,20.0,", "2025-01-01T00:07:00.000Z,2,2025-01-01T00:05:00.000Z,2025-01-01T00:07:00.000Z,2025-01-01T00:07:00.000Z,2025-01-01T00:05:00.000Z,10.0,30.0," }; - tableResultSetEqualTest( + assertUnorderedTableResult( "SELECT m.time, m.match, m.firstTime, m.lastTime, m.maxTime, m.minTime, m.firstVal, m.lastVal " + "FROM t1 " + "MATCH_RECOGNIZE ( " @@ -426,10 +426,9 @@ public void test5() { + " B AS totalprice = 20, " + " C AS totalprice = 30, " + " D AS totalprice = 40 " - + ") AS m ORDER BY m.time ", + + ") AS m ", expectedHeader, - retArray, - DATABASE_NAME); + retArray); } @Test @@ -453,7 +452,7 @@ public void test6() { "2025-01-01T00:05:00.000Z,1,7.0,-8.0,3.2,40.7,40.7,32.56,6.379655,6.379655,5.706137," }; - tableResultSetEqualTest( + assertUnorderedTableResult( "SELECT m.time, m.match, m.mode, m.extreme, " + "ROUND(m.avg, 6) AS avg, " + "ROUND(m.var_0, 6) AS var_0, ROUND(m.var_1, 6) AS var_1, ROUND(m.var_2, 6) AS var_2, " @@ -476,10 +475,9 @@ public void test6() { + " PATTERN (A+) " + " DEFINE " + " A AS true " - + ") AS m ORDER BY m.time ", + + ") AS m ", expectedHeader, - retArray, - DATABASE_NAME); + retArray); } @Test @@ -493,7 +491,7 @@ public void testAggregationsInDefineClause() { "2025-01-01T00:03:00.000Z,1,A,5.0,4.0,5.0,", "2025-01-01T00:04:00.000Z,1,B,7.0,8.5,13.0,", }; - tableResultSetEqualTest( + assertUnorderedTableResult( "SELECT m.time, m.match, m.label, m.avg, m.running_avg_b, m.totalprice " + "FROM t2 " + "MATCH_RECOGNIZE ( " @@ -507,10 +505,9 @@ public void testAggregationsInDefineClause() { + " PATTERN ((A | B)*) " + " DEFINE " + " A AS AVG(totalprice) = 5 " - + ") AS m ORDER BY m.time ", + + ") AS m ", expectedHeader, - retArray, - DATABASE_NAME); + retArray); } @Test @@ -579,4 +576,28 @@ public void testDifferentCast() { retArray, DATABASE_NAME); } + + /** + * Asserts the query result with order-insensitive, multiplicity-preserving comparison. + * + *

These queries deliberately have no final ORDER BY: adding one (e.g. a final ORDER BY on + * m.time) would create an outer full Sort whose static sort-buffer reservation can exhaust the + * fragment instance's operators memory block and trigger error 719 ("There is not enough memory + * to execute current fragment instance") when fragment-instance memory releases are delayed under + * the tests' tight memory configuration. Unordered assertions keep the tests deterministic + * without the Sort. + */ + private static void assertUnorderedTableResult( + String sql, String[] expectedHeader, String[] retArray) { + String header = String.join(",", expectedHeader) + ","; + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = statement.executeQuery(sql)) { + assertUnorderedQueryResult(sql, resultSet, header, Arrays.asList(retArray)); + } + } catch (SQLException e) { + fail(e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java index 9dd2ec8809bd0..5edc559d52471 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java @@ -355,43 +355,37 @@ public void testLogicalNavigationFunction() { + " PATTERN (A+) " + " DEFINE " + " A AS true " - + ") AS m ORDER BY m.time"; + + ") AS m"; // LAST(totalprice) - tableResultSetEqualTest(format(sql, "totalprice"), expectedHeader, retArray1, DATABASE_NAME); - tableResultSetEqualTest( - format(sql, "RPR_LAST(totalprice)"), expectedHeader, retArray1, DATABASE_NAME); - tableResultSetEqualTest( - format(sql, "RPR_LAST(totalprice, 0)"), expectedHeader, retArray1, DATABASE_NAME); - tableResultSetEqualTest( - format(sql, "RUNNING RPR_LAST(totalprice)"), expectedHeader, retArray1, DATABASE_NAME); + assertUnorderedTableResult(format(sql, "totalprice"), expectedHeader, retArray1); + assertUnorderedTableResult(format(sql, "RPR_LAST(totalprice)"), expectedHeader, retArray1); + assertUnorderedTableResult(format(sql, "RPR_LAST(totalprice, 0)"), expectedHeader, retArray1); + assertUnorderedTableResult( + format(sql, "RUNNING RPR_LAST(totalprice)"), expectedHeader, retArray1); // FINAL LAST(totalprice) - tableResultSetEqualTest( - format(sql, "FINAL RPR_LAST(totalprice)"), expectedHeader, retArray2, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "FINAL RPR_LAST(totalprice)"), expectedHeader, retArray2); // FIRST(totalprice) - tableResultSetEqualTest( - format(sql, "RPR_FIRST(totalprice)"), expectedHeader, retArray3, DATABASE_NAME); - tableResultSetEqualTest( - format(sql, "RPR_FIRST(totalprice, 0)"), expectedHeader, retArray3, DATABASE_NAME); - tableResultSetEqualTest( - format(sql, "RUNNING RPR_FIRST(totalprice)"), expectedHeader, retArray3, DATABASE_NAME); - tableResultSetEqualTest( - format(sql, "FINAL RPR_FIRST(totalprice)"), expectedHeader, retArray3, DATABASE_NAME); + assertUnorderedTableResult(format(sql, "RPR_FIRST(totalprice)"), expectedHeader, retArray3); + assertUnorderedTableResult(format(sql, "RPR_FIRST(totalprice, 0)"), expectedHeader, retArray3); + assertUnorderedTableResult( + format(sql, "RUNNING RPR_FIRST(totalprice)"), expectedHeader, retArray3); + assertUnorderedTableResult( + format(sql, "FINAL RPR_FIRST(totalprice)"), expectedHeader, retArray3); // LAST(totalprice, 2) - tableResultSetEqualTest( - format(sql, "RPR_LAST(totalprice, 2)"), expectedHeader, retArray4, DATABASE_NAME); + assertUnorderedTableResult(format(sql, "RPR_LAST(totalprice, 2)"), expectedHeader, retArray4); // FINAL LAST(totalprice, 2) - tableResultSetEqualTest( - format(sql, "FINAL RPR_LAST(totalprice, 2)"), expectedHeader, retArray5, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "FINAL RPR_LAST(totalprice, 2)"), expectedHeader, retArray5); // FIRST(totalprice, 2) - tableResultSetEqualTest( - format(sql, "RPR_FIRST(totalprice, 2)"), expectedHeader, retArray6, DATABASE_NAME); - tableResultSetEqualTest( - format(sql, "FINAL RPR_FIRST(totalprice, 2)"), expectedHeader, retArray6, DATABASE_NAME); + assertUnorderedTableResult(format(sql, "RPR_FIRST(totalprice, 2)"), expectedHeader, retArray6); + assertUnorderedTableResult( + format(sql, "FINAL RPR_FIRST(totalprice, 2)"), expectedHeader, retArray6); } @Test @@ -450,26 +444,20 @@ public void testPhysicalNavigationFunction() { + " PATTERN (A+) " + " DEFINE " + " A AS true " - + ") AS m ORDER BY m.time"; + + ") AS m"; // PREV(totalprice) - tableResultSetEqualTest( - format(sql1, "PREV(totalprice)"), expectedHeader1, retArray1, DATABASE_NAME); - tableResultSetEqualTest( - format(sql1, "PREV(totalprice, 1)"), expectedHeader1, retArray1, DATABASE_NAME); + assertUnorderedTableResult(format(sql1, "PREV(totalprice)"), expectedHeader1, retArray1); + assertUnorderedTableResult(format(sql1, "PREV(totalprice, 1)"), expectedHeader1, retArray1); // PREV(totalprice, 2) - tableResultSetEqualTest( - format(sql1, "PREV(totalprice, 2)"), expectedHeader1, retArray2, DATABASE_NAME); + assertUnorderedTableResult(format(sql1, "PREV(totalprice, 2)"), expectedHeader1, retArray2); // NEXT(totalprice) - tableResultSetEqualTest( - format(sql1, "NEXT(totalprice)"), expectedHeader1, retArray3, DATABASE_NAME); - tableResultSetEqualTest( - format(sql1, "NEXT(totalprice, 1)"), expectedHeader1, retArray3, DATABASE_NAME); + assertUnorderedTableResult(format(sql1, "NEXT(totalprice)"), expectedHeader1, retArray3); + assertUnorderedTableResult(format(sql1, "NEXT(totalprice, 1)"), expectedHeader1, retArray3); // NEXT(totalprice, 2) - tableResultSetEqualTest( - format(sql1, "NEXT(totalprice, 2)"), expectedHeader1, retArray4, DATABASE_NAME); + assertUnorderedTableResult(format(sql1, "NEXT(totalprice, 2)"), expectedHeader1, retArray4); String[] expectedHeader2 = new String[] {"price"}; String sql2 = @@ -584,62 +572,62 @@ public void testNestedNavigation() { + " PATTERN (A+) " + " DEFINE " + " A AS true " - + ") AS m ORDER BY m.time"; + + ") AS m"; // PREV(FIRST(totalprice)) - tableResultSetEqualTest( - format(sql, "PREV(RPR_FIRST(totalprice))"), expectedHeader, retArray1, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "PREV(RPR_FIRST(totalprice))"), expectedHeader, retArray1); // PREV(FIRST(totalprice), 2) - tableResultSetEqualTest( - format(sql, "PREV(RPR_FIRST(totalprice), 2)"), expectedHeader, retArray1, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "PREV(RPR_FIRST(totalprice), 2)"), expectedHeader, retArray1); // PREV(FIRST(totalprice, 2)) - tableResultSetEqualTest( - format(sql, "PREV(RPR_FIRST(totalprice, 2))"), expectedHeader, retArray2, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "PREV(RPR_FIRST(totalprice, 2))"), expectedHeader, retArray2); // PREV(FIRST(totalprice, 2), 2) - tableResultSetEqualTest( - format(sql, "PREV(RPR_FIRST(totalprice, 2), 2)"), expectedHeader, retArray3, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "PREV(RPR_FIRST(totalprice, 2), 2)"), expectedHeader, retArray3); // PREV(LAST(totalprice)) - tableResultSetEqualTest( - format(sql, "PREV(RPR_LAST(totalprice))"), expectedHeader, retArray4, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "PREV(RPR_LAST(totalprice))"), expectedHeader, retArray4); // PREV(LAST(totalprice), 2) - tableResultSetEqualTest( - format(sql, "PREV(RPR_LAST(totalprice), 2)"), expectedHeader, retArray5, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "PREV(RPR_LAST(totalprice), 2)"), expectedHeader, retArray5); // PREV(LAST(totalprice, 1)) - tableResultSetEqualTest( - format(sql, "PREV(RPR_LAST(totalprice, 1))"), expectedHeader, retArray5, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "PREV(RPR_LAST(totalprice, 1))"), expectedHeader, retArray5); // PREV(LAST(totalprice, 1), 2) - tableResultSetEqualTest( - format(sql, "PREV(RPR_LAST(totalprice, 1), 2)"), expectedHeader, retArray1, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "PREV(RPR_LAST(totalprice, 1), 2)"), expectedHeader, retArray1); // NEXT(FIRST(totalprice)) - tableResultSetEqualTest( - format(sql, "NEXT(RPR_FIRST(totalprice))"), expectedHeader, retArray2, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "NEXT(RPR_FIRST(totalprice))"), expectedHeader, retArray2); // NEXT(FIRST(totalprice), 2) - tableResultSetEqualTest( - format(sql, "NEXT(RPR_FIRST(totalprice), 2)"), expectedHeader, retArray6, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "NEXT(RPR_FIRST(totalprice), 2)"), expectedHeader, retArray6); // NEXT(FIRST(totalprice, 1)) - tableResultSetEqualTest( - format(sql, "NEXT(RPR_FIRST(totalprice, 1))"), expectedHeader, retArray6, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "NEXT(RPR_FIRST(totalprice, 1))"), expectedHeader, retArray6); // NEXT(FIRST(totalprice, 1), 2) - tableResultSetEqualTest( - format(sql, "NEXT(RPR_FIRST(totalprice, 1), 2)"), expectedHeader, retArray1, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "NEXT(RPR_FIRST(totalprice, 1), 2)"), expectedHeader, retArray1); // NEXT(LAST(totalprice)) - tableResultSetEqualTest( - format(sql, "NEXT(RPR_LAST(totalprice))"), expectedHeader, retArray7, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "NEXT(RPR_LAST(totalprice))"), expectedHeader, retArray7); // NEXT(LAST(totalprice), 2) - tableResultSetEqualTest( - format(sql, "NEXT(RPR_LAST(totalprice), 2)"), expectedHeader, retArray8, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "NEXT(RPR_LAST(totalprice), 2)"), expectedHeader, retArray8); // NEXT(LAST(totalprice, 2)) - tableResultSetEqualTest( - format(sql, "NEXT(RPR_LAST(totalprice, 2))"), expectedHeader, retArray9, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "NEXT(RPR_LAST(totalprice, 2))"), expectedHeader, retArray9); // NEXT(LAST(totalprice, 2), 2) - tableResultSetEqualTest( - format(sql, "NEXT(RPR_LAST(totalprice, 2), 2)"), expectedHeader, retArray10, DATABASE_NAME); + assertUnorderedTableResult( + format(sql, "NEXT(RPR_LAST(totalprice, 2), 2)"), expectedHeader, retArray10); } @Test @@ -673,9 +661,9 @@ public void testUnionVariable() { + " A AS A.totalprice = 80, " + " L AS L.totalprice < 80, " + " H AS H.totalprice > 80 " - + ") AS m ORDER BY m.time"; + + ") AS m"; - tableResultSetEqualTest(sql, expectedHeader, retArray1, DATABASE_NAME); + assertUnorderedTableResult(sql, expectedHeader, retArray1); } @Test @@ -709,9 +697,9 @@ public void testClassifierFunction() { + " A AS A.totalprice = 80, " + " L AS L.totalprice < 80, " + " H AS H.totalprice > 80 " - + ") AS m ORDER BY m.time"; + + ") AS m"; - tableResultSetEqualTest(sql, expectedHeader, retArray1, DATABASE_NAME); + assertUnorderedTableResult(sql, expectedHeader, retArray1); } @Test @@ -796,52 +784,48 @@ public void testRowPattern() { DATABASE_NAME); // pattern concatenation - tableResultSetEqualTest( + assertUnorderedTableResult( format( - sql + " ORDER BY m.time", + sql, "PATTERN (A B C) " + "DEFINE " + " B AS B.totalprice < PREV (B.totalprice), " + " C AS C.totalprice = PREV (C.totalprice)"), expectedHeader, - retArray3, - DATABASE_NAME); + retArray3); // pattern alternation - tableResultSetEqualTest( + assertUnorderedTableResult( format( - sql + " ORDER BY m.time", + sql, "PATTERN (B | C | A) " + "DEFINE " + " B AS B.totalprice < PREV (B.totalprice), " + " C AS C.totalprice <= PREV (C.totalprice)"), expectedHeader, - retArray4, - DATABASE_NAME); + retArray4); // pattern permutation - tableResultSetEqualTest( + assertUnorderedTableResult( format( - sql + " ORDER BY m.time", + sql, "PATTERN (PERMUTE(B, C)) " + "DEFINE " + " B AS B.totalprice < PREV (B.totalprice), " + " C AS C.totalprice < PREV (C.totalprice)"), expectedHeader, - retArray5, - DATABASE_NAME); + retArray5); // grouped pattern - tableResultSetEqualTest( + assertUnorderedTableResult( format( - sql + " ORDER BY m.time", + sql, "PATTERN (((A) (B (C)))) " + "DEFINE " + " B AS B.totalprice < PREV (B.totalprice), " + " C AS C.totalprice = PREV (C.totalprice)"), expectedHeader, - retArray3, - DATABASE_NAME); + retArray3); } @Test @@ -901,68 +885,29 @@ public void testPatternQuantifier() { + " B AS B.totalprice <= PREV(B.totalprice) " + ") AS m"; - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B*)"), expectedHeader, retArray1, DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B*)"), expectedHeader, retArray1); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B*?)"), - expectedHeader, - retArray2, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B*?)"), expectedHeader, retArray2); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B+)"), expectedHeader, retArray3, DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B+)"), expectedHeader, retArray3); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B+?)"), - expectedHeader, - retArray4, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B+?)"), expectedHeader, retArray4); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B?)"), expectedHeader, retArray5, DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B?)"), expectedHeader, retArray5); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B??)"), - expectedHeader, - retArray2, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B??)"), expectedHeader, retArray2); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{,})"), - expectedHeader, - retArray1, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{,})"), expectedHeader, retArray1); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{,}?)"), - expectedHeader, - retArray2, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{,}?)"), expectedHeader, retArray2); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{1,})"), - expectedHeader, - retArray3, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{1,})"), expectedHeader, retArray3); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{1,}?)"), - expectedHeader, - retArray4, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{1,}?)"), expectedHeader, retArray4); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{2,})"), - expectedHeader, - retArray3, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{2,})"), expectedHeader, retArray3); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{2,}?)"), - expectedHeader, - retArray6, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{2,}?)"), expectedHeader, retArray6); tableResultSetEqualTest( format(sql, "PATTERN (B{5,})"), expectedHeader, new String[] {}, DATABASE_NAME); @@ -970,43 +915,29 @@ public void testPatternQuantifier() { tableResultSetEqualTest( format(sql, "PATTERN (B{5,}?)"), expectedHeader, new String[] {}, DATABASE_NAME); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{,1})"), - expectedHeader, - retArray5, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{,1})"), expectedHeader, retArray5); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{,1}?)"), - expectedHeader, - retArray2, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{,1}?)"), expectedHeader, retArray2); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{1,1})"), - expectedHeader, - retArray4, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{1,1})"), expectedHeader, retArray4); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{1,1}?)"), - expectedHeader, - retArray4, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{1,1}?)"), expectedHeader, retArray4); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{1})"), - expectedHeader, - retArray4, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{1})"), expectedHeader, retArray4); - tableResultSetEqualTest( - format(sql + " ORDER BY m.time", "PATTERN (B{1}?)"), - expectedHeader, - retArray4, - DATABASE_NAME); + assertUnorderedTableResult(format(sql, "PATTERN (B{1}?)"), expectedHeader, retArray4); } + /** + * Asserts the query result with order-insensitive, multiplicity-preserving comparison. + * + *

These queries deliberately have no final ORDER BY: adding one (e.g. a final ORDER BY on + * m.time) would create an outer full Sort whose static sort-buffer reservation can exhaust the + * fragment instance's operators memory block and trigger error 719 ("There is not enough memory + * to execute current fragment instance") when fragment-instance memory releases are delayed under + * the tests' tight memory configuration. Unordered assertions keep the tests deterministic + * without the Sort. + */ private static void assertUnorderedTableResult( String sql, String[] expectedHeader, String[] retArray) { String header = String.join(",", expectedHeader) + ","; From 895edc3be068e408509106ebba4c41215edeafa4 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 21 Oct 2026 14:03:03 +0800 Subject: [PATCH 25/29] Replace cross-node result comparison with per-node checks Ready-first Collect makes row interleaving timing-dependent, so the cluster framework's cross-node result comparison can throw InconsistentDataException for queries without a final ordering. Move the unordered, multiplicity- preserving assertion into TestUtils and check each node's result against the full expectation individually. --- .../db/it/utils/CollectQueryAssertions.java | 111 ------------------ .../apache/iotdb/db/it/utils/TestUtils.java | 51 +++++--- .../it/triple/IoTDBShowReceiversTripleIT.java | 83 +++++++------ .../it/db/it/IoTDBMultiDeviceTableIT.java | 27 +++-- .../it/db/it/IoTDBPatternAggregationIT.java | 4 +- .../it/db/it/IoTDBPatternRecognitionIT.java | 4 +- .../relational/it/db/it/IoTDBWindowTVFIT.java | 3 +- .../udf/IoTDBUserDefinedTableFunctionIT.java | 45 +++---- .../query/recent/IoTDBClusterResultSetIT.java | 76 +++++------- .../it/query/recent/IoTDBCollectFanInIT.java | 4 +- .../it/query/recent/IoTDBCteIT.java | 82 +++++++------ .../query/recent/IoTDBTableAggregationIT.java | 28 +++-- .../IoTDBCorrelatedExistsSubqueryIT.java | 4 +- ...IoTDBCorrelatedQuantifiedComparisonIT.java | 4 +- .../IoTDBCorrelatedScalarSubqueryIT.java | 4 +- .../IoTDBUncorrelatedExistsSubqueryIT.java | 4 +- ...oTDBUncorrelatedInPredicateSubqueryIT.java | 4 +- ...TDBUncorrelatedQuantifiedComparisonIT.java | 4 +- .../IoTDBUncorrelatedScalarSubqueryIT.java | 4 +- 19 files changed, 236 insertions(+), 310 deletions(-) delete mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java deleted file mode 100644 index 0c5906a9e3c37..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/CollectQueryAssertions.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.it.utils; - -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Query; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QueryBody; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QuerySpecification; -import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.TableSubquery; -import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; -import org.apache.iotdb.itbase.runtime.ClusterTestResultSet; - -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** Explicit SQL-aware assertions for queries affected by ready-input Collect consumption. */ -public final class CollectQueryAssertions { - private CollectQueryAssertions() {} - - /** Only queries without final ordering or an arbitrary top-level slice may use this path. */ - public static List getUnorderedResultSets(String sql, ResultSet resultSet) { - org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement statement = - new SqlParser().createStatement(sql, ZoneOffset.UTC, null); - assertTrue("Unordered assertions require a SELECT query: " + sql, statement instanceof Query); - Query query = (Query) statement; - while (true) { - assertFalse( - "Final ORDER BY requires ordered assertions: " + sql, query.getOrderBy().isPresent()); - assertFalse( - "LIMIT requires deterministic result selection: " + sql, query.getLimit().isPresent()); - assertFalse( - "OFFSET requires deterministic result selection: " + sql, query.getOffset().isPresent()); - QueryBody body = query.getQueryBody(); - if (body instanceof QuerySpecification) { - QuerySpecification specification = (QuerySpecification) body; - assertFalse( - "Final ORDER BY requires ordered assertions: " + sql, - specification.getOrderBy().isPresent()); - assertFalse( - "LIMIT requires deterministic result selection: " + sql, - specification.getLimit().isPresent()); - assertFalse( - "OFFSET requires deterministic result selection: " + sql, - specification.getOffset().isPresent()); - } - if (!(body instanceof TableSubquery)) { - break; - } - // Parentheses around the whole query retain its ordering; nested FROM/UNION branches do not. - query = ((TableSubquery) body).getQuery(); - } - return nodeResults(resultSet); - } - - private static List nodeResults(ResultSet resultSet) { - return resultSet instanceof ClusterTestResultSet - ? ClusterTestResultSet.getNodeResultSets(resultSet) - : Collections.singletonList(resultSet); - } - - /** Compare every node independently while retaining duplicate multiplicity. */ - public static void assertUnorderedQueryResult( - String sql, ResultSet resultSet, String expectedHeader, List expectedRows) - throws SQLException { - List sortedExpected = new ArrayList<>(expectedRows); - Collections.sort(sortedExpected); - for (ResultSet nodeResult : getUnorderedResultSets(sql, resultSet)) { - ResultSetMetaData metadata = nodeResult.getMetaData(); - StringBuilder header = new StringBuilder(); - for (int i = 1; i <= metadata.getColumnCount(); i++) { - header.append(metadata.getColumnName(i)).append(","); - } - assertEquals(expectedHeader, header.toString()); - List actualRows = new ArrayList<>(); - while (nodeResult.next()) { - StringBuilder row = new StringBuilder(); - for (int i = 1; i <= metadata.getColumnCount(); i++) { - row.append(nodeResult.getString(i)).append(","); - } - actualRows.add(row.toString()); - } - Collections.sort(actualRows); - assertEquals(sortedExpected, actualRows); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java index 79a2f1c984769..1de8211b06b17 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java @@ -65,6 +65,7 @@ import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT; import static org.apache.iotdb.itbase.env.BaseEnv.TREE_SQL_DIALECT; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -807,6 +808,15 @@ public static void assertResultSetEqual( final String expectedHeader, final Collection expectedResult, final boolean printResult) { + if (expectedResult instanceof Set) { + List nodeResults = getNodeResultSets(actualResultSet); + if (nodeResults.size() != 1 || nodeResults.get(0) != actualResultSet) { + for (ResultSet nodeResult : nodeResults) { + assertResultSetEqual(nodeResult, expectedHeader, expectedResult, printResult); + } + return; + } + } try { final ResultSetMetaData resultSetMetaData = actualResultSet.getMetaData(); final StringBuilder header = new StringBuilder(); @@ -868,27 +878,36 @@ public static void assertResultSetEqual( String expectedHeader, Set expectedRetSet, Consumer consumer) { - try { - ResultSetMetaData resultSetMetaData = actualResultSet.getMetaData(); + assertResultSetEqual(actualResultSet, expectedHeader, expectedRetSet, false); + } + + /** + * Checks unordered rows on every node, preserving duplicate multiplicity. + * + *

The caller is responsible for ensuring that the query's final result has no ordering or + * arbitrary slice semantics that require deterministic row order. + */ + public static void assertResultSetEqualUnordered( + ResultSet resultSet, String expectedHeader, List expectedRows) throws SQLException { + List sortedExpected = new ArrayList<>(expectedRows); + Collections.sort(sortedExpected); + for (ResultSet nodeResult : getNodeResultSets(resultSet)) { + ResultSetMetaData metadata = nodeResult.getMetaData(); StringBuilder header = new StringBuilder(); - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - header.append(resultSetMetaData.getColumnName(i)).append(","); + for (int i = 1; i <= metadata.getColumnCount(); i++) { + header.append(metadata.getColumnName(i)).append(","); } assertEquals(expectedHeader, header.toString()); - - Set actualRetSet = new HashSet<>(); - - while (actualResultSet.next()) { - StringBuilder builder = new StringBuilder(); - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - builder.append(actualResultSet.getString(i)).append(","); + List actualRows = new ArrayList<>(); + while (nodeResult.next()) { + StringBuilder row = new StringBuilder(); + for (int i = 1; i <= metadata.getColumnCount(); i++) { + row.append(nodeResult.getString(i)).append(","); } - actualRetSet.add(builder.toString()); + actualRows.add(row.toString()); } - assertStringSetEqual(expectedRetSet, actualRetSet); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(String.valueOf(e)); + Collections.sort(actualRows); + assertEquals(sortedExpected, actualRows); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java index 0712387775b9d..7fec946aac751 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBShowReceiversTripleIT.java @@ -44,6 +44,7 @@ import java.sql.Connection; import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; @@ -54,6 +55,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; + @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT3.class}) public class IoTDBShowReceiversTripleIT { @@ -214,39 +217,44 @@ private boolean hasReceiversOnDifferentDataNodes( final String secondPipeName, final int secondReceiverDataNodeId) throws SQLException { - boolean hasFirstPipe = false; - boolean hasSecondPipe = false; - final Set receiverDataNodeIds = new HashSet<>(); try (final Connection connection = receiverEnv.getConnection( SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect); final Statement statement = connection.createStatement(); - final ResultSet resultSet = statement.executeQuery(sql)) { - while (resultSet.next()) { - final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids"); - if (!"DataNode".equals(getString(resultSet, "ReceiverNodeType", "receiver_node_type")) - || !"thrift".equals(getString(resultSet, "Protocol", "protocol")) - || pipeIds == null) { - continue; - } - final int receiverDataNodeId = getInt(resultSet, "ReceiverNodeId", "receiver_node_id"); - if (receiverDataNodeId == firstReceiverDataNodeId - && pipeIds.contains(firstPipeName + "@") - && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 - && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { - hasFirstPipe = true; - receiverDataNodeIds.add(receiverDataNodeId); + final ResultSet clusterResult = statement.executeQuery(sql)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + boolean hasFirstPipe = false; + boolean hasSecondPipe = false; + final Set receiverDataNodeIds = new HashSet<>(); + while (resultSet.next()) { + final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids"); + if (!"DataNode".equals(getString(resultSet, "ReceiverNodeType", "receiver_node_type")) + || !"thrift".equals(getString(resultSet, "Protocol", "protocol")) + || pipeIds == null) { + continue; + } + final int receiverDataNodeId = getInt(resultSet, "ReceiverNodeId", "receiver_node_id"); + if (receiverDataNodeId == firstReceiverDataNodeId + && pipeIds.contains(firstPipeName + "@") + && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 + && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { + hasFirstPipe = true; + receiverDataNodeIds.add(receiverDataNodeId); + } + if (receiverDataNodeId == secondReceiverDataNodeId + && pipeIds.contains(secondPipeName + "@") + && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 + && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { + hasSecondPipe = true; + receiverDataNodeIds.add(receiverDataNodeId); + } } - if (receiverDataNodeId == secondReceiverDataNodeId - && pipeIds.contains(secondPipeName + "@") - && getInt(resultSet, "ConnectionCount", "connection_count") >= 1 - && getInt(resultSet, "PipeCount", "pipe_count") >= 1) { - hasSecondPipe = true; - receiverDataNodeIds.add(receiverDataNodeId); + if (!hasFirstPipe || !hasSecondPipe || receiverDataNodeIds.size() < 2) { + return false; } } } - return hasFirstPipe && hasSecondPipe && receiverDataNodeIds.size() >= 2; + return true; } private int getDataNodeId(final DataNodeWrapper targetDataNode) throws Exception { @@ -268,20 +276,27 @@ private int getDataNodeId(final DataNodeWrapper targetDataNode) throws Exception private static String getString( final ResultSet resultSet, final String treeColumnName, final String tableColumnName) throws SQLException { - try { - return resultSet.getString(treeColumnName); - } catch (final SQLException ignored) { - return resultSet.getString(tableColumnName); - } + return resultSet.getString(getColumnIndex(resultSet, treeColumnName, tableColumnName)); } private static int getInt( final ResultSet resultSet, final String treeColumnName, final String tableColumnName) throws SQLException { - try { - return resultSet.getInt(treeColumnName); - } catch (final SQLException ignored) { - return resultSet.getInt(tableColumnName); + return resultSet.getInt(getColumnIndex(resultSet, treeColumnName, tableColumnName)); + } + + private static int getColumnIndex( + final ResultSet resultSet, final String treeColumnName, final String tableColumnName) + throws SQLException { + // SHOW RECEIVERS and information_schema.receivers expose different column names. + final ResultSetMetaData metadata = resultSet.getMetaData(); + for (int column = 1; column <= metadata.getColumnCount(); column++) { + final String name = metadata.getColumnName(column); + if (treeColumnName.equals(name) || tableColumnName.equals(name)) { + return column; + } } + throw new AssertionError( + "Missing receiver column: " + treeColumnName + " / " + tableColumnName); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java index 16d92c213436d..096c1383c10d6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiDeviceTableIT.java @@ -37,6 +37,7 @@ import java.util.HashMap; import java.util.Map; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -269,20 +270,22 @@ private void testSelectAll() { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("use \"test\""); - Map lastTimeMap = new HashMap<>(); - try (ResultSet resultSet = statement.executeQuery(selectSql)) { - int cnt = 0; - while (resultSet.next()) { - String tag = resultSet.getString("tag1") + "_" + resultSet.getString("tag2"); - long before = lastTimeMap.getOrDefault(tag, -1L); - long cur = resultSet.getTimestamp("time").getTime(); - if (cur <= before) { - fail("time order wrong!"); + try (ResultSet clusterResult = statement.executeQuery(selectSql)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + Map lastTimeMap = new HashMap<>(); + int cnt = 0; + while (resultSet.next()) { + String tag = resultSet.getString("tag1") + "_" + resultSet.getString("tag2"); + long before = lastTimeMap.getOrDefault(tag, -1L); + long cur = resultSet.getTimestamp("time").getTime(); + if (cur <= before) { + fail("time order wrong!"); + } + lastTimeMap.put(tag, cur); + cnt++; } - lastTimeMap.put(tag, cur); - cnt++; + assertEquals(16030, cnt); } - assertEquals(16030, cnt); } } catch (Exception e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java index 79f3531a6d341..222eb37a9eacb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java @@ -36,7 +36,7 @@ import java.sql.Statement; import java.util.Arrays; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -594,7 +594,7 @@ private static void assertUnorderedTableResult( Statement statement = connection.createStatement()) { statement.execute("use " + DATABASE_NAME); try (ResultSet resultSet = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, resultSet, header, Arrays.asList(retArray)); + assertResultSetEqualUnordered(resultSet, header, Arrays.asList(retArray)); } } catch (SQLException e) { fail(e.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java index 5edc559d52471..9a3d17f20fa20 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java @@ -37,7 +37,7 @@ import java.util.Arrays; import static java.lang.String.format; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -945,7 +945,7 @@ private static void assertUnorderedTableResult( Statement statement = connection.createStatement()) { statement.execute("use " + DATABASE_NAME); try (ResultSet resultSet = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, resultSet, header, Arrays.asList(retArray)); + assertResultSetEqualUnordered(resultSet, header, Arrays.asList(retArray)); } } catch (SQLException e) { fail(e.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java index 533c284b2a707..b6f720d7c0ee8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java @@ -41,7 +41,6 @@ import java.util.Comparator; import java.util.List; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; import static org.apache.iotdb.db.it.utils.TestUtils.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -1710,7 +1709,7 @@ private static void assertUnorderedTableResult( Statement statement = connection.createStatement()) { statement.execute("use " + DATABASE_NAME); try (ResultSet resultSet = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, resultSet, header, Arrays.asList(retArray)); + assertResultSetEqualUnordered(resultSet, header, Arrays.asList(retArray)); } } catch (SQLException e) { fail(e.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java index 5fd2ef8dd8ee7..849f76dd72076 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java @@ -41,6 +41,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -200,37 +201,39 @@ public void testLargeResultIsSplitWithoutDataLoss() throws Exception { "large_result", "org.apache.iotdb.db.query.udf.example.relational.LargeResultTableFunction"); - Set returnedRows = new HashSet<>(); try (Connection connection = EnvFactory.getEnv().getTableConnection(); Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); - try (ResultSet resultSet = + try (ResultSet clusterResult = statement.executeQuery( "SELECT * FROM large_result(vehicle, " + LARGE_RESULT_REPEAT_COUNT + ", " + LARGE_RESULT_PAYLOAD_SIZE + ")")) { - while (resultSet.next()) { - int repeatIndex = resultSet.getInt("repeat_index"); - long time = resultSet.getLong("time"); - Assert.assertTrue(repeatIndex >= 0 && repeatIndex < LARGE_RESULT_REPEAT_COUNT); - Assert.assertEquals( - repeatIndex + ":" + "x".repeat(LARGE_RESULT_PAYLOAD_SIZE), - resultSet.getString("payload")); - assertPassThroughColumns(resultSet, time); - Assert.assertTrue( - "Duplicate result row for time " + time + " and repeat index " + repeatIndex, - returnedRows.add(time + ":" + repeatIndex)); - } - } - } + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + Set returnedRows = new HashSet<>(); + while (resultSet.next()) { + int repeatIndex = resultSet.getInt("repeat_index"); + long time = resultSet.getLong("time"); + Assert.assertTrue(repeatIndex >= 0 && repeatIndex < LARGE_RESULT_REPEAT_COUNT); + Assert.assertEquals( + repeatIndex + ":" + "x".repeat(LARGE_RESULT_PAYLOAD_SIZE), + resultSet.getString("payload")); + assertPassThroughColumns(resultSet, time); + Assert.assertTrue( + "Duplicate result row for time " + time + " and repeat index " + repeatIndex, + returnedRows.add(time + ":" + repeatIndex)); + } - long[] inputTimes = new long[] {1, 2, 3, 5}; - Assert.assertEquals(inputTimes.length * LARGE_RESULT_REPEAT_COUNT, returnedRows.size()); - for (long time : inputTimes) { - for (int repeatIndex = 0; repeatIndex < LARGE_RESULT_REPEAT_COUNT; repeatIndex++) { - Assert.assertTrue(returnedRows.contains(time + ":" + repeatIndex)); + long[] inputTimes = new long[] {1, 2, 3, 5}; + Assert.assertEquals(inputTimes.length * LARGE_RESULT_REPEAT_COUNT, returnedRows.size()); + for (long time : inputTimes) { + for (int repeatIndex = 0; repeatIndex < LARGE_RESULT_REPEAT_COUNT; repeatIndex++) { + Assert.assertTrue(returnedRows.contains(time + ":" + repeatIndex)); + } + } + } } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java index 3371c3d4dd988..601d4ae1fe3fe 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBClusterResultSetIT.java @@ -38,9 +38,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Set; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.getUnorderedResultSets; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; @@ -51,89 +52,72 @@ public class IoTDBClusterResultSetIT { @Test public void testUnorderedRowsCheckEveryNode() throws SQLException { try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { - assertUnorderedQueryResult( - "SELECT value FROM t", result, "value,", Arrays.asList("a,", "b,")); + assertResultSetEqualUnordered(result, "value,", Arrays.asList("a,", "b,")); } try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "c"})) { assertThrows( AssertionError.class, - () -> - assertUnorderedQueryResult( - "SELECT value FROM t", result, "value,", Arrays.asList("a,", "b,"))); + () -> assertResultSetEqualUnordered(result, "value,", Arrays.asList("a,", "b,"))); } } @Test public void testUnorderedRowsPreserveDuplicateMultiplicity() throws SQLException { try (ResultSet result = cluster(new String[] {"a", "a", "b"}, new String[] {"b", "a", "a"})) { - assertUnorderedQueryResult( - "SELECT value FROM t", result, "value,", Arrays.asList("a,", "a,", "b,")); + assertResultSetEqualUnordered(result, "value,", Arrays.asList("a,", "a,", "b,")); } for (String[] values : new String[][] {{"a", "b"}, {"a", "a", "a", "b"}}) { try (ResultSet result = cluster(new String[] {"a", "a", "b"}, values)) { assertThrows( AssertionError.class, - () -> - assertUnorderedQueryResult( - "SELECT value FROM t", result, "value,", Arrays.asList("a,", "a,", "b,"))); + () -> assertResultSetEqualUnordered(result, "value,", Arrays.asList("a,", "a,", "b,"))); } } } @Test - public void testFinalOrderingRejectsUnorderedAssertions() throws SQLException { - for (String sql : - new String[] { - "SELECT value FROM t ORDER BY value", - "(SELECT value FROM t ORDER BY value)", - "SELECT value FROM t UNION ALL SELECT value FROM t ORDER BY value" - }) { - try (ResultSet result = rows("a")) { - assertThrows(AssertionError.class, () -> getUnorderedResultSets(sql, result)); - } - } + public void testSetAssertionChecksEveryNode() throws SQLException { try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { + TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,")); + } + try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"a"})) { assertThrows( AssertionError.class, - () -> TestUtils.assertResultSetEqual(result, "value,", Arrays.asList("a,", "b,"))); + () -> TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,"))); } } @Test - public void testNestedOrderingDoesNotOrderFinalResult() throws SQLException { - for (String sql : - new String[] { - "(SELECT value FROM t ORDER BY value LIMIT 1) UNION ALL (SELECT value FROM t ORDER BY value LIMIT 1)", - "SELECT * FROM (SELECT value FROM t ORDER BY value)", - "SELECT row_number() OVER (ORDER BY value) FROM t", - "SELECT 'order by' FROM t /* ORDER BY value */" - }) { - try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { - assertUnorderedQueryResult(sql, result, "value,", Arrays.asList("a,", "b,")); - } + public void testSetConsumerAssertionChecksEveryNode() throws SQLException { + try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { + TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,"), message -> {}); + } + try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"a"})) { + assertThrows( + AssertionError.class, + () -> + TestUtils.assertResultSetEqual(result, "value,", Set.of("a,", "b,"), message -> {})); } } @Test - public void testArbitrarySlicesNeedDeterministicSelection() throws SQLException { - for (String sql : - new String[] {"SELECT value FROM t LIMIT 1", "SELECT value FROM t OFFSET 1"}) { - try (ResultSet result = rows("a")) { - assertThrows(AssertionError.class, () -> getUnorderedResultSets(sql, result)); - } + public void testOrderedRowsStillRejectDifferentNodeOrder() throws SQLException { + try (ResultSet result = cluster(new String[] {"a", "b"}, new String[] {"b", "a"})) { + assertThrows( + AssertionError.class, + () -> TestUtils.assertResultSetEqual(result, "value,", Arrays.asList("a,", "b,"))); } } @Test public void testSingleNodeAndNullValues() throws SQLException { try (ResultSet result = rows("a", null, "a")) { - assertEquals(1, getUnorderedResultSets("SELECT value FROM t", result).size()); - assertUnorderedQueryResult( - "SELECT value FROM t", result, "value,", Arrays.asList("null,", "a,", "a,")); + assertEquals(1, getNodeResultSets(result).size()); + assertResultSetEqualUnordered(result, "value,", Arrays.asList("null,", "a,", "a,")); } try (ResultSet result = cluster(new String[] {"a"})) { - assertEquals(1, getUnorderedResultSets("SELECT value FROM t", result).size()); - assertUnorderedQueryResult("SELECT value FROM t", result, "value,", Arrays.asList("a,")); + assertEquals(1, getNodeResultSets(result).size()); + assertResultSetEqualUnordered(result, "value,", Arrays.asList("a,")); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java index 4c04dfc68898a..8afd6dc97515e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java @@ -41,7 +41,7 @@ import java.util.List; import java.util.Set; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -109,7 +109,7 @@ public void testUnorderedUnionPreservesEveryDuplicateOnEachNode() throws Excepti "SELECT device, zone, value FROM readings1 UNION ALL " + "SELECT device, zone, value FROM readings1"; try (ResultSet result = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, result, "device,zone,value,", expectedRows); + assertResultSetEqualUnordered(result, "device,zone,value,", expectedRows); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java index 65d3f277e52de..e8089db62f13b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCteIT.java @@ -60,6 +60,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -361,19 +362,22 @@ public void testNestExplain1() throws SQLException { statement.execute("USE testdb"); // explain - ResultSet resultSet = statement.executeQuery(sql); - ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(metaData.getColumnCount(), 1); - assertEquals(metaData.getColumnName(1), "distribution plan"); - - StringBuilder sb = new StringBuilder(); - while (resultSet.next()) { - sb.append(resultSet.getString(1)).append(System.lineSeparator()); + try (ResultSet clusterResult = statement.executeQuery(sql)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnCount(), 1); + assertEquals(metaData.getColumnName(1), "distribution plan"); + + StringBuilder sb = new StringBuilder(); + while (resultSet.next()) { + sb.append(resultSet.getString(1)).append(System.lineSeparator()); + } + String result = sb.toString(); + assertFalse(result.contains("CTE Query : 'cte1'")); + assertTrue(result.contains("CTE Query : 'cte2'")); + assertTrue(result.contains("Main Query")); + } } - String result = sb.toString(); - assertFalse(result.contains("CTE Query : 'cte1'")); - assertTrue(result.contains("CTE Query : 'cte2'")); - assertTrue(result.contains("Main Query")); } } @@ -388,19 +392,22 @@ public void testNestExplain2() throws SQLException { statement.execute("USE testdb"); // explain - ResultSet resultSet = statement.executeQuery(sql); - ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(metaData.getColumnCount(), 1); - assertEquals(metaData.getColumnName(1), "distribution plan"); - - StringBuilder sb = new StringBuilder(); - while (resultSet.next()) { - sb.append(resultSet.getString(1)).append(System.lineSeparator()); + try (ResultSet clusterResult = statement.executeQuery(sql)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnCount(), 1); + assertEquals(metaData.getColumnName(1), "distribution plan"); + + StringBuilder sb = new StringBuilder(); + while (resultSet.next()) { + sb.append(resultSet.getString(1)).append(System.lineSeparator()); + } + String result = sb.toString(); + assertTrue(result.contains("CTE Query : 'cte1'")); + assertTrue(result.contains("CTE Query : 'cte2'")); + assertTrue(result.contains("Main Query")); + } } - String result = sb.toString(); - assertTrue(result.contains("CTE Query : 'cte1'")); - assertTrue(result.contains("CTE Query : 'cte2'")); - assertTrue(result.contains("Main Query")); } } @@ -415,19 +422,22 @@ public void testExplainJoin() throws SQLException { statement.execute("USE testdb"); // explain - ResultSet resultSet = statement.executeQuery(sql); - ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(metaData.getColumnCount(), 1); - assertEquals(metaData.getColumnName(1), "distribution plan"); - - StringBuilder sb = new StringBuilder(); - while (resultSet.next()) { - sb.append(resultSet.getString(1)).append(System.lineSeparator()); + try (ResultSet clusterResult = statement.executeQuery(sql)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnCount(), 1); + assertEquals(metaData.getColumnName(1), "distribution plan"); + + StringBuilder sb = new StringBuilder(); + while (resultSet.next()) { + sb.append(resultSet.getString(1)).append(System.lineSeparator()); + } + String result = sb.toString(); + assertFalse(result.contains("CTE Query : 'cte1'")); + assertTrue(result.contains("CTE Query : 'cte2'")); + assertTrue(result.contains("Main Query")); + } } - String result = sb.toString(); - assertFalse(result.contains("CTE Query : 'cte1'")); - assertTrue(result.contains("CTE Query : 'cte2'")); - assertTrue(result.contains("Main Query")); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index 518b3da1fdd21..99188903b99a7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -44,6 +44,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.buildHeaders; import static org.apache.iotdb.relational.it.db.it.IoTDBMultiTAGsWithAttributesTableIT.repeatTest; @@ -5902,19 +5903,22 @@ private void assertRateQueryUsesIntermediateAggregation(String query) { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); - try (ResultSet resultSet = statement.executeQuery("EXPLAIN (FORMAT JSON) " + query)) { - Assert.assertTrue(resultSet.next()); - String plan = resultSet.getString(1); - // The PARTIAL/FINAL split only exists when the scanned data spans multiple DataRegions. - // Under randomized partition allocation (e.g. the SHUFFLE strategy used by - // IoTDBTableAggregation2IT), all devices may occasionally land in one single region, - // where a SINGLE-step aggregation is the correct plan, so the check must be skipped. - if (countDistinctRegionIds(plan) < 2) { - return; + try (ResultSet clusterResult = statement.executeQuery("EXPLAIN (FORMAT JSON) " + query)) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + Assert.assertTrue(resultSet.next()); + String plan = resultSet.getString(1); + // The PARTIAL/FINAL split only exists when the scanned data spans multiple DataRegions. + // Under randomized partition allocation (e.g. the SHUFFLE strategy used by + // IoTDBTableAggregation2IT), all devices may occasionally land in one single region, + // where a SINGLE-step aggregation is the correct plan, so the check must be skipped. + if (countDistinctRegionIds(plan) < 2) { + continue; + } + Assert.assertTrue( + "Expected a PARTIAL aggregation in plan: " + plan, plan.contains("PARTIAL")); + Assert.assertTrue( + "Expected a FINAL aggregation in plan: " + plan, plan.contains("FINAL")); } - Assert.assertTrue( - "Expected a PARTIAL aggregation in plan: " + plan, plan.contains("PARTIAL")); - Assert.assertTrue("Expected a FINAL aggregation in plan: " + plan, plan.contains("FINAL")); } } catch (Exception exception) { throw new AssertionError(exception); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java index fe725976bd0c2..326b22d0790b2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java @@ -38,7 +38,7 @@ import java.util.Arrays; import java.util.List; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; @@ -430,7 +430,7 @@ private static void assertUnorderedResult( Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); try (ResultSet resultSet = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); } } catch (SQLException e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java index 5c233cdd6fbd8..1098b62bf16f5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java @@ -37,7 +37,7 @@ import java.util.Arrays; import java.util.List; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; @@ -162,7 +162,7 @@ private static void assertUnorderedResult( Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); try (ResultSet resultSet = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); } } catch (SQLException e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java index 91a7e99f7752f..3b4aff7ebed58 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java @@ -38,7 +38,7 @@ import java.util.Arrays; import java.util.List; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; @@ -421,7 +421,7 @@ private static void assertUnorderedResult( Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); try (ResultSet resultSet = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); } } catch (SQLException e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java index ea9a7235d527d..136243fb2a81a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java @@ -37,7 +37,7 @@ import java.util.Arrays; import java.util.List; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; @@ -390,7 +390,7 @@ private static void assertUnorderedResult( Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); try (ResultSet resultSet = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); } } catch (SQLException e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java index 9144eb49a4a32..db8c2d4bbd235 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -37,7 +37,7 @@ import java.util.Arrays; import java.util.List; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; @@ -300,7 +300,7 @@ private static void assertUnorderedResult( Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); try (ResultSet resultSet = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); } } catch (SQLException e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java index ff4a091ab99ed..5bb2424936078 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java @@ -37,7 +37,7 @@ import java.util.Arrays; import java.util.List; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; @@ -650,7 +650,7 @@ private static void assertUnorderedResult( Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); try (ResultSet resultSet = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); } } catch (SQLException e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java index efa8327eafb3b..b98c240d5bbb9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java @@ -37,7 +37,7 @@ import java.util.Arrays; import java.util.List; -import static org.apache.iotdb.db.it.utils.CollectQueryAssertions.assertUnorderedQueryResult; +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; @@ -383,7 +383,7 @@ private static void assertUnorderedResult( Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE_NAME); try (ResultSet resultSet = statement.executeQuery(sql)) { - assertUnorderedQueryResult(sql, resultSet, expectedHeader, expectedRows); + assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); } } catch (SQLException e) { e.printStackTrace(); From 180e40289f82f27f2ffd0f55f36d5ac33c9b7536 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 21 Oct 2026 15:21:57 +0800 Subject: [PATCH 26/29] Revert unneeded test adaptations for queries without Collect fan-in Audit against the PR base: these queries are single-device or single-region (no multi-child Collect) or have a deterministic total order above the fan-in, so ready-first consumption does not change their output order. Restore the original ordered assertions and queries. --- .../it/db/it/IoTDBPatternAggregationIT.java | 63 ++--- .../it/db/it/IoTDBPatternRecognitionIT.java | 254 +++++++++--------- .../relational/it/db/it/IoTDBWindowTVFIT.java | 67 ++--- .../query/old/IoTDBDatetimeFormatTableIT.java | 3 +- .../IoTDBOrderByWithAlignByDeviceTableIT.java | 2 +- .../scalar/IoTDBDiffFunctionTableIT.java | 6 +- .../scalar/IoTDBRoundFunctionTableIT.java | 80 ++---- ...IoTDBCorrelatedQuantifiedComparisonIT.java | 67 ++--- ...DBOrderByWithAlignByDeviceTableViewIT.java | 2 +- 9 files changed, 227 insertions(+), 317 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java index 222eb37a9eacb..e2000380b48db 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternAggregationIT.java @@ -31,12 +31,8 @@ import org.junit.runner.RunWith; import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; import java.sql.Statement; -import java.util.Arrays; -import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -209,7 +205,7 @@ public void test1() { "2025-01-01T00:06:00.000Z,2,2,3,20.0,10.0,30.0,60.0,15.0,20.0,20.0,", "2025-01-01T00:07:00.000Z,2,3,3,30.0,10.0,60.0,60.0,20.0,20.0,30.0,", }; - assertUnorderedTableResult( + tableResultSetEqualTest( "SELECT m.time, m.match, m.count1, m.count2, m.max, m.min, m.sum1, m.sum2, m.avg1, m.avg2, m.totalprice " + "FROM t1 " + "MATCH_RECOGNIZE ( " @@ -233,7 +229,8 @@ public void test1() { + " D AS D.totalprice = 40 " + ") AS m ", expectedHeader, - retArray); + retArray, + DATABASE_NAME); } @Test @@ -262,7 +259,7 @@ public void test2() { "2025-01-01T00:06:00.000Z,2,B,2,2,2,0,1,1,1,20.0,", "2025-01-01T00:07:00.000Z,2,C,3,3,3,1,1,1,1,30.0,", }; - assertUnorderedTableResult( + tableResultSetEqualTest( "SELECT m.time, m.match, m.label, m.count_0, m.count_1, m.count_2, m.count_c, m.final_count_c, m.count_u, m.final_count_u, m.totalprice " + "FROM t1 " + "MATCH_RECOGNIZE ( " @@ -287,7 +284,8 @@ public void test2() { + " D AS D.totalprice = 40 " + ") AS m ", expectedHeader, - retArray); + retArray, + DATABASE_NAME); } @Test @@ -314,7 +312,7 @@ public void test3() { "2025-01-01T00:06:00.000Z,2,B,30.0,0.0,30.0,20.0,20.0,20.0,", "2025-01-01T00:07:00.000Z,2,C,60.0,30.0,30.0,20.0,20.0,30.0,", }; - assertUnorderedTableResult( + tableResultSetEqualTest( "SELECT m.time, m.match, m.label, m.sum, m.sum_c, m.final_sum_c, m.sum_u, m.final_sum_u, m.totalprice " + "FROM t1 " + "MATCH_RECOGNIZE ( " @@ -337,7 +335,8 @@ public void test3() { + " D AS D.totalprice = 40 " + ") AS m ", expectedHeader, - retArray); + retArray, + DATABASE_NAME); } @Test @@ -364,7 +363,7 @@ public void test4() { "2025-01-01T00:06:00.000Z,2,B,15.0,0.0,30.0,20.0,20.0,20.0,", "2025-01-01T00:07:00.000Z,2,C,20.0,30.0,30.0,20.0,20.0,30.0,", }; - assertUnorderedTableResult( + tableResultSetEqualTest( "SELECT m.time, m.match, m.label, m.avg, m.avg_c, m.final_avg_c, m.avg_u, m.final_avg_u, m.totalprice " + "FROM t1 " + "MATCH_RECOGNIZE ( " @@ -387,7 +386,8 @@ public void test4() { + " D AS D.totalprice = 40 " + ") AS m ", expectedHeader, - retArray); + retArray, + DATABASE_NAME); } @Test @@ -406,7 +406,7 @@ public void test5() { "2025-01-01T00:06:00.000Z,2,2025-01-01T00:05:00.000Z,2025-01-01T00:06:00.000Z,2025-01-01T00:06:00.000Z,2025-01-01T00:05:00.000Z,10.0,20.0,", "2025-01-01T00:07:00.000Z,2,2025-01-01T00:05:00.000Z,2025-01-01T00:07:00.000Z,2025-01-01T00:07:00.000Z,2025-01-01T00:05:00.000Z,10.0,30.0," }; - assertUnorderedTableResult( + tableResultSetEqualTest( "SELECT m.time, m.match, m.firstTime, m.lastTime, m.maxTime, m.minTime, m.firstVal, m.lastVal " + "FROM t1 " + "MATCH_RECOGNIZE ( " @@ -428,7 +428,8 @@ public void test5() { + " D AS totalprice = 40 " + ") AS m ", expectedHeader, - retArray); + retArray, + DATABASE_NAME); } @Test @@ -452,7 +453,7 @@ public void test6() { "2025-01-01T00:05:00.000Z,1,7.0,-8.0,3.2,40.7,40.7,32.56,6.379655,6.379655,5.706137," }; - assertUnorderedTableResult( + tableResultSetEqualTest( "SELECT m.time, m.match, m.mode, m.extreme, " + "ROUND(m.avg, 6) AS avg, " + "ROUND(m.var_0, 6) AS var_0, ROUND(m.var_1, 6) AS var_1, ROUND(m.var_2, 6) AS var_2, " @@ -477,7 +478,8 @@ public void test6() { + " A AS true " + ") AS m ", expectedHeader, - retArray); + retArray, + DATABASE_NAME); } @Test @@ -491,7 +493,7 @@ public void testAggregationsInDefineClause() { "2025-01-01T00:03:00.000Z,1,A,5.0,4.0,5.0,", "2025-01-01T00:04:00.000Z,1,B,7.0,8.5,13.0,", }; - assertUnorderedTableResult( + tableResultSetEqualTest( "SELECT m.time, m.match, m.label, m.avg, m.running_avg_b, m.totalprice " + "FROM t2 " + "MATCH_RECOGNIZE ( " @@ -507,7 +509,8 @@ public void testAggregationsInDefineClause() { + " A AS AVG(totalprice) = 5 " + ") AS m ", expectedHeader, - retArray); + retArray, + DATABASE_NAME); } @Test @@ -576,28 +579,4 @@ public void testDifferentCast() { retArray, DATABASE_NAME); } - - /** - * Asserts the query result with order-insensitive, multiplicity-preserving comparison. - * - *

These queries deliberately have no final ORDER BY: adding one (e.g. a final ORDER BY on - * m.time) would create an outer full Sort whose static sort-buffer reservation can exhaust the - * fragment instance's operators memory block and trigger error 719 ("There is not enough memory - * to execute current fragment instance") when fragment-instance memory releases are delayed under - * the tests' tight memory configuration. Unordered assertions keep the tests deterministic - * without the Sort. - */ - private static void assertUnorderedTableResult( - String sql, String[] expectedHeader, String[] retArray) { - String header = String.join(",", expectedHeader) + ","; - try (Connection connection = EnvFactory.getEnv().getTableConnection(); - Statement statement = connection.createStatement()) { - statement.execute("use " + DATABASE_NAME); - try (ResultSet resultSet = statement.executeQuery(sql)) { - assertResultSetEqualUnordered(resultSet, header, Arrays.asList(retArray)); - } - } catch (SQLException e) { - fail(e.getMessage()); - } - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java index 9a3d17f20fa20..070ba0b6348b4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBPatternRecognitionIT.java @@ -31,13 +31,9 @@ import org.junit.runner.RunWith; import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; import java.sql.Statement; -import java.util.Arrays; import static java.lang.String.format; -import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -283,24 +279,32 @@ public void testOutputMode() { + " B AS B.totalprice < PREV(B.totalprice) " + ") AS m"; - assertUnorderedTableResult( - format(sql, "ONE ROW PER MATCH", "PATTERN (B*)"), expectedHeader, retArray1); + tableResultSetEqualTest( + format(sql, "ONE ROW PER MATCH", "PATTERN (B*)"), expectedHeader, retArray1, DATABASE_NAME); - assertUnorderedTableResult( - format(sql, "ALL ROWS PER MATCH", "PATTERN (B*)"), expectedHeader, retArray2); + tableResultSetEqualTest( + format(sql, "ALL ROWS PER MATCH", "PATTERN (B*)"), + expectedHeader, + retArray2, + DATABASE_NAME); - assertUnorderedTableResult( - format(sql, "ALL ROWS PER MATCH", "PATTERN (B+)"), expectedHeader, retArray3); + tableResultSetEqualTest( + format(sql, "ALL ROWS PER MATCH", "PATTERN (B+)"), + expectedHeader, + retArray3, + DATABASE_NAME); - assertUnorderedTableResult( + tableResultSetEqualTest( format(sql, "ALL ROWS PER MATCH OMIT EMPTY MATCHES", "PATTERN (B*)"), expectedHeader, - retArray4); + retArray4, + DATABASE_NAME); - assertUnorderedTableResult( + tableResultSetEqualTest( format(sql, "ALL ROWS PER MATCH WITH UNMATCHED ROWS", "PATTERN (B+)"), expectedHeader, - retArray5); + retArray5, + DATABASE_NAME); } @Test @@ -357,35 +361,41 @@ public void testLogicalNavigationFunction() { + " A AS true " + ") AS m"; // LAST(totalprice) - assertUnorderedTableResult(format(sql, "totalprice"), expectedHeader, retArray1); - assertUnorderedTableResult(format(sql, "RPR_LAST(totalprice)"), expectedHeader, retArray1); - assertUnorderedTableResult(format(sql, "RPR_LAST(totalprice, 0)"), expectedHeader, retArray1); - assertUnorderedTableResult( - format(sql, "RUNNING RPR_LAST(totalprice)"), expectedHeader, retArray1); + tableResultSetEqualTest(format(sql, "totalprice"), expectedHeader, retArray1, DATABASE_NAME); + tableResultSetEqualTest( + format(sql, "RPR_LAST(totalprice)"), expectedHeader, retArray1, DATABASE_NAME); + tableResultSetEqualTest( + format(sql, "RPR_LAST(totalprice, 0)"), expectedHeader, retArray1, DATABASE_NAME); + tableResultSetEqualTest( + format(sql, "RUNNING RPR_LAST(totalprice)"), expectedHeader, retArray1, DATABASE_NAME); // FINAL LAST(totalprice) - assertUnorderedTableResult( - format(sql, "FINAL RPR_LAST(totalprice)"), expectedHeader, retArray2); + tableResultSetEqualTest( + format(sql, "FINAL RPR_LAST(totalprice)"), expectedHeader, retArray2, DATABASE_NAME); // FIRST(totalprice) - assertUnorderedTableResult(format(sql, "RPR_FIRST(totalprice)"), expectedHeader, retArray3); - assertUnorderedTableResult(format(sql, "RPR_FIRST(totalprice, 0)"), expectedHeader, retArray3); - assertUnorderedTableResult( - format(sql, "RUNNING RPR_FIRST(totalprice)"), expectedHeader, retArray3); - assertUnorderedTableResult( - format(sql, "FINAL RPR_FIRST(totalprice)"), expectedHeader, retArray3); + tableResultSetEqualTest( + format(sql, "RPR_FIRST(totalprice)"), expectedHeader, retArray3, DATABASE_NAME); + tableResultSetEqualTest( + format(sql, "RPR_FIRST(totalprice, 0)"), expectedHeader, retArray3, DATABASE_NAME); + tableResultSetEqualTest( + format(sql, "RUNNING RPR_FIRST(totalprice)"), expectedHeader, retArray3, DATABASE_NAME); + tableResultSetEqualTest( + format(sql, "FINAL RPR_FIRST(totalprice)"), expectedHeader, retArray3, DATABASE_NAME); // LAST(totalprice, 2) - assertUnorderedTableResult(format(sql, "RPR_LAST(totalprice, 2)"), expectedHeader, retArray4); + tableResultSetEqualTest( + format(sql, "RPR_LAST(totalprice, 2)"), expectedHeader, retArray4, DATABASE_NAME); // FINAL LAST(totalprice, 2) - assertUnorderedTableResult( - format(sql, "FINAL RPR_LAST(totalprice, 2)"), expectedHeader, retArray5); + tableResultSetEqualTest( + format(sql, "FINAL RPR_LAST(totalprice, 2)"), expectedHeader, retArray5, DATABASE_NAME); // FIRST(totalprice, 2) - assertUnorderedTableResult(format(sql, "RPR_FIRST(totalprice, 2)"), expectedHeader, retArray6); - assertUnorderedTableResult( - format(sql, "FINAL RPR_FIRST(totalprice, 2)"), expectedHeader, retArray6); + tableResultSetEqualTest( + format(sql, "RPR_FIRST(totalprice, 2)"), expectedHeader, retArray6, DATABASE_NAME); + tableResultSetEqualTest( + format(sql, "FINAL RPR_FIRST(totalprice, 2)"), expectedHeader, retArray6, DATABASE_NAME); } @Test @@ -446,18 +456,24 @@ public void testPhysicalNavigationFunction() { + " A AS true " + ") AS m"; // PREV(totalprice) - assertUnorderedTableResult(format(sql1, "PREV(totalprice)"), expectedHeader1, retArray1); - assertUnorderedTableResult(format(sql1, "PREV(totalprice, 1)"), expectedHeader1, retArray1); + tableResultSetEqualTest( + format(sql1, "PREV(totalprice)"), expectedHeader1, retArray1, DATABASE_NAME); + tableResultSetEqualTest( + format(sql1, "PREV(totalprice, 1)"), expectedHeader1, retArray1, DATABASE_NAME); // PREV(totalprice, 2) - assertUnorderedTableResult(format(sql1, "PREV(totalprice, 2)"), expectedHeader1, retArray2); + tableResultSetEqualTest( + format(sql1, "PREV(totalprice, 2)"), expectedHeader1, retArray2, DATABASE_NAME); // NEXT(totalprice) - assertUnorderedTableResult(format(sql1, "NEXT(totalprice)"), expectedHeader1, retArray3); - assertUnorderedTableResult(format(sql1, "NEXT(totalprice, 1)"), expectedHeader1, retArray3); + tableResultSetEqualTest( + format(sql1, "NEXT(totalprice)"), expectedHeader1, retArray3, DATABASE_NAME); + tableResultSetEqualTest( + format(sql1, "NEXT(totalprice, 1)"), expectedHeader1, retArray3, DATABASE_NAME); // NEXT(totalprice, 2) - assertUnorderedTableResult(format(sql1, "NEXT(totalprice, 2)"), expectedHeader1, retArray4); + tableResultSetEqualTest( + format(sql1, "NEXT(totalprice, 2)"), expectedHeader1, retArray4, DATABASE_NAME); String[] expectedHeader2 = new String[] {"price"}; String sql2 = @@ -574,60 +590,60 @@ public void testNestedNavigation() { + " A AS true " + ") AS m"; // PREV(FIRST(totalprice)) - assertUnorderedTableResult( - format(sql, "PREV(RPR_FIRST(totalprice))"), expectedHeader, retArray1); + tableResultSetEqualTest( + format(sql, "PREV(RPR_FIRST(totalprice))"), expectedHeader, retArray1, DATABASE_NAME); // PREV(FIRST(totalprice), 2) - assertUnorderedTableResult( - format(sql, "PREV(RPR_FIRST(totalprice), 2)"), expectedHeader, retArray1); + tableResultSetEqualTest( + format(sql, "PREV(RPR_FIRST(totalprice), 2)"), expectedHeader, retArray1, DATABASE_NAME); // PREV(FIRST(totalprice, 2)) - assertUnorderedTableResult( - format(sql, "PREV(RPR_FIRST(totalprice, 2))"), expectedHeader, retArray2); + tableResultSetEqualTest( + format(sql, "PREV(RPR_FIRST(totalprice, 2))"), expectedHeader, retArray2, DATABASE_NAME); // PREV(FIRST(totalprice, 2), 2) - assertUnorderedTableResult( - format(sql, "PREV(RPR_FIRST(totalprice, 2), 2)"), expectedHeader, retArray3); + tableResultSetEqualTest( + format(sql, "PREV(RPR_FIRST(totalprice, 2), 2)"), expectedHeader, retArray3, DATABASE_NAME); // PREV(LAST(totalprice)) - assertUnorderedTableResult( - format(sql, "PREV(RPR_LAST(totalprice))"), expectedHeader, retArray4); + tableResultSetEqualTest( + format(sql, "PREV(RPR_LAST(totalprice))"), expectedHeader, retArray4, DATABASE_NAME); // PREV(LAST(totalprice), 2) - assertUnorderedTableResult( - format(sql, "PREV(RPR_LAST(totalprice), 2)"), expectedHeader, retArray5); + tableResultSetEqualTest( + format(sql, "PREV(RPR_LAST(totalprice), 2)"), expectedHeader, retArray5, DATABASE_NAME); // PREV(LAST(totalprice, 1)) - assertUnorderedTableResult( - format(sql, "PREV(RPR_LAST(totalprice, 1))"), expectedHeader, retArray5); + tableResultSetEqualTest( + format(sql, "PREV(RPR_LAST(totalprice, 1))"), expectedHeader, retArray5, DATABASE_NAME); // PREV(LAST(totalprice, 1), 2) - assertUnorderedTableResult( - format(sql, "PREV(RPR_LAST(totalprice, 1), 2)"), expectedHeader, retArray1); + tableResultSetEqualTest( + format(sql, "PREV(RPR_LAST(totalprice, 1), 2)"), expectedHeader, retArray1, DATABASE_NAME); // NEXT(FIRST(totalprice)) - assertUnorderedTableResult( - format(sql, "NEXT(RPR_FIRST(totalprice))"), expectedHeader, retArray2); + tableResultSetEqualTest( + format(sql, "NEXT(RPR_FIRST(totalprice))"), expectedHeader, retArray2, DATABASE_NAME); // NEXT(FIRST(totalprice), 2) - assertUnorderedTableResult( - format(sql, "NEXT(RPR_FIRST(totalprice), 2)"), expectedHeader, retArray6); + tableResultSetEqualTest( + format(sql, "NEXT(RPR_FIRST(totalprice), 2)"), expectedHeader, retArray6, DATABASE_NAME); // NEXT(FIRST(totalprice, 1)) - assertUnorderedTableResult( - format(sql, "NEXT(RPR_FIRST(totalprice, 1))"), expectedHeader, retArray6); + tableResultSetEqualTest( + format(sql, "NEXT(RPR_FIRST(totalprice, 1))"), expectedHeader, retArray6, DATABASE_NAME); // NEXT(FIRST(totalprice, 1), 2) - assertUnorderedTableResult( - format(sql, "NEXT(RPR_FIRST(totalprice, 1), 2)"), expectedHeader, retArray1); + tableResultSetEqualTest( + format(sql, "NEXT(RPR_FIRST(totalprice, 1), 2)"), expectedHeader, retArray1, DATABASE_NAME); // NEXT(LAST(totalprice)) - assertUnorderedTableResult( - format(sql, "NEXT(RPR_LAST(totalprice))"), expectedHeader, retArray7); + tableResultSetEqualTest( + format(sql, "NEXT(RPR_LAST(totalprice))"), expectedHeader, retArray7, DATABASE_NAME); // NEXT(LAST(totalprice), 2) - assertUnorderedTableResult( - format(sql, "NEXT(RPR_LAST(totalprice), 2)"), expectedHeader, retArray8); + tableResultSetEqualTest( + format(sql, "NEXT(RPR_LAST(totalprice), 2)"), expectedHeader, retArray8, DATABASE_NAME); // NEXT(LAST(totalprice, 2)) - assertUnorderedTableResult( - format(sql, "NEXT(RPR_LAST(totalprice, 2))"), expectedHeader, retArray9); + tableResultSetEqualTest( + format(sql, "NEXT(RPR_LAST(totalprice, 2))"), expectedHeader, retArray9, DATABASE_NAME); // NEXT(LAST(totalprice, 2), 2) - assertUnorderedTableResult( - format(sql, "NEXT(RPR_LAST(totalprice, 2), 2)"), expectedHeader, retArray10); + tableResultSetEqualTest( + format(sql, "NEXT(RPR_LAST(totalprice, 2), 2)"), expectedHeader, retArray10, DATABASE_NAME); } @Test @@ -663,7 +679,7 @@ public void testUnionVariable() { + " H AS H.totalprice > 80 " + ") AS m"; - assertUnorderedTableResult(sql, expectedHeader, retArray1); + tableResultSetEqualTest(sql, expectedHeader, retArray1, DATABASE_NAME); } @Test @@ -699,7 +715,7 @@ public void testClassifierFunction() { + " H AS H.totalprice > 80 " + ") AS m"; - assertUnorderedTableResult(sql, expectedHeader, retArray1); + tableResultSetEqualTest(sql, expectedHeader, retArray1, DATABASE_NAME); } @Test @@ -784,7 +800,7 @@ public void testRowPattern() { DATABASE_NAME); // pattern concatenation - assertUnorderedTableResult( + tableResultSetEqualTest( format( sql, "PATTERN (A B C) " @@ -792,10 +808,11 @@ public void testRowPattern() { + " B AS B.totalprice < PREV (B.totalprice), " + " C AS C.totalprice = PREV (C.totalprice)"), expectedHeader, - retArray3); + retArray3, + DATABASE_NAME); // pattern alternation - assertUnorderedTableResult( + tableResultSetEqualTest( format( sql, "PATTERN (B | C | A) " @@ -803,10 +820,11 @@ public void testRowPattern() { + " B AS B.totalprice < PREV (B.totalprice), " + " C AS C.totalprice <= PREV (C.totalprice)"), expectedHeader, - retArray4); + retArray4, + DATABASE_NAME); // pattern permutation - assertUnorderedTableResult( + tableResultSetEqualTest( format( sql, "PATTERN (PERMUTE(B, C)) " @@ -814,10 +832,11 @@ public void testRowPattern() { + " B AS B.totalprice < PREV (B.totalprice), " + " C AS C.totalprice < PREV (C.totalprice)"), expectedHeader, - retArray5); + retArray5, + DATABASE_NAME); // grouped pattern - assertUnorderedTableResult( + tableResultSetEqualTest( format( sql, "PATTERN (((A) (B (C)))) " @@ -825,7 +844,8 @@ public void testRowPattern() { + " B AS B.totalprice < PREV (B.totalprice), " + " C AS C.totalprice = PREV (C.totalprice)"), expectedHeader, - retArray3); + retArray3, + DATABASE_NAME); } @Test @@ -885,29 +905,35 @@ public void testPatternQuantifier() { + " B AS B.totalprice <= PREV(B.totalprice) " + ") AS m"; - assertUnorderedTableResult(format(sql, "PATTERN (B*)"), expectedHeader, retArray1); + tableResultSetEqualTest(format(sql, "PATTERN (B*)"), expectedHeader, retArray1, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B*?)"), expectedHeader, retArray2); + tableResultSetEqualTest(format(sql, "PATTERN (B*?)"), expectedHeader, retArray2, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B+)"), expectedHeader, retArray3); + tableResultSetEqualTest(format(sql, "PATTERN (B+)"), expectedHeader, retArray3, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B+?)"), expectedHeader, retArray4); + tableResultSetEqualTest(format(sql, "PATTERN (B+?)"), expectedHeader, retArray4, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B?)"), expectedHeader, retArray5); + tableResultSetEqualTest(format(sql, "PATTERN (B?)"), expectedHeader, retArray5, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B??)"), expectedHeader, retArray2); + tableResultSetEqualTest(format(sql, "PATTERN (B??)"), expectedHeader, retArray2, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{,})"), expectedHeader, retArray1); + tableResultSetEqualTest( + format(sql, "PATTERN (B{,})"), expectedHeader, retArray1, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{,}?)"), expectedHeader, retArray2); + tableResultSetEqualTest( + format(sql, "PATTERN (B{,}?)"), expectedHeader, retArray2, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{1,})"), expectedHeader, retArray3); + tableResultSetEqualTest( + format(sql, "PATTERN (B{1,})"), expectedHeader, retArray3, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{1,}?)"), expectedHeader, retArray4); + tableResultSetEqualTest( + format(sql, "PATTERN (B{1,}?)"), expectedHeader, retArray4, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{2,})"), expectedHeader, retArray3); + tableResultSetEqualTest( + format(sql, "PATTERN (B{2,})"), expectedHeader, retArray3, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{2,}?)"), expectedHeader, retArray6); + tableResultSetEqualTest( + format(sql, "PATTERN (B{2,}?)"), expectedHeader, retArray6, DATABASE_NAME); tableResultSetEqualTest( format(sql, "PATTERN (B{5,})"), expectedHeader, new String[] {}, DATABASE_NAME); @@ -915,40 +941,22 @@ public void testPatternQuantifier() { tableResultSetEqualTest( format(sql, "PATTERN (B{5,}?)"), expectedHeader, new String[] {}, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{,1})"), expectedHeader, retArray5); - - assertUnorderedTableResult(format(sql, "PATTERN (B{,1}?)"), expectedHeader, retArray2); + tableResultSetEqualTest( + format(sql, "PATTERN (B{,1})"), expectedHeader, retArray5, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{1,1})"), expectedHeader, retArray4); + tableResultSetEqualTest( + format(sql, "PATTERN (B{,1}?)"), expectedHeader, retArray2, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{1,1}?)"), expectedHeader, retArray4); + tableResultSetEqualTest( + format(sql, "PATTERN (B{1,1})"), expectedHeader, retArray4, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{1})"), expectedHeader, retArray4); + tableResultSetEqualTest( + format(sql, "PATTERN (B{1,1}?)"), expectedHeader, retArray4, DATABASE_NAME); - assertUnorderedTableResult(format(sql, "PATTERN (B{1}?)"), expectedHeader, retArray4); - } + tableResultSetEqualTest( + format(sql, "PATTERN (B{1})"), expectedHeader, retArray4, DATABASE_NAME); - /** - * Asserts the query result with order-insensitive, multiplicity-preserving comparison. - * - *

These queries deliberately have no final ORDER BY: adding one (e.g. a final ORDER BY on - * m.time) would create an outer full Sort whose static sort-buffer reservation can exhaust the - * fragment instance's operators memory block and trigger error 719 ("There is not enough memory - * to execute current fragment instance") when fragment-instance memory releases are delayed under - * the tests' tight memory configuration. Unordered assertions keep the tests deterministic - * without the Sort. - */ - private static void assertUnorderedTableResult( - String sql, String[] expectedHeader, String[] retArray) { - String header = String.join(",", expectedHeader) + ","; - try (Connection connection = EnvFactory.getEnv().getTableConnection(); - Statement statement = connection.createStatement()) { - statement.execute("use " + DATABASE_NAME); - try (ResultSet resultSet = statement.executeQuery(sql)) { - assertResultSetEqualUnordered(resultSet, header, Arrays.asList(retArray)); - } - } catch (SQLException e) { - fail(e.getMessage()); - } + tableResultSetEqualTest( + format(sql, "PATTERN (B{1}?)"), expectedHeader, retArray4, DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java index b6f720d7c0ee8..83488cf6930df 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java @@ -304,7 +304,7 @@ public void testVariationWithMultiTypeAndNullValue() { "3,2021-01-01T09:05:00.000Z,device1,3," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, long_val FROM variation(multi_type ORDER BY time, 'long_val', 1.0, false) ORDER BY window_index, time", + "SELECT window_index, time, device_id, long_val FROM variation(multi_type ORDER BY time, 'long_val', 1.0, false)", expectedHeader, retArray, DATABASE_NAME); @@ -321,7 +321,7 @@ public void testVariationWithMultiTypeAndNullValue() { "1,2021-01-01T09:05:00.000Z,device1,3.0," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, float_val FROM variation(multi_type ORDER BY time, 'float_val', 1.0, true) ORDER BY window_index, time", + "SELECT window_index, time, device_id, float_val FROM variation(multi_type ORDER BY time, 'float_val', 1.0, true)", expectedHeader, retArray, DATABASE_NAME); @@ -338,7 +338,7 @@ public void testVariationWithMultiTypeAndNullValue() { "0,2021-01-01T09:05:00.000Z,device1,3.0," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, double_val FROM variation(multi_type ORDER BY time, 'double_val', 1.0, true) ORDER BY window_index, time", + "SELECT window_index, time, device_id, double_val FROM variation(multi_type ORDER BY time, 'double_val', 1.0, true)", expectedHeader, retArray, DATABASE_NAME); @@ -355,7 +355,7 @@ public void testVariationWithMultiTypeAndNullValue() { "1,2021-01-01T09:05:00.000Z,device1,false," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, bool_val FROM variation(multi_type ORDER BY time, 'bool_val', 0.0, true) ORDER BY window_index, time", + "SELECT window_index, time, device_id, bool_val FROM variation(multi_type ORDER BY time, 'bool_val', 0.0, true)", expectedHeader, retArray, DATABASE_NAME); @@ -372,7 +372,7 @@ public void testVariationWithMultiTypeAndNullValue() { "1,2021-01-01T09:05:00.000Z,device1,2," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, str_val FROM variation(multi_type ORDER BY time, 'str_val', 0.0, true) ORDER BY window_index, time", + "SELECT window_index, time, device_id, str_val FROM variation(multi_type ORDER BY time, 'str_val', 0.0, true)", expectedHeader, retArray, DATABASE_NAME); @@ -389,7 +389,7 @@ public void testVariationWithMultiTypeAndNullValue() { "4,2021-01-01T09:05:00.000Z,device1,0x02," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, blob_val FROM variation(multi_type ORDER BY time, 'blob_val', 0.0, false) ORDER BY window_index, time", + "SELECT window_index, time, device_id, blob_val FROM variation(multi_type ORDER BY time, 'blob_val', 0.0, false)", expectedHeader, retArray, DATABASE_NAME); @@ -406,7 +406,7 @@ public void testVariationWithMultiTypeAndNullValue() { "3,2021-01-01T09:05:00.000Z,device1,2021-01-01T10:00:00.000Z," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, ts_val FROM variation(DATA=>multi_type ORDER BY time, COL=>'ts_val', ignore_null=>false) ORDER BY window_index, time", + "SELECT window_index, time, device_id, ts_val FROM variation(DATA=>multi_type ORDER BY time, COL=>'ts_val', ignore_null=>false)", expectedHeader, retArray, DATABASE_NAME); @@ -423,7 +423,7 @@ public void testVariationWithMultiTypeAndNullValue() { "1,2021-01-01T09:05:00.000Z,device1,2021-01-02," }; tableResultSetEqualTest( - "SELECT window_index, time, device_id, date_val FROM variation(multi_type ORDER BY time, 'date_val', 0.0) ORDER BY window_index, time", + "SELECT window_index, time, device_id, date_val FROM variation(multi_type ORDER BY time, 'date_val', 0.0)", expectedHeader, retArray, DATABASE_NAME); @@ -779,7 +779,7 @@ public void testPatternMatchFunction() { "2,0.6797687270429319,1970-01-01T00:00:00.052Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false) ORDER BY match_index, time", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false)", expectedHeader, retArray, DATABASE_NAME); @@ -810,7 +810,7 @@ public void testPatternMatchFunction() { "1,0.6797687270429319,1970-01-01T00:00:00.052Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 1.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false) ORDER BY match_index, time", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 1.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false)", expectedHeader, retArray, DATABASE_NAME); @@ -837,7 +837,7 @@ public void testPatternMatchFunction() { "1,0.6465473220616865,1970-01-01T00:00:00.021Z,1.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 10.0, height => 500.0, smooth_On_Pattern => false) ORDER BY match_index, time", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 10.0, height => 500.0, smooth_On_Pattern => false)", expectedHeader, retArray, DATABASE_NAME); @@ -855,7 +855,7 @@ public void testPatternMatchFunction() { "0,4.637180787192477,1970-01-01T00:00:00.008Z,1.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 1000.0, height => 2.0, smooth_On_Pattern => false) ORDER BY match_index, time", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,2.0,3.0,2.0,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 10.0, width => 1000.0, height => 2.0, smooth_On_Pattern => false)", expectedHeader, retArray, DATABASE_NAME); @@ -908,7 +908,7 @@ public void testPatternMatchFunction() { "3,0.6797687270429319,1970-01-01T00:00:00.052Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,{2.0,3.0,2.0}+,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false) ORDER BY match_index, time", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,{2.0,3.0,2.0}+,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false)", expectedHeader, retArray, DATABASE_NAME); @@ -955,7 +955,7 @@ public void testPatternMatchFunction() { "3,0.0,1970-01-01T00:00:00.052Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,1.0,{1.0,2.0,3.0,2.0}*,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false) ORDER BY match_index, time", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,1.0,{1.0,2.0,3.0,2.0}*,1.0,1.0,1.0,1.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => false)", expectedHeader, retArray, DATABASE_NAME); @@ -985,7 +985,7 @@ public void testPatternMatchFunction() { "3,1.7704361692956723,1970-01-01T00:00:00.046Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1,2.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => true) ORDER BY match_index, time", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1,2.0,1.0', smooth => 0.5, threshold => 100.0, width => 1000.0, height => 500.0, smooth_On_Pattern => true)", expectedHeader, retArray, DATABASE_NAME); @@ -1013,7 +1013,7 @@ public void testPatternMatchFunction() { "2,0.09999999999999994,1970-01-01T00:00:00.052Z,2.0,0,", }; tableResultSetEqualByDataTypeTest( - "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,1.2,1.0', smooth => 0.5, threshold => 1.0, width => 1000.0, height => 500.0, smooth_On_Pattern => true) ORDER BY match_index, time", + "select * from pattern_match(data => t1 ORDER BY time, time_col => 'time', data_col => 'value', pattern => '1.0,1.2,1.0', smooth => 0.5, threshold => 1.0, width => 1000.0, height => 500.0, smooth_On_Pattern => true)", expectedHeader, retArray, DATABASE_NAME); @@ -1547,10 +1547,12 @@ public void testXCorrP0() { "TESL,29997.0,", "TESL,19890.0," }; - assertUnorderedTableResult( - "SELECT * FROM XCORR(DATA => bid PARTITION BY stock_id ORDER BY time, TIMECOL => 'time')", + tableResultSetEqualTest( + "SELECT * FROM XCORR(DATA => bid PARTITION BY stock_id ORDER BY time, " + + "TIMECOL => 'time') ORDER BY stock_id", expectedHeader, - retArray); + retArray, + DATABASE_NAME); } @Test @@ -1568,12 +1570,13 @@ public void testXCorrWithNullValues() { "device_1,120.0,", "device_1,480.0," }; - assertUnorderedTableResult( + tableResultSetEqualTest( "SELECT * FROM XCORR(DATA => " + "(SELECT time, device_id, s1, s2 FROM table1 WHERE device_id = 'device_1') " + "PARTITION BY device_id ORDER BY time, TIMECOL => 'time')", expectedHeader, - retArray); + retArray, + DATABASE_NAME); } @Test @@ -1591,12 +1594,14 @@ public void testXCorrWithMultiplePartitionColumns() { "F1,device_2,500.0,", "F2,device_1,900.0," }; - assertUnorderedTableResult( + tableResultSetEqualTest( "SELECT * FROM XCORR(DATA => " + "(SELECT time, factory_id, device_id, s1, s1 AS s2 FROM table5) " - + "PARTITION BY (factory_id, device_id) ORDER BY time, TIMECOL => 'time')", + + "PARTITION BY (factory_id, device_id) ORDER BY time, TIMECOL => 'time') " + + "ORDER BY factory_id, device_id", expectedHeader, - retArray); + retArray, + DATABASE_NAME); } @Test @@ -1701,18 +1706,4 @@ private static void tableResultSetEqualWithTolerance( fail(e.getMessage()); } } - - private static void assertUnorderedTableResult( - String sql, String[] expectedHeader, String[] retArray) { - String header = String.join(",", expectedHeader) + ","; - try (Connection connection = EnvFactory.getEnv().getTableConnection(); - Statement statement = connection.createStatement()) { - statement.execute("use " + DATABASE_NAME); - try (ResultSet resultSet = statement.executeQuery(sql)) { - assertResultSetEqualUnordered(resultSet, header, Arrays.asList(retArray)); - } - } catch (SQLException e) { - fail(e.getMessage()); - } - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java index 03166ce0fc9aa..08e9ccc8635ad 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBDatetimeFormatTableIT.java @@ -111,8 +111,7 @@ public void testDatetimeInputFormat() { } try (ResultSet resultSet = - statement.executeQuery( - "SELECT time, s1 FROM table1 where device_id='d1' ORDER BY time")) { + statement.executeQuery("SELECT time, s1 FROM table1 where device_id='d1'")) { Assert.assertNotNull(resultSet); int cnt = 0; while (resultSet.next()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java index db0a7364867aa..5c9ade9530b52 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableIT.java @@ -228,7 +228,7 @@ public void overFlowTest() { } tableResultSetEqualTest( - "SELECT time, value FROM overflow ORDER BY time", expectedHeader, retArray, DATABASE_NAME); + "SELECT time, value FROM overflow", expectedHeader, retArray, DATABASE_NAME); } // ORDER BY DEVICE diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java index 2d64f67dd4dd6..2eaa96e5b85ee 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBDiffFunctionTableIT.java @@ -102,7 +102,7 @@ public void testNewTransformerIgnoreNull() { "1970-02-27T20:53:20.001Z,3.0,null," }; tableResultSetEqualTest( - "select time,diff(s1), diff(s2) from (select * from table1 where device_id='d1' order by time) order by time", + "select time,diff(s1), diff(s2) from table1 where device_id='d1'", expectedHeader, retArray, DATABASE_NAME); @@ -123,7 +123,7 @@ public void testNewTransformerRespectNull() { "1970-02-27T20:53:20.001Z,null,null," }; tableResultSetEqualTest( - "select time, Diff(s1, false), diff(s2, false) from (select * from table1 where device_id='d1' order by time) order by time", + "select time, Diff(s1, false), diff(s2, false) from table1 where device_id='d1'", expectedHeader, retArray, DATABASE_NAME); @@ -141,7 +141,7 @@ public void testCaseInSensitive() { "1970-01-01T00:00:00.005Z,5,5.0," }; tableResultSetEqualTest( - "select time, s1, s2 from (select * from table1 where device_id='d1' order by time) where diff(s1) between 1 and 2 order by time", + "select time, s1, s2 from table1 where device_id='d1' and diff(s1) between 1 and 2", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java index 37d63edde2223..8c0c74bf36a9f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBRoundFunctionTableIT.java @@ -99,10 +99,7 @@ public void testRound() { "2023-03-13T08:16:19.764Z,2.0," }; tableResultSetEqualTest( - "select time, round(s1) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s1) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -115,10 +112,7 @@ public void testRound() { "2023-03-13T08:16:19.764Z,12345.0," }; tableResultSetEqualTest( - "select time, round(s2) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s2) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -131,10 +125,7 @@ public void testRound() { "2023-03-13T08:16:19.764Z,121.0," }; tableResultSetEqualTest( - "select time, round(s3) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s3) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -147,10 +138,7 @@ public void testRound() { "2023-03-13T08:16:19.764Z,10.0," }; tableResultSetEqualTest( - "select time, round(s4) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s4) from table1", expectedHeader, intRetArray, DATABASE_NAME); } @Test @@ -167,10 +155,7 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,2.0," }; tableResultSetEqualTest( - "select time, round(s1, 1) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s1, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -183,10 +168,7 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,12345.0," }; tableResultSetEqualTest( - "select time, round(s2, 1) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s2, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -199,10 +181,7 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,120.5," }; tableResultSetEqualTest( - "select time, round(s3, 1) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s3, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -215,10 +194,7 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,120.51," }; tableResultSetEqualTest( - "select time, round(s3, 2) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s3, 2) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -231,10 +207,7 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,120.511," }; tableResultSetEqualTest( - "select time, round(s3, 3) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s3, 3) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -247,10 +220,7 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,120.5112," }; tableResultSetEqualTest( - "select time, round(s3, 4) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s3, 4) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -263,10 +233,7 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,10.14342," }; tableResultSetEqualTest( - "select time, round(s4, 5) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s4, 5) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -279,10 +246,7 @@ public void testRoundWithPlaces() { "2023-03-13T08:16:19.764Z,10.1," }; tableResultSetEqualTest( - "select time, round(s4, 1) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s4, 1) from table1", expectedHeader, intRetArray, DATABASE_NAME); } @Test @@ -300,10 +264,7 @@ public void testRoundWithNegativePlaces() { "2023-03-13T08:16:19.764Z,0.0," }; tableResultSetEqualTest( - "select time, round(s1, -1) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s1, -1) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -316,10 +277,7 @@ public void testRoundWithNegativePlaces() { "2023-03-13T08:16:19.764Z,12340.0," }; tableResultSetEqualTest( - "select time, round(s2, -1) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s2, -1) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -332,10 +290,7 @@ public void testRoundWithNegativePlaces() { "2023-03-13T08:16:19.764Z,120.0," }; tableResultSetEqualTest( - "select time, round(s3,-1) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s3,-1) from table1", expectedHeader, intRetArray, DATABASE_NAME); intRetArray = new String[] { @@ -348,10 +303,7 @@ public void testRoundWithNegativePlaces() { "2023-03-13T08:16:19.764Z,10.0," }; tableResultSetEqualTest( - "select time, round(s4, -1) from table1 ORDER BY time", - expectedHeader, - intRetArray, - DATABASE_NAME); + "select time, round(s4, -1) from table1", expectedHeader, intRetArray, DATABASE_NAME); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java index 1098b62bf16f5..c4c41f2e196d7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedQuantifiedComparisonIT.java @@ -30,20 +30,12 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.List; - -import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; -import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -65,77 +57,80 @@ public static void tearDown() throws Exception { @Test public void testCorrelatedQuantifiedComparisonInWhereClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; sql = "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= any(SELECT %s from table3 t3 where t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } sql = "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= some(SELECT %s from table3 t3 where t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } sql = "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= all(SELECT %s from table3 t3 where t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testQuantifiedComparisonInSelectClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; sql = "SELECT %s >= any(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.s1 = t3.s1) from table1 t1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } sql = "SELECT %s >= some(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.s1 = t3.s1) from table1 t1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } sql = "SELECT %s >= all(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.s1 = t3.s1) from table1 t1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"true,", "true,", "true,", "true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } } @@ -155,18 +150,4 @@ public void testCorrelatedQuantifiedComparisonLegalityCheck() { "Decorrelation for LIMIT with row count greater than 1 is not supported yet", DATABASE_NAME); } - - private static void assertUnorderedResult( - String sql, String expectedHeader, List expectedRows) { - try (Connection connection = EnvFactory.getEnv().getTableConnection(); - Statement statement = connection.createStatement()) { - statement.execute("USE " + DATABASE_NAME); - try (ResultSet resultSet = statement.executeQuery(sql)) { - assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); - } - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java index e4d99ec0a442a..8dbf6b892f01e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBOrderByWithAlignByDeviceTableViewIT.java @@ -249,7 +249,7 @@ public void overFlowTest() { } tableResultSetEqualTest( - "SELECT time, value FROM overflow ORDER BY time", expectedHeader, retArray, DATABASE_NAME); + "SELECT time, value FROM overflow", expectedHeader, retArray, DATABASE_NAME); } // ORDER BY DEVICE From ffbee7c20b3c16afbf433b9e224f2670439c23ff Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 21 Oct 2026 15:22:00 +0800 Subject: [PATCH 27/29] Check partial-column insert results on every node ready-first Collect makes the cross-node row order of the multi-device vehicle1 scan nondeterministic, and the cluster framework's cross-node comparison throws InconsistentDataException. Check each node's rows independently instead. --- .../it/insertquery/IoTDBInsertQueryIT.java | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java index 75a7725c41498..0d7c40cd0d123 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/insertquery/IoTDBInsertQueryIT.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Locale; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -269,10 +270,14 @@ public void testPartialColumn() throws SQLException { // It is allowed to insert without tags statement.execute("INSERT INTO vehicle1(time, s1, s2) SELECT time, s1, s2 FROM vehicle0"); - ResultSet resultSet = statement.executeQuery("SELECT deviceId, manufacturer FROM vehicle1"); - while (resultSet.next()) { - for (int i = 1; i <= 2; i++) { - assertNull(resultSet.getString(i)); + try (ResultSet clusterResult = + statement.executeQuery("SELECT deviceId, manufacturer FROM vehicle1")) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + while (resultSet.next()) { + for (int i = 1; i <= 2; i++) { + assertNull(resultSet.getString(i)); + } + } } } @@ -280,13 +285,17 @@ public void testPartialColumn() throws SQLException { statement.execute("DELETE FROM vehicle1"); statement.execute( "INSERT INTO vehicle1(time, s2, manufacturer) SELECT time, s2, manufacturer FROM vehicle0"); - resultSet = statement.executeQuery("SELECT deviceId, manufacturer, s1, s2 FROM vehicle1"); - while (resultSet.next()) { - for (int i = 1; i <= 4; i++) { - if (i % 2 == 1) { - assertNull(resultSet.getString(i)); - } else { - assertNotNull(resultSet.getString(i)); + try (ResultSet clusterResult = + statement.executeQuery("SELECT deviceId, manufacturer, s1, s2 FROM vehicle1")) { + for (ResultSet resultSet : getNodeResultSets(clusterResult)) { + while (resultSet.next()) { + for (int i = 1; i <= 4; i++) { + if (i % 2 == 1) { + assertNull(resultSet.getString(i)); + } else { + assertNotNull(resultSet.getString(i)); + } + } } } } From 8e691ee6141bafbdcceea2c77ddc3b9bb40f480d Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 21 Oct 2026 18:01:02 +0800 Subject: [PATCH 28/29] Revert test adaptations not affected by ready-first Collect Audit against the distributed plans of every adapted query: the reverted queries contain no multi-child CollectNode (their fan-ins are MergeSort nodes with final ORDER BY, SemiJoin Sort nodes, or single-region scans after tag pruning), so ready-first consumption does not change their output order. Keep unordered assertions only where the plan really fans in through a ready-first Collect. --- .../IoTDBMultiTAGsWithAttributesTableIT.java | 5 +- .../it/db/it/IoTDBWindowFunction3IT.java | 2 +- .../relational/it/db/it/IoTDBWindowTVFIT.java | 6 +- .../udf/IoTDBUserDefinedTableFunctionIT.java | 8 +- .../IoTDBAlignByDeviceTableIT.java | 8 +- .../it/query/recent/IoTDBCollectFanInIT.java | 11 +- .../IoTDBCorrelatedExistsSubqueryIT.java | 183 +++++++++--------- .../IoTDBCorrelatedScalarSubqueryIT.java | 139 ++++++------- .../IoTDBUncorrelatedExistsSubqueryIT.java | 180 +++++++++-------- ...oTDBUncorrelatedInPredicateSubqueryIT.java | 127 ++++++------ ...TDBUncorrelatedQuantifiedComparisonIT.java | 174 +++++++++-------- .../IoTDBUncorrelatedScalarSubqueryIT.java | 135 +++++++------ .../IoTDBAlignByDeviceTableViewIT.java | 8 +- .../recent/IoTDBTableViewAggregationIT.java | 2 +- .../IoTDBTableViewAggregationNonStreamIT.java | 2 +- .../apache/iotdb/calc/i18n/CalcMessages.java | 1 + 16 files changed, 500 insertions(+), 491 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java index f5af1271ea4a0..1b293f75865c3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java @@ -2087,7 +2087,7 @@ public void fourTableJoinTest() { + " c.course_name," + " g.score " + "from students s, teachers t, courses c, grades g " - + "where s.time=c.time and c.time=g.time and t.teacher_tag = 1005 order by g.score desc limit 1"; + + "where s.time=c.time and c.time=g.time and t.teacher_tag = 1005 limit 1"; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @@ -2196,8 +2196,7 @@ public void innerJoinOnMultiColumns() { expectedHeader = new String[] {"attr1", "attr2"}; sql = - "select t0.attr1,t0.attr2 from table0 t0 join table1 t1 on t0.attr1=t1.attr1 AND t0.attr2=t1.attr2" - + " ORDER BY t0.attr1, t0.attr2"; + "select t0.attr1,t0.attr2 from table0 t0 join table1 t1 on t0.attr1=t1.attr1 AND t0.attr2=t1.attr2"; retArray = new String[] { "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunction3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunction3IT.java index ff13d8fc154c4..9922948550478 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunction3IT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunction3IT.java @@ -346,7 +346,7 @@ public void testReplaceWindowWithRowNumber() { "2021-01-01T09:15:00.000Z,d2,4.0,2,", }; tableResultSetEqualTest( - "SELECT *, row_number() OVER (PARTITION BY device ORDER BY time) AS rn FROM demo ORDER BY device, time", + "SELECT *, row_number() OVER (PARTITION BY device) AS rn FROM demo ORDER BY device, time", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java index 83488cf6930df..2c8a822fb09de 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowTVFIT.java @@ -142,7 +142,7 @@ public void testHopFunction() { "2021-01-01T09:15:00.000Z,2021-01-01T09:25:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", }; tableResultSetEqualTest( - "SELECT * FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => 5m, SIZE => 10m) ORDER BY stock_id, time, window_start, window_end", + "SELECT * FROM HOP(DATA => bid, TIMECOL => 'time', SLIDE => 5m, SIZE => 10m) ORDER BY stock_id, time", expectedHeader, retArray, DATABASE_NAME); @@ -671,7 +671,7 @@ public void testCumulateFunction() { "2021-01-01T09:12:00.000Z,2021-01-01T09:24:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", }; tableResultSetEqualTest( - "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m) ORDER BY stock_id, time, window_start, window_end", + "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m) ORDER BY stock_id, time", expectedHeader, retArray, DATABASE_NAME); @@ -694,7 +694,7 @@ public void testCumulateFunction() { "2021-01-01T09:06:00.000Z,2021-01-01T09:18:00.000Z,2021-01-01T09:15:00.000Z,TESL,195.0,332.0,", }; tableResultSetEqualTest( - "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m, ORIGIN => 2021-01-01T09:06:00) ORDER BY stock_id, time, window_start, window_end", + "SELECT * FROM CUMULATE(DATA => bid, TIMECOL => 'time', STEP => 6m, SIZE => 12m, ORIGIN => 2021-01-01T09:06:00) ORDER BY stock_id, time", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java index 849f76dd72076..25967b8caeb83 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/IoTDBUserDefinedTableFunctionIT.java @@ -130,12 +130,12 @@ public void testMySplit() { "2,2,", "4,4,", }; tableResultSetEqualTest( - "select * from TABLE(SPLIT('1,2,4,5')) a(o1) join TABLE(SPLIT('2、3、4', '、')) b(o2) on a.o1=b.o2 ORDER BY o1", + "select * from TABLE(SPLIT('1,2,4,5')) a(o1) join TABLE(SPLIT('2、3、4', '、')) b(o2) on a.o1=b.o2", expectedHeader, retArray, DATABASE_NAME); tableResultSetEqualTest( - "select * from SPLIT('1,2,4,5') a(o1) join TABLE(SPLIT('2、3、4', '、')) b(o2) on a.o1=b.o2 ORDER BY o1", + "select * from SPLIT('1,2,4,5') a(o1) join TABLE(SPLIT('2、3、4', '、')) b(o2) on a.o1=b.o2", expectedHeader, retArray, DATABASE_NAME); @@ -332,10 +332,10 @@ public void testPassThroughPartitionColumn() { String[] expectedHeader = new String[] {"s1", "device_id"}; String[] retArray = new String[] { - "1,d0,", "3,d0,", "null,d0,", "4,d1,", + "1,d0,", "null,d0,", "3,d0,", "4,d1,", }; tableResultSetEqualTest( - "select * from MY_SELECT(vehicle PARTITION BY device_id, 's1') ORDER BY device_id, s1", + "select * from MY_SELECT(vehicle PARTITION BY device_id, 's1') ORDER BY device_id", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java index 89ed30f43c977..3a882b7118c47 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceTableIT.java @@ -141,7 +141,7 @@ public void selectTest() { "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", }; tableResultSetEqualTest( - "select * from vehicle order by device_id, time", expectedHeader, retArray, DATABASE_NAME); + "select * from vehicle order by device_id", expectedHeader, retArray, DATABASE_NAME); } @Test @@ -208,7 +208,7 @@ public void selectWithDuplicatedPathsTest() { "1970-01-01T00:00:01.000Z,d1,888,888,null,", }; tableResultSetEqualTest( - "select time, device_id, s0,s0,s1 from vehicle where device_id = 'd0' or device_id = 'd1' order by device_id, time", + "select time, device_id, s0,s0,s1 from vehicle where device_id = 'd0' or device_id = 'd1' order by device_id", expectedHeader, retArray, DATABASE_NAME); @@ -272,7 +272,7 @@ public void selectDifferentSeriesWithValueFilterWithoutCacheTest() { "1999-12-31T16:00:00.000Z,d0,null,", }; tableResultSetEqualTest( - "select time, device_id, s0 from vehicle where s1 < 200 order by device_id, time", + "select time, device_id, s0 from vehicle where s1 < 200 order by device_id", expectedHeader, retArray, DATABASE_NAME); @@ -530,7 +530,7 @@ public void selectWithRegularExpressionTest() { "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", }; tableResultSetEqualTest( - "select * from vehicle where device_id like 'd%' order by device_id, time", + "select * from vehicle where device_id like 'd%' order by device_id", expectedHeader, retArray, DATABASE_NAME); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java index 8afd6dc97515e..745ec969d181d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBCollectFanInIT.java @@ -19,7 +19,6 @@ package org.apache.iotdb.relational.it.query.recent; -import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; @@ -115,7 +114,7 @@ public void testUnorderedUnionPreservesEveryDuplicateOnEachNode() throws Excepti } @Test - public void testPipeVerificationQueryUsesDeterministicPrimaryKeyOrder() throws Exception { + public void testPipeVerificationQueryRowsAreCompletePerNode() throws Exception { TableModelUtils.createDataBaseAndTable(EnvFactory.getEnv(), "pipe_readings", DATABASE); TableModelUtils.insertData(DATABASE, "pipe_readings", 0, DEVICES * ROWS, EnvFactory.getEnv()); List expected = @@ -124,11 +123,11 @@ public void testPipeVerificationQueryUsesDeterministicPrimaryKeyOrder() throws E try (Connection connection = EnvFactory.getEnv().getTableConnection(); Statement statement = connection.createStatement()) { statement.execute("USE " + DATABASE); + // The pipe verification query has no ORDER BY and fans in through a ready-first Collect, + // so each node's row set must be checked independently without relying on row order. try (ResultSet result = - statement.executeQuery( - "select s0,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,time from pipe_readings " - + "order by s0,s1,s2,s3,time")) { - TestUtils.assertResultSetEqual(result, TableModelUtils.generateHeaderResults(), expected); + statement.executeQuery(TableModelUtils.getQuerySql("pipe_readings"))) { + assertResultSetEqualUnordered(result, TableModelUtils.generateHeaderResults(), expected); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java index 326b22d0790b2..4951fd6670a02 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedExistsSubqueryIT.java @@ -31,20 +31,12 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.List; - -import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; -import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -65,7 +57,7 @@ public static void tearDown() throws Exception { @Test public void testCorrelatedExistsSubqueryInWhereClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: exists and other filter @@ -73,29 +65,31 @@ public void testCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: only exists sql = "SELECT s1 FROM table3 t3 WHERE exists(SELECT s1 from table1 t1 WHERE t1.s1 = t3.s1)"; retArray = new String[] {"30,", "30,", "40,"}; - expectedHeader = "s1,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); // Test case: exists with distinct sql = "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: aggregation in exists @@ -103,11 +97,12 @@ public void testCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(min(%s) as INT32) as %s FROM table1 t1 WHERE exists(SELECT avg(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s)"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: aggregation with group by in exists(subquery returns empty result with having @@ -116,11 +111,12 @@ public void testCorrelatedExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE exists(SELECT count(*) from table3 t3 where t1.%s = t3.%s group by device_id having count(*) > 5)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: limit 1 in exists @@ -128,60 +124,61 @@ public void testCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s limit 1)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testNestedExistsSubquery() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: Nested exists sql = "select distinct s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 = t3.s1 and exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25))"; retArray = new String[] {"30,"}; - expectedHeader = "s1,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); // Test case: Nested exists with not(64 - 4 = 60 rows remain) sql = "select count(*) as cnt from table1 t1 where not exists(select s1 from table3 t3 where t1.s1 = t3.s1 and exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25))"; retArray = new String[] {"60,"}; - expectedHeader = "cnt,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"cnt"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test public void testMultipleExistsSubquery() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: multiple exists sql = "select distinct s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 = t3.s1) and exists(select s1 from table2 t2 where t2.s1 = t1.s1 - 25)"; retArray = new String[] {"30,"}; - expectedHeader = "s1,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); // Test case: multiple exists with not sql = "select distinct s1 from table1 t1 where exists(select s1 from table3 t3 where t1.s1 = t3.s1) and not exists(select s1 from table2 t2 where t2.s1 = t1.s1)"; retArray = new String[] {"30,", "40,"}; - expectedHeader = "s1,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubquery() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: with InPredicate @@ -189,8 +186,8 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and %s in (select %s from table3)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -201,7 +198,8 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with InPredicate in exists @@ -209,8 +207,8 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s not in (select %s from table2 where %s is not null))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -222,7 +220,8 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with Scalar Subquery @@ -230,12 +229,13 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 > (select min(%s) from table3)"; retArray = new String[] {"40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with Scalar Subquery in exists @@ -243,12 +243,13 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and s1 = (select min(%s) from table3))"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with Quantified Comparison @@ -256,12 +257,13 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 != any(select %s from table2)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with Quantified Comparison in exists @@ -269,8 +271,8 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s = any (select %s from table3))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -281,14 +283,15 @@ public void testCorrelatedExistsSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testCorrelatedExistsSubqueryWithMultipleCorrelation() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: Multiple correlation in exists @@ -296,89 +299,91 @@ public void testCorrelatedExistsSubqueryWithMultipleCorrelation() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(SELECT (%s) from table3 t3 WHERE t1.%s = t3.%s and t1.s1 = t3.s1 and t1.s2 = t3.s2)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } sql = "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and not exists(SELECT (%s) from table3 t3 WHERE t1.%s = t3.%s and t1.s1 = t3.s1 and t1.s2 = t3.s2)"; retArray = new String[] {"50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testCorrelatedExistsSubqueryInHavingClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: exists in having sql = "SELECT device_id, count(*) from table1 t1 group by device_id having exists(SELECT 1 from table3 t3 where t3.device_id = t1.device_id)"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] {"d01,5,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); } sql = "SELECT device_id, count(*) from table1 t1 group by device_id having exists(SELECT 1 from table3 t3 where t3.device_id != 'd01' and t3.device_id = t1.device_id)"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: not exists in having sql = "SELECT device_id, count(*) from table1 t1 group by device_id having count(*) = 5 and not exists(SELECT 1 from table3 t3 where t3.device_id = t1.device_id)"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] {"d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); } } @Test public void testCorrelatedExistsSubqueryInSelectClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: exists in Select clause sql = "select exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3 where exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25)"; retArray = new String[] {"true,", "true,"}; - expectedHeader = "_col0,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); sql = "select exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3"; retArray = new String[] {"true,", "true,", "true,", "false,"}; - expectedHeader = "_col0,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); sql = "select not exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3 where exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25)"; retArray = new String[] {"false,", "false,"}; - expectedHeader = "_col0,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); sql = "select not exists(select s1 from table1 t1 where t1.s1 = t3.s1) from table3 t3"; retArray = new String[] {"false,", "false,", "false,", "true,"}; - expectedHeader = "_col0,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test @@ -423,18 +428,4 @@ public void testCorrelatedExistsLegalityCheck() { "Decorrelation for LIMIT with row count greater than 1 is not supported yet", DATABASE_NAME); } - - private static void assertUnorderedResult( - String sql, String expectedHeader, List expectedRows) { - try (Connection connection = EnvFactory.getEnv().getTableConnection(); - Statement statement = connection.createStatement()) { - statement.execute("USE " + DATABASE_NAME); - try (ResultSet resultSet = statement.executeQuery(sql)) { - assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); - } - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java index 3b4aff7ebed58..071635f53f6c8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/correlated/IoTDBCorrelatedScalarSubqueryIT.java @@ -31,20 +31,12 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.List; - -import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; -import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -66,7 +58,7 @@ public static void tearDown() throws Exception { @Test public void testCorrelatedScalarSubqueryInWhereClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: Aggregation with correlated filter in scalar subquery @@ -74,12 +66,13 @@ public void testCorrelatedScalarSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT max(%s) from table3 t3 where t1.%s = t3.%s)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: Non-Aggregation with correlated filter in scalar subquery @@ -87,8 +80,8 @@ public void testCorrelatedScalarSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT distinct %s from table3 t3 where t1.%s = t3.%s and %s > 30)"; retArray = new String[] {"40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -99,7 +92,8 @@ public void testCorrelatedScalarSubqueryInWhereClause() { measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: limit 1 in scalar subquery @@ -107,8 +101,8 @@ public void testCorrelatedScalarSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s >= (SELECT %s from table3 t3 where t1.%s = t3.%s and %s > 30 limit 1)"; retArray = new String[] {"40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -119,50 +113,51 @@ public void testCorrelatedScalarSubqueryInWhereClause() { measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testNestedCorrelatedScalarSubquery() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: Nested exists sql = "select distinct s1 from table1 t1 where s1 >= (select max(s1) from table3 t3 where t1.s1 = t3.s1 and s1 = (select max(s1) from table1 t1_2 where t1_2.s1 = t3.s1))"; retArray = new String[] {"30,", "40,"}; - expectedHeader = "s1,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test public void testMultipleScalarSubquery() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: multiple scalar subquery sql = "select distinct s1 from table1 t1 where s1 = (select max(s1) from table3 t3 where t1.s1 = t3.s1) and s1 = (select min(s1) from table3 t3 where t1.s1 = t3.s1)"; retArray = new String[] {"30,", "40,"}; - expectedHeader = "s1,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"s1"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test public void testCorrelatedScalarSubqueryInWhereClauseWithOtherCorrelatedSubquery() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: with Exists sql = "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and exists(select s1 from table3 t3 where t1.s1 = t3.s1)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -174,14 +169,15 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherCorrelatedSubquery measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubquery() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: with In Predicate @@ -189,8 +185,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and %s in (select %s from table3)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -202,7 +198,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with InPredicate in scalar subquery @@ -210,8 +207,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s not in (select %s from table2 where %s is not null))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -224,7 +221,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with Scalar Subquery @@ -232,8 +230,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 > (select min(%s) from table3)"; retArray = new String[] {"40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -244,7 +242,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with nested Scalar Subquery @@ -252,8 +251,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT (%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and s1 = (select min(%s) from table3))"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -264,7 +263,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with Quantified Comparison @@ -272,8 +272,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s) and s1 != any(select %s from table2)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -284,7 +284,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with Quantified Comparison in scalar subquery @@ -292,8 +293,8 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE device_id = 'd01' and t1.%s = t3.%s and %s = any (select %s from table3))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -305,14 +306,15 @@ public void testCorrelatedScalarSubqueryInWhereClauseWithOtherUncorrelatedSubque measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testCorrelatedScalarSubqueryWithMultipleCorrelation() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: Multiple correlation in exists @@ -320,49 +322,50 @@ public void testCorrelatedScalarSubqueryWithMultipleCorrelation() { "SELECT cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table3 t3 WHERE t1.%s = t3.%s and t1.s1 = t3.s1 and t1.s2 = t3.s2)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testCorrelatedScalarSubqueryInHavingClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: scalar subquery in having sql = "SELECT device_id, count(*) from table1 t1 group by device_id having count(*) + 35 = (SELECT max(s1) from table3 t3 where t3.device_id = t1.device_id)"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] {"d01,5,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); } } @Test public void testCorrelatedScalarSubqueryInSelectClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: exists in Select clause sql = "select s1 = (select max(s1) from table1 t1 where t1.s1 = t3.s1) from table3 t3 where exists(select s1 from table2 t2 where t2.s1 = t3.s1 - 25)"; retArray = new String[] {"true,", "true,"}; - expectedHeader = "_col0,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); sql = "select (select max(s1) from table1 t1 where t1.s1 = t3.s1) from table3 t3"; retArray = new String[] {"30,", "30,", "40,", "null,"}; - expectedHeader = "_col0,"; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {"_col0"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test @@ -414,18 +417,4 @@ public void testCorrelatedScalarSubqueryLegalityCheck() { "701: Scalar sub-query has returned multiple rows", DATABASE_NAME); } - - private static void assertUnorderedResult( - String sql, String expectedHeader, List expectedRows) { - try (Connection connection = EnvFactory.getEnv().getTableConnection(); - Statement statement = connection.createStatement()) { - statement.execute("USE " + DATABASE_NAME); - try (ResultSet resultSet = statement.executeQuery(sql)) { - assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); - } - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java index 136243fb2a81a..216f176ed4747 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedExistsSubqueryIT.java @@ -39,6 +39,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; @@ -63,7 +64,7 @@ public static void tearDown() throws Exception { @Test public void testUnCorrelatedExistsSubqueryInWhereClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: aggregation in exist subquery @@ -71,9 +72,9 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: group by in exist subquery @@ -81,9 +82,9 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1 group by device_id)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: expression in select clause with exist subquery @@ -91,9 +92,9 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) + 1 as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1 group by device_id)"; retArray = new String[] {"31,", "41,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: exists with other conjuncts @@ -101,11 +102,12 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE %s = 30 and exists(select max(s1) from table1)"; retArray = new String[] {"30,", "30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: false exist @@ -113,11 +115,12 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE %s = 30 and exists(select s1 from table1 where s1 < 0)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: limit in exist subquery @@ -125,11 +128,12 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select s1 from table1 limit 10)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: Nested exists @@ -137,11 +141,12 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select s1 from table1 where exists(select s1 from table2))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: Multiple exists @@ -149,18 +154,19 @@ public void testUnCorrelatedExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select avg(s1) from table1 limit 1) and exists(select s1 from table1 where exists(select s1 from table2))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testUnCorrelatedNotExistsSubqueryInWhereClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: aggregation in exist subquery @@ -168,9 +174,9 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and not exists(select max(s1) from table1)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: limit in exist subquery @@ -178,9 +184,9 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE not exists(select s1 from table1 limit 1)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: subquery has output with not exists @@ -188,11 +194,12 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table3 t3 WHERE %s = 30 and not exists(select s1 from table1 where s1 < 0)"; retArray = new String[] {"30,", "30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with distinct @@ -200,11 +207,12 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and not exists(select s1 from table1 where s1 < 0)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: Nested exists @@ -212,11 +220,12 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and exists(select s1 from table1 where not exists(select s1 from table1 where s1 < 0))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: Multiple exists @@ -224,18 +233,19 @@ public void testUnCorrelatedNotExistsSubqueryInWhereClause() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE %s is not null and not exists(select s1 from table1 where s1 < 0) and exists(select s1 from table1 where not exists(select s1 from table1 where s1 < 0))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: with InPredicate @@ -243,9 +253,9 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(select max(s1) from table1) and s1 in (select s1 from table3)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: with InPredicate in exists @@ -253,9 +263,9 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table3 t3 WHERE device_id = 'd01' and exists(select max(s1) from table1 where s1 in (select s1 from table3))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: with Scalar Subquery @@ -263,11 +273,12 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(select s1 from table1) and s1 = (select min(s1) from table3)"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with Scalar Subquery in exists @@ -275,11 +286,12 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s < 50 and exists(select s1 from table1 where s1 = (select min(s1) from table3))"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with Quantified Comparison @@ -287,11 +299,12 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and exists(select s1 from table1) and s1 = any(select s1 from table3)"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: with Quantified Comparison in exists @@ -299,69 +312,74 @@ public void testUnCorrelatedExistsSubqueryInWhereClauseWithOtherSubquery() { "SELECT distinct cast(%s AS INT32) as %s FROM table1 t1 WHERE device_id = 'd01' and %s = 30 and exists(select s1 from table1 where s1 = any (select s1 from table3))"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testUnCorrelatedExistsSubqueryInHavingClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: exists in having sql = "SELECT device_id, count(*) from table1 group by device_id having exists(SELECT %s from table1) and count(*) = 5"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; for (String measurement : NUMERIC_MEASUREMENTS) { assertUnorderedResult( - String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); + String.format(sql, measurement), + String.join(",", expectedHeader) + ",", + Arrays.asList(retArray)); } sql = "SELECT device_id, count(*) from table1 group by device_id having exists(SELECT %s from table1 where s1 < 0)"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: not exists in having sql = "SELECT device_id, count(*) from table1 group by device_id having not exists(SELECT %s from table1 where s1 < 0) and count(*) = 5"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; for (String measurement : NUMERIC_MEASUREMENTS) { assertUnorderedResult( - String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); + String.format(sql, measurement), + String.join(",", expectedHeader) + ",", + Arrays.asList(retArray)); } sql = "SELECT device_id, count(*) from table1 group by device_id having not exists(SELECT %s from table1) and count(*) = 5"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement), expectedHeader, retArray, DATABASE_NAME); } } @Test public void testUnCorrelatedExistsSubqueryInSelectClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: exists in Select clause @@ -369,18 +387,18 @@ public void testUnCorrelatedExistsSubqueryInSelectClause() { "SELECT exists(select max(s1) from table1) as %s FROM table3 t3 WHERE %s is not null and device_id = 'd01'"; retArray = new String[] {"true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } sql = "SELECT not exists(select max(s1) from table1) as %s FROM table3 t3 WHERE %s is not null and device_id = 'd01'"; retArray = new String[] {"false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java index db8c2d4bbd235..add410b3bee17 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -30,20 +30,12 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.List; - -import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; -import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -64,7 +56,7 @@ public static void tearDown() throws Exception { @Test public void testInPredicateSubqueryInWhereClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: where s in (subquery) @@ -72,11 +64,12 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: where s not in (subquery) @@ -84,11 +77,12 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {"50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: where s in (subquery), subquery returns empty set @@ -96,11 +90,12 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: where s not in (subquery), subquery returns empty set. Should return all rows @@ -108,11 +103,12 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: where s in (subquery), subquery contains scalar subquery. @@ -120,11 +116,12 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT min(%s) from table3 WHERE device_id = 'd01')"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: where s in (subquery), subquery contains scalar subquery. @@ -132,12 +129,13 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01' and (%s) > (select avg(%s) from table3 where device_id = 'd01'))"; retArray = new String[] {"40,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: where s in (subquery), subquery contains expression. @@ -145,116 +143,117 @@ public void testInPredicateSubqueryInWhereClause() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s AS INT32) in (SELECT cast(((%s) + 30) AS INT32) from table3 WHERE device_id = 'd01')"; retArray = new String[] {"60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: where s in (subquery), s contains null value sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 in (SELECT s1 from table3)"; - expectedHeader = "s1,"; + expectedHeader = new String[] {"s1"}; retArray = new String[] {"30,"}; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); // Test case: where s not in (subquery), s contains null value, the resutl should be empty sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 not in (SELECT s1 from table3)"; - expectedHeader = "s1,"; + expectedHeader = new String[] {"s1"}; retArray = new String[] {}; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test public void testInPredicateSubqueryInHavingClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: having s in (subquery) sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); // Test case: having s not in (subquery) sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 30 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01') and count(*) > 3"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); // Test case: having s in (subquery), subquery returns empty set sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd010')"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] {}; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); // Test case: having s not in (subquery), subquery returns empty set, should return all rows sql = "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd11') and count(*) > 3"; - expectedHeader = "device_id,_col1,"; + expectedHeader = new String[] {"device_id", "_col1"}; retArray = new String[] { "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," }; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test public void testInPredicateSubqueryInSelectClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: select s in (subquery) sql = "SELECT %s in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s not in (subquery) sql = "SELECT %s not in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"false,", "false,", "true,", "true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s in (subquery), s contains null value. The result should also be null when // s is null. sql = "SELECT s1 in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"true,", "null,"}; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); // Test case: select s in (subquery), s contains null value. The result should also be null when // s is null. sql = "SELECT s1 not in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"false,", "null,"}; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); // Test case: select s in (subquery), s contains null value. 36 not in(30, 40, null) returns // null sql = "SELECT s1 not in (SELECT s1 from table3) from table1 where device_id = 'd02'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"null,", "false,", "null,"}; - assertUnorderedResult(sql, expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @Test @@ -293,18 +292,4 @@ public void testInPredicateSubqueryLegalityCheck() { tableAssertTestFail( "select s1 from table1 where s1 in (select s1 from)", "mismatched input", DATABASE_NAME); } - - private static void assertUnorderedResult( - String sql, String expectedHeader, List expectedRows) { - try (Connection connection = EnvFactory.getEnv().getTableConnection(); - Statement statement = connection.createStatement()) { - statement.execute("USE " + DATABASE_NAME); - try (ResultSet resultSet = statement.executeQuery(sql)) { - assertResultSetEqualUnordered(resultSet, expectedHeader, expectedRows); - } - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java index 5bb2424936078..7e0da3cedb38d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedQuantifiedComparisonIT.java @@ -40,6 +40,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; @@ -65,7 +66,8 @@ public static void tearDown() throws Exception { @Test public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { String sql; - String expectedHeader; + String expectedHeaderStr; + String[] expectedHeader; String[] retArray; // Test case: where s > any (subquery), s does not contain null value @@ -73,10 +75,10 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -85,10 +87,10 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -97,10 +99,10 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -109,10 +111,10 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -121,10 +123,10 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -133,10 +135,10 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -145,10 +147,10 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -157,10 +159,10 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -169,11 +171,12 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: where s = some (subquery), s does not contain null value @@ -181,11 +184,12 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: where s != any (subquery), s does not contain null value @@ -193,10 +197,10 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != any (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -205,10 +209,10 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != some (SELECT %s FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } } @@ -216,7 +220,8 @@ public void testAnyAndSomeComparisonInWhereClauseWithoutNull() { @Test public void testAllComparisonInWhereClauseWithoutNull() { String sql; - String expectedHeader; + String expectedHeaderStr; + String[] expectedHeader; String[] retArray; // Test case: where s > all (subquery), s does not contain null value @@ -224,10 +229,10 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {"50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -236,10 +241,10 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {"40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -248,10 +253,10 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -260,10 +265,10 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -272,10 +277,10 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -284,11 +289,12 @@ public void testAllComparisonInWhereClauseWithoutNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != all (SELECT %s FROM table3 WHERE device_id = 'd01')"; retArray = new String[] {"50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @@ -374,7 +380,8 @@ public void testAnyAndSomeComparisonInWhereClauseWithNull() { @Test public void testAllComparisonInWhereClauseWithNull() { String sql; - String expectedHeader; + String expectedHeaderStr; + String[] expectedHeader; String[] retArray; // Test case: where s1 > all (subquery), s1 in table3 contains null value. @@ -382,10 +389,10 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -394,10 +401,10 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -406,10 +413,10 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -418,10 +425,10 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s <= all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -430,10 +437,10 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; + expectedHeaderStr = measurement + ","; assertUnorderedResult( String.format(sql, measurement, measurement, measurement, measurement), - expectedHeader, + expectedHeaderStr, Arrays.asList(retArray)); } @@ -442,11 +449,12 @@ public void testAllComparisonInWhereClauseWithNull() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s as INT32) != all (SELECT s1 FROM table3)"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @@ -544,82 +552,82 @@ public void testQuantifiedComparisonInHavingClause() { public void testQuantifiedComparisonInSelectClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: select s > any(subquery) sql = "SELECT %s > any(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"false,", "true,", "true,", "true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s > some(subquery) sql = "SELECT %s > some(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"false,", "true,", "true,", "true,", "true,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s > all(subquery) sql = "SELECT %s > all(SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"false,", "false,", "false,", "false,", "false,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s < any(subquery), subquery contains null value sql = "SELECT %s < any(SELECT (%s) from table3) from table1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"null,", "null,", "null,", "null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s < some(subquery), subquery contains null value sql = "SELECT %s < some(SELECT (%s) from table3) from table1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"null,", "null,", "null,", "null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s <= any(subquery), subquery contains null value sql = "SELECT %s <= any(SELECT (%s) from table3) from table1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"true,", "null,", "null,", "null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s <= some(subquery), subquery contains null value sql = "SELECT %s <= some(SELECT (%s) from table3) from table1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"true,", "null,", "null,", "null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s != all(subquery), subquery contains null value sql = "SELECT %s != all(SELECT (%s) from table3) from table1 where device_id = 'd01'"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"false,", "false,", "null,", "null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } // Test case: select s != all(subquery), subquery result contains null value and s not in @@ -627,11 +635,11 @@ public void testQuantifiedComparisonInSelectClause() { // value result set sql = "SELECT %s != all(SELECT (%s) from table3 where device_id = 'd_null') from table1 where device_id = 'd02' and %s != 30"; - expectedHeader = "_col0,"; + expectedHeader = new String[] {"_col0"}; retArray = new String[] {"null,", "null,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - assertUnorderedResult( - String.format(sql, measurement, measurement), expectedHeader, Arrays.asList(retArray)); + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java index b98c240d5bbb9..4068e53a75d9d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java @@ -40,6 +40,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqualUnordered; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; @@ -66,7 +67,7 @@ public static void tearDown() throws Exception { @Test public void testScalarSubqueryAfterComparisonInOneTable() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: s equals to the maximum value of s in table1 @@ -74,11 +75,12 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 WHERE device_id = 'd01')"; retArray = new String[] {"70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: s not equals to the maximum value of s in table1 @@ -86,11 +88,12 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s != ((SELECT max(%s) FROM table1 WHERE device_id = 'd01'))"; retArray = new String[] {"30,", "40,", "50,", "60,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: s greater than the average value of s in table1 @@ -98,11 +101,12 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s >= ((SELECT AVG(%s) FROM table1 WHERE device_id = 'd01'))"; retArray = new String[] {"50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: s greater than the max value of s in table1 @@ -110,11 +114,12 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > ((SELECT max(%s) FROM table1 WHERE device_id = 'd01'))"; retArray = new String[] {}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: s is less than the maximum value of s in table1 and greater than the minimum value @@ -123,12 +128,13 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < (SELECT max(%s) from table1 WHERE device_id = 'd01') and %s > (SELECT min(%s) from table1 WHERE device_id = 'd01') "; retArray = new String[] {"40,", "50,", "60,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: s greater than the avg value of s in table1 and s5 = true @@ -136,11 +142,12 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > ((SELECT avg(%s) FROM table1 WHERE device_id = 'd01' and s5 = true))"; retArray = new String[] {"60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: s greater than the count value of s in table1 @@ -148,11 +155,12 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > (SELECT count(%s) FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: s less than the sum value of s in table1 @@ -160,11 +168,12 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < (SELECT sum(%s) FROM table1 WHERE device_id = 'd01')"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: subquery is not aggregation but returns exactly one row @@ -172,19 +181,20 @@ public void testScalarSubqueryAfterComparisonInOneTable() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT %s FROM table1 WHERE device_id = 'd01' and %s = 30)"; retArray = new String[] {"30,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testScalarSubqueryAfterComparisonInDifferentTables() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: s greater than the count value of s in table2 @@ -192,11 +202,12 @@ public void testScalarSubqueryAfterComparisonInDifferentTables() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s > (SELECT count(%s) from table2)"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: s less than the max value of s in table2 * the count value of s in table2 * 10 @@ -204,18 +215,19 @@ public void testScalarSubqueryAfterComparisonInDifferentTables() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s < ((SELECT max(%s) from table2) * (SELECT count(%s) from table2)) * 10"; retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @Test public void testNestedScalarSubqueryAfterComparison() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: nested scalar subquery in where clause @@ -223,12 +235,13 @@ public void testNestedScalarSubqueryAfterComparison() { "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 where %s = (SELECT max(%s) from table1 WHERE device_id = 'd01'))"; retArray = new String[] {"70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, measurement, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: nested scalar subquery with table subquery @@ -236,8 +249,8 @@ public void testNestedScalarSubqueryAfterComparison() { "SELECT %s from (SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s = (SELECT max(%s) from table1 where %s = (SELECT max(%s) from table1 WHERE device_id = 'd01')))"; retArray = new String[] {"70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format( sql, measurement, @@ -248,7 +261,8 @@ public void testNestedScalarSubqueryAfterComparison() { measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } @@ -288,7 +302,7 @@ public void testScalarSubqueryAfterComparisonLegalityCheck() { @Test public void testScalarSubqueryInSelectClause() { String sql; - String expectedHeader; + String[] expectedHeader; String[] retArray; // Test case: select scalar subquery as one constant column @@ -296,11 +310,12 @@ public void testScalarSubqueryInSelectClause() { "SELECT cast((SELECT max(%s) from table1 where device_id = 'd01') AS INT32) as %s from table1 where device_id = 'd01'"; retArray = new String[] {"70,", "70,", "70,", "70,", "70,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: scalar subquery in arithmetic expression and function expression @@ -308,22 +323,24 @@ public void testScalarSubqueryInSelectClause() { "SELECT cast((s1 + (SELECT max(%s) from table1 where device_id = 'd01')) AS INT32) as %s from table1 where device_id = 'd01'"; retArray = new String[] {"100,", "110,", "120,", "130,", "140,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } sql = "SELECT cast(((SELECT max(%s) from table1 where device_id = 'd01') + (SELECT max(%s) from table1 where device_id = 'd01')) AS INT32) as %s from table1 where device_id = 'd01'"; retArray = new String[] {"140,", "140,", "140,", "140,", "140,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: scalar subquery in udf expression @@ -331,22 +348,24 @@ public void testScalarSubqueryInSelectClause() { "SELECT cast(floor(sin((SELECT max(%s) from table1 where device_id = 'd01'))) AS INT32) as %s from table1 where device_id = 'd01'"; retArray = new String[] {"0,", "0,", "0,", "0,", "0,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } // Test case: scalar subquery in aggregation function sql = "SELECT count((select count(s1) from table1)) as %s from table1"; retArray = new String[] {"64,"}; for (String measurement : NUMERIC_MEASUREMENTS) { - expectedHeader = measurement + ","; - assertUnorderedResult( + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( String.format(sql, measurement, measurement, measurement, measurement), expectedHeader, - Arrays.asList(retArray)); + retArray, + DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java index 7d9b4be5a93aa..ca92a1e1de6b6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/old/alignbydevice/IoTDBAlignByDeviceTableViewIT.java @@ -162,7 +162,7 @@ public void selectTest() { "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", }; tableResultSetEqualTest( - "select * from vehicle order by device_id, time", expectedHeader, retArray, DATABASE_NAME); + "select * from vehicle order by device_id", expectedHeader, retArray, DATABASE_NAME); } @Test @@ -224,7 +224,7 @@ public void selectWithDuplicatedPathsTest() { "1970-01-01T00:00:01.000Z,d1,888,888,null,", }; tableResultSetEqualTest( - "select time, device_id, s0,s0,s1 from vehicle where device_id = 'd0' or device_id = 'd1' order by device_id, time", + "select time, device_id, s0,s0,s1 from vehicle where device_id = 'd0' or device_id = 'd1' order by device_id", expectedHeader, retArray, DATABASE_NAME); @@ -289,7 +289,7 @@ public void selectDifferentSeriesWithValueFilterWithoutCacheTest() { "1999-12-31T16:00:00.000Z,d0,null,", }; tableResultSetEqualTest( - "select time, device_id, s0 from vehicle where s1 < 200 order by device_id, time", + "select time, device_id, s0 from vehicle where s1 < 200 order by device_id", expectedHeader, retArray, DATABASE_NAME); @@ -547,7 +547,7 @@ public void selectWithRegularExpressionTest() { "1970-01-01T00:00:01.000Z,d1,888,null,null,null,null,", }; tableResultSetEqualTest( - "select * from vehicle where device_id like 'd%' order by device_id, time", + "select * from vehicle where device_id like 'd%' order by device_id", expectedHeader, retArray, DATABASE_NAME); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java index 64ed2e54f51ad..e3c01f5e6d4ce 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationIT.java @@ -581,7 +581,7 @@ public void countIfTest() { "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d16,0,", }; tableResultSetEqualTest( - "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id ORDER BY device_id, 1", + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java index b451beb6aef6d..dbe22b168f1fd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewAggregationNonStreamIT.java @@ -581,7 +581,7 @@ public void countIfTest() { "2024-09-24T06:15:55.000Z,2024-09-24T06:16:00.000Z,d16,0,", }; tableResultSetEqualTest( - "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id ORDER BY device_id, 1", + "select date_bin(5s, time), (date_bin(5s, time) + 5000) as end_time, device_id, count_if(device_id = 'd01') from table1 group by 1,device_id", expectedHeader, retArray, DATABASE_NAME); diff --git a/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java b/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java index 1b1c57fe8abc3..750def8c4a3b0 100644 --- a/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java +++ b/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java @@ -581,4 +581,5 @@ private CalcMessages() {} public static final String EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_ARG_ARGUMENTS_BUT_GOT_ARG_D6249DD2 = "Aggregate function [%s] requires %d arguments, but got %d"; + } From 31fb18ca83c5120ee091b6ebccfd54122e1afb74 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 21 Oct 2026 18:01:08 +0800 Subject: [PATCH 29/29] Close shared budget test coverage gaps Add a test-only deregistration-mark hook in MemoryPool and use it to deterministically cover publication rejection into a closed account and cross-fragment waiter non-poisoning. Assert the per-fragment single multi-child-level invariant, the OUTPUT-side queue registration, and direct eligibility cases for Intersect/Except and source fragments. Rename stale test names, and remove tests that pass identically on the baseline or fall outside the eligibility causal chain. --- .../execution/memory/MemoryPool.java | 15 +++ .../exchange/SharedTsBlockQueueTest.java | 13 +++ .../execution/exchange/SourceHandleTest.java | 3 +- .../execution/memory/MemoryPoolTest.java | 87 ++++++++-------- .../LocalExecutionPlanContextBudgetTest.java | 98 ++++++------------- .../planner/CollectInputPropertyTest.java | 67 ++++++------- .../ParallelizeGroupingTest.java | 12 ++- 7 files changed, 135 insertions(+), 160 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index 62b95d80d281e..63cf3ce1d05a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -467,6 +467,21 @@ public boolean tryReserveForTest( return tryReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve); } + /** + * Marks a fragment account as deregistered without removing it from the map, mirroring the + * production mark that precedes the pending-request drain. Lets tests deterministically exercise + * the publication rejection of a closed account. + */ + @TestOnly + public void markFragmentDeregisteredForTest(String queryId, String fragmentInstanceId) { + FragmentMemoryReservation fragment = + queryMemoryReservations.get(queryId).get(fragmentInstanceId); + synchronized (fragment) { + fragment.deregistered = true; + setPlanNodesDeregistered(fragment, true); + } + } + /** * Cancel the specified memory reservation. If the reservation has finished, do nothing. * diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java index 8b97de5f3a259..391393cf5fa0e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java @@ -20,9 +20,11 @@ package org.apache.iotdb.db.queryengine.execution.exchange; import org.apache.iotdb.commons.memory.MemoryManager; +import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; +import org.apache.iotdb.db.queryengine.execution.memory.MemorySide; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; import com.google.common.util.concurrent.ListenableFuture; @@ -277,6 +279,17 @@ public void testDependentQueuesKeepIndependentBudgetsByDefault() { Assert.assertEquals(4, upstream.getNumOfBufferedTsBlocks()); Assert.assertEquals(1, downstream.getNumOfBufferedTsBlocks()); Assert.assertEquals(320, pool.getReservedBytes()); + // Both queues are producer-side OUTPUT buffers of the same fragment instance. + String fullFragmentInstanceId = + FragmentInstanceId.createFragmentInstanceIdFromTFragmentInstanceId(id); + Assert.assertEquals( + 320, + pool.getFragmentMemoryReservedBytes( + id.queryId, fullFragmentInstanceId, MemorySide.OUTPUT)); + Assert.assertEquals( + 0, + pool.getFragmentMemoryReservedBytes( + id.queryId, fullFragmentInstanceId, MemorySide.INPUT)); } finally { downstream.close(); upstream.close(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java index df59dffb6c37a..95c9b810ce195 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java @@ -66,10 +66,9 @@ public class SourceHandleTest { public static void beforeClass() { maxBytesPerFI = IoTDBDescriptor.getInstance().getMemoryConfig().getMaxBytesPerFragmentInstance(); - // INPUT receives half of the fragment budget, allowing five buffered blocks. IoTDBDescriptor.getInstance() .getMemoryConfig() - .setMaxBytesPerFragmentInstance(10 * MOCK_TSBLOCK_SIZE); + .setMaxBytesPerFragmentInstance(5 * MOCK_TSBLOCK_SIZE); } @AfterClass diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index 411e249bd2382..ae0f8ba325752 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -230,55 +230,50 @@ public void testForceDeregisterDetachesPendingReservationsEvenWhenReportingLeak( @Test @SuppressWarnings("unchecked") - public void testPublicationDuringDeregistrationCannotPoisonOtherQueries() throws Exception { - pool.registerPlanNodeIdToQueryMemoryMap("holder", "fi", "a", MemorySide.INPUT); - pool.registerPlanNodeIdToQueryMemoryMap("holder", "fi", "b", MemorySide.INPUT); - Assert.assertTrue(pool.tryReserve("holder", "fi", "a", 512, 512)); - Assert.assertTrue(pool.tryReserve("holder", "fi", "b", 512, 512)); - Field accounts = MemoryPool.class.getDeclaredField("queryMemoryReservations"); - accounts.setAccessible(true); - Map> queries = - (Map>) accounts.get(pool); - Object fragment = queries.get(QUERY_ID).get(FRAGMENT_INSTANCE_ID); - Field pendingField = fragment.getClass().getDeclaredField("pendingReservations"); - pendingField.setAccessible(true); - AtomicInteger publicationAttempts = new AtomicInteger(); - List detached = new ArrayList<>(); - List healthy = new ArrayList<>(); - pendingField.set( - fragment, - new java.util.concurrent.ConcurrentHashMap< - Long, MemoryPool.MemoryReservationFuture>() { - @Override - public Collection> values() { - // A waiter published after this snapshot was the missed-request race in deregistration. - Collection> snapshot = - new ArrayList<>(super.values()); - publicationAttempts.incrementAndGet(); - try { - detached.add( - pool.reserveWithPriority( - QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 32, 512, false)); - } catch (IllegalArgumentException expected) { - // Publication must be refused once deregistration has closed this account. - } - pool.registerPlanNodeIdToQueryMemoryMap("waiting", "fi", "later", MemorySide.INPUT); - healthy.add(pool.reserveWithPriority("waiting", "fi", "later", 32, 512, false)); - return snapshot; - } - }); + public void testPublicationIntoDeregisteredFragmentIsRejected() { + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, MemorySide.INPUT); + pool.markFragmentDeregisteredForTest(QUERY_ID, FRAGMENT_INSTANCE_ID); + // The account is still in the map but marked deregistered, so publication must be refused + // without charging any bytes. + Assert.assertThrows( + IllegalArgumentException.class, + () -> + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 32, 512, false)); + Assert.assertEquals(0, pool.getReservedBytes()); + Assert.assertEquals(0, pool.getMemoryReservationSize()); pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); - Assert.assertEquals(1, publicationAttempts.get()); - pool.free("holder", "fi", "a", 512); - Assert.assertTrue(healthy.get(0).getFuture().isDone()); - pool.free("holder", "fi", "b", 512); + } + + @Test + public void testDeregistrationDoesNotPoisonOtherQueries() throws Exception { + // Fill the pool from an unrelated fragment so that both later waiters block on capacity. + pool.registerPlanNodeIdToQueryMemoryMap("other", "fi", "o1", MemorySide.INPUT); + pool.registerPlanNodeIdToQueryMemoryMap("other", "fi", "o2", MemorySide.INPUT); + Assert.assertTrue(pool.tryReserve("other", "fi", "o1", 512, 512)); + Assert.assertTrue(pool.tryReserve("other", "fi", "o2", 512, 512)); + // A pending waiter in the fragment about to be deregistered, and one in another fragment. + pool.registerPlanNodeIdToQueryMemoryMap( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, MemorySide.INPUT); + MemoryPool.MemoryReservationResult detached = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 32, 512, false); + Assert.assertFalse(detached.getFuture().isDone()); + pool.registerPlanNodeIdToQueryMemoryMap("waiting", "fi", "later", MemorySide.INPUT); + MemoryPool.MemoryReservationResult healthy = + pool.reserveWithPriority("waiting", "fi", "later", 32, 512, false); + Assert.assertFalse(healthy.getFuture().isDone()); + pool.deRegisterFragmentInstanceFromQueryMemoryMap(QUERY_ID, FRAGMENT_INSTANCE_ID, true); + // The deregistered fragment's waiter is detached without completion; the other one is + // untouched and stays wakeable by released capacity. + Assert.assertFalse(detached.getFuture().isDone()); + Assert.assertFalse(healthy.getFuture().isDone()); + pool.free("other", "fi", "o1", 512); + Assert.assertTrue(healthy.getFuture().isDone()); + pool.free("other", "fi", "o2", 512); + Assert.assertEquals(32, pool.tryCancel(detached.getFuture())); pool.free("waiting", "fi", "later", 32); - for (MemoryReservationResult request : detached) { - Assert.assertFalse(request.getFuture().isDone()); - Assert.assertEquals(32, pool.tryCancel(request.getFuture())); - } - pool.deRegisterFragmentInstanceFromQueryMemoryMap("holder", "fi", true); pool.deRegisterFragmentInstanceFromQueryMemoryMap("waiting", "fi", true); + pool.deRegisterFragmentInstanceFromQueryMemoryMap("other", "fi", true); Assert.assertEquals(0, pool.getReservedBytes()); Assert.assertEquals(0, pool.getMemoryReservationSize()); Assert.assertEquals(0, pool.getQueryMemoryReservationSize()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java index d010fc4a3275f..e1515684116b2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanContextBudgetTest.java @@ -19,10 +19,8 @@ package org.apache.iotdb.db.queryengine.plan.planner; -import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; import org.apache.iotdb.calc.execution.operator.Operator; import org.apache.iotdb.calc.execution.operator.process.CollectOperator; -import org.apache.iotdb.calc.execution.operator.process.TableMergeSortOperator; import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.client.IClientManager; @@ -57,18 +55,13 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.util.concurrent.SettableFuture; import org.apache.tsfile.common.conf.TSFileDescriptor; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.block.TsBlock; -import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.TsBlockSerde; import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.concurrent.ExecutorService; @@ -395,7 +388,7 @@ public void testTreePlanningRetainsLegacySourceQuota() { } @Test - public void testTableCollectEnablesReadyConsumptionOnlyForEligibleFragment() { + public void testEligibleTableCollectEnablesReadyConsumptionAndSharedBudget() { PlanNode plan = collect(); CollectOperator collect = generateTableCollect(plan); addSource(context); @@ -407,7 +400,7 @@ public void testTableCollectEnablesReadyConsumptionOnlyForEligibleFragment() { } @Test - public void testTableUnionEnablesReadyConsumptionOnlyForEligibleFragment() { + public void testEligibleTableUnionEnablesReadyConsumptionAndSharedBudget() { PlanNode union = new UnionNode( new PlanNodeId("union"), @@ -446,7 +439,7 @@ public void testSingleChildTopKRetainsLegacyBudgetAndConsumption() { } @Test - public void testMultiplePipelinesRetainSequentialCollectConsumption() { + public void testMultiplePipelinesKeepLegacyBudgetWithReadyCollectConsumption() { PlanNode plan = collect(); CollectOperator collect = generateTableCollect(plan); LocalExecutionPlanContext child = context.createSubContext(); @@ -508,6 +501,15 @@ public void testSequentialAndUnknownMultiInputOperatorsDisableSharing() { assertFalse( SharedExchangeBudgetEligibility.isEligible( node(PlanNodeType.SCHEMA_QUERY_MERGE, collect(), collect()))); + for (PlanNodeType multiChildSetOperation : + new PlanNodeType[] {PlanNodeType.TABLE_INTERSECT_NODE, PlanNodeType.TABLE_EXCEPT_NODE}) { + assertFalse( + SharedExchangeBudgetEligibility.isEligible( + node( + multiChildSetOperation, + node(PlanNodeType.TABLE_EXCHANGE_NODE), + node(PlanNodeType.TABLE_EXCHANGE_NODE)))); + } } @Test @@ -521,6 +523,23 @@ public void testExchangeBoundaryDoesNotInspectUpstreamFragment() { node(PlanNodeType.TABLE_EXCHANGE_NODE)))); } + @Test + public void testSourceFragmentRootedAtExchangeKeepsLegacyBudget() { + // The source fragment's root is an ExchangeNode; its upstream child belongs to another + // fragment and must not make this fragment eligible for a shared budget. + assertFalse( + SharedExchangeBudgetEligibility.isEligible( + node( + PlanNodeType.TABLE_EXCHANGE_NODE, + node(PlanNodeType.TABLE_COLLECT_NODE, collect(), collect())))); + context.setExchangeMemoryLimits( + node( + PlanNodeType.TABLE_EXCHANGE_NODE, + node(PlanNodeType.TABLE_COLLECT_NODE, collect(), collect())), + pool); + assertAdmissionIsShared(false); + } + @Test public void testNoCollectKeepsLegacyBudget() { context.setExchangeMemoryLimits( @@ -574,63 +593,4 @@ public void testSinkOnlyFragmentCanStillReserveOneFullBlock() { node(PlanNodeType.IDENTITY_SINK, node(PlanNodeType.SERIES_SCAN)), pool); verify(sink).setMaxBytesCanReserve(64); } - - @Test - public void testMergeSortGetsBothHeadsDespitePrefetchOnOneInput() throws Exception { - context.setExchangeMemoryLimits( - node(PlanNodeType.TABLE_MERGESORT_NODE, collect(), node(PlanNodeType.TABLE_EXCHANGE_NODE)), - pool); - String fi = context.getFragmentInstanceId().getFragmentInstanceId(); - pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "a", MemorySide.INPUT); - pool.registerPlanNodeIdToQueryMemoryMap("budget", fi, "b", MemorySide.INPUT); - for (int i = 0; i < 4; i++) { - assertTrue(pool.reserveWithPriority("budget", fi, "a", 64, 256, false).isReserveSuccess()); - } - MemoryPool.MemoryReservationResult prefetch = - pool.reserveWithPriority("budget", fi, "a", 64, 256, false); - assertFalse(prefetch.getFuture().isDone()); - MemoryPool.MemoryReservationResult headB = - pool.reserveWithPriority("budget", fi, "b", 64, 256, false); - assertTrue(headB.isReserveSuccess()); - Operator a = mock(Operator.class); - Operator b = mock(Operator.class); - doReturn(Operator.NOT_BLOCKED).when(a).isBlocked(); - doReturn(headB.getFuture()).when(b).isBlocked(); - when(a.hasNextWithTimer()).thenReturn(true); - when(b.hasNextWithTimer()).thenReturn(true); - TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.INT32)); - builder.getTimeColumnBuilder().writeLong(1); - builder.getColumnBuilder(0).writeInt(1); - builder.declarePosition(); - TsBlock first = builder.build(); - builder.reset(); - builder.getTimeColumnBuilder().writeLong(2); - builder.getColumnBuilder(0).writeInt(2); - builder.declarePosition(); - TsBlock second = builder.build(); - when(a.nextWithTimer()) - .thenAnswer( - call -> { - pool.free("budget", fi, "a", 64); - return first; - }); - when(b.nextWithTimer()) - .thenAnswer( - call -> { - pool.free("budget", fi, "b", 64); - return second; - }); - TableMergeSortOperator merge = - new TableMergeSortOperator( - mock(CommonOperatorContext.class), - new ArrayList<>(Arrays.asList(a, b)), - Collections.singletonList(TSDataType.INT32), - Comparator.comparingLong(key -> key.tsBlock.getTimeByIndex(key.rowIndex))); - assertTrue(merge.isBlocked().isDone()); - TsBlock result = merge.next(); - assertTrue(result != null && !result.isEmpty()); - assertTrue(prefetch.getFuture().isDone()); - pool.free("budget", fi, "a", 256); - merge.close(); - } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java index 0bf8e170a6f98..3f4240c78271f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CollectInputPropertyTest.java @@ -24,7 +24,6 @@ import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.CollectNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GroupNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; -import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.WindowNode; import org.junit.Test; @@ -62,6 +61,15 @@ private static List nodes(List plans, Class type) { return result; } + private static List allNodes(PlanNode node) { + List result = new ArrayList<>(); + result.add(node); + for (PlanNode child : node.getChildren()) { + result.addAll(allNodes(child)); + } + return result; + } + @Test public void testAggregationAboveGlobalWindowHashesInterleavedGroups() { for (String aggregate : new String[] {"SUM(s1)", "COUNT(DISTINCT s1)"}) { @@ -121,46 +129,27 @@ public void testNestedWindowRestoresPartitionContiguityWithSort() { } @Test - public void testStreamSortAfterGlobalWindowUsesFullSort() { - List fragments = - plans( - "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " - + "FROM table1 ORDER BY tag1, tag2, tag3, s1"); - PlanNode root = fragments.get(0); - assertFalse(nodes(fragments, CollectNode.class).isEmpty()); - assertFalse(nodes(root, SortNode.class).isEmpty()); - assertTrue(nodes(root, StreamSortNode.class).isEmpty()); - } - - @Test - public void testConstantFillDoesNotRemoveRestoredSort() { - List fragments = - plans( - "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n FROM table1 " - + "FILL METHOD CONSTANT 0 ORDER BY tag1, tag2, tag3"); - PlanNode root = fragments.get(0); - assertFalse(nodes(fragments, CollectNode.class).isEmpty()); - assertFalse(nodes(root, SortNode.class).isEmpty()); - assertTrue(nodes(root, StreamSortNode.class).isEmpty()); - } - - @Test - public void testUnionFinalAggregationDoesNotInheritBranchGrouping() { - List fragments = - plans( - "SELECT tag1, tag2, tag3, SUM(s1) FROM (" - + "SELECT tag1, tag2, tag3, s1 FROM table1 UNION ALL " - + "SELECT tag1, tag2, tag3, s1 FROM table2) u GROUP BY tag1, tag2, tag3"); - List aggregations = nodes(fragments, AggregationNode.class); - assertFalse(aggregations.isEmpty()); - boolean foundFinalCollect = false; - for (AggregationNode aggregation : aggregations) { - if (aggregation.getChild() instanceof CollectNode) { - foundFinalCollect = true; - assertTrue(aggregation.getPreGroupedSymbols().isEmpty()); + public void testEveryFragmentHasAtMostOneMultiChildLevel() { + String[] queries = { + "SELECT tag1, tag2, tag3, SUM(s1) FROM table1 GROUP BY tag1, tag2, tag3", + "SELECT tag1, tag2, tag3, SUM(s1) FROM (" + + "SELECT tag1, tag2, tag3, s1 FROM table1 UNION ALL " + + "SELECT tag1, tag2, tag3, s1 FROM table2) u GROUP BY tag1, tag2, tag3", + "SELECT tag1, tag2, tag3, SUM(s1) FROM (" + + "SELECT tag1, tag2, tag3, s1, COUNT(*) OVER () AS n " + + "FROM table1 WHERE tag1 != 'shenzhen') t WHERE n > 0 GROUP BY tag1, tag2, tag3" + }; + for (String sql : queries) { + for (PlanNode fragment : plans(sql)) { + int multiChildLevels = 0; + for (PlanNode node : allNodes(fragment)) { + if (node.getChildren().size() > 1) { + multiChildLevels++; + } + } + assertTrue("Fragment contains more than one multi-child level", multiChildLevels <= 1); } } - assertTrue(foundFinalCollect); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGroupingTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGroupingTest.java index b5b69f7663f03..ab852fddb074d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGroupingTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/ParallelizeGroupingTest.java @@ -52,6 +52,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.SESSION_INFO; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.TEST_MATADATA; import static org.apache.tsfile.read.common.type.StringType.STRING; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; @@ -138,9 +139,10 @@ public void testUnionDoesNotInheritFirstBranchGrouping() { ImmutableList.of(left, right), ImmutableListMultimap.of(key, key, key, key), ImmutableList.of(key)); - assertTrue( - optimizeGroup(new GroupNode(new PlanNodeId("group"), union, ordering, 1)) - instanceof SortNode); + PlanNode optimized = optimizeGroup(new GroupNode(new PlanNodeId("group"), union, ordering, 1)); + assertTrue(optimized instanceof SortNode); + // The multi-branch fan-in must not leave a segmented GroupNode above it. + assertFalse(optimized instanceof GroupNode); // An explicit sort above the fan-in restores the grouping property. SortNode restored = new SortNode(new PlanNodeId("restored"), union, ordering, false, false); assertTrue( @@ -149,12 +151,14 @@ public void testUnionDoesNotInheritFirstBranchGrouping() { } private PlanNode optimizeGroup(GroupNode group) { + Analysis analysis = new Analysis(null, Collections.emptyMap()); + analysis.setQuery(true); return new ParallelizeGrouping() .optimize( group, new PlanOptimizer.Context( SESSION_INFO, - new Analysis(null, Collections.emptyMap()), + analysis, TEST_MATADATA, new MPPQueryContext(new QueryId("group_union")), new SymbolAllocator(),