Skip to content

[To dev/1.3] Preserve LOAD failure details in logs and responses (#18935) #18940

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Caideyipi wants to merge 4 commits into
base: dev/1.3
Choose a base branch
from
+388 −43
Open

[To dev/1.3] Preserve LOAD failure details in logs and responses (#18935) #18940

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
302de70
[To dev/1.3] Preserve LOAD failure details in logs and responses (#18…
Caideyipi Oct 20, 2026
e52e34e
Fix LoadTsFile analyzer diagnostics test reflection
Caideyipi Oct 20, 2026
5862a2f
Fix unused PowerMock dependency
Caideyipi Oct 21, 2026
16f5e2e
Fix PowerMock access to Java reflection internals
Caideyipi Oct 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,29 @@ public final class DataNodeQueryMessages {

private DataNodeQueryMessages() {}

public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477 =
"Failed to convert TsFile %s to tablets. Status: %s";
public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD =
"Failed to convert TsFile %s to tablets: %s";
public static final String MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3 =
"Tablet conversion returned no status";
public static final String
EXCEPTION_FAILED_TO_LOAD_TSFILES_AFTER_TABLET_CONVERSION_DETAILS_ARG_53647EC5 =
"Failed to load TsFiles after tablet conversion. Details: %s";
public static final String
LOG_LOAD_REJECTED_UUID_ARG_REGION_ARG_ENDPOINT_ARG_STATUS_ARG_8ED0679E =
"LOAD rejected: uuid={}, region={}, endpoint={}, status={}";
public static final String LOG_LOAD_COMMAND_REJECTED_UUID_ARG_ENDPOINT_ARG_STATUS_ARG_E8D0031C =
"LOAD command rejected: uuid={}, endpoint={}, status={}";
public static final String
MESSAGE_TARGET_NODE_REJECTED_LOAD_WITHOUT_A_STATUS_OR_A_MESSAGE_F4E4C645 =
"Target node rejected LOAD without a status or a message";
public static final String
MESSAGE_FAILED_TO_DISPATCH_LOAD_ARG_FOR_REGION_ARG_TO_NODE_ARG_ARG_57EFEC3A =
"Failed to dispatch LOAD %s for region %s to node %s: %s";
public static final String MESSAGE_FAILED_TO_DISPATCH_LOAD_COMMAND_ARG_TO_NODE_ARG_ARG_53D2BF0B =
"Failed to dispatch LOAD command %s to node %s: %s";

public static final String RESULT_SET_COLUMN_MEMORY_SHORTAGE_EQUIVALENT =
"The failed memory reservation exceeds available memory by the equivalent of at least "
+ "%,d columns, estimated from the observed average column size. ";
Expand Down
23 changes: 23 additions & 0 deletions iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,29 @@ public final class DataNodeQueryMessages {

private DataNodeQueryMessages() {}

public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477 =
"将 TsFile %s 转换为 tablet 失败。状态：%s";
public static final String EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD =
"将 TsFile %s 转换为 tablet 失败：%s";
public static final String MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3 =
"tablet 转换未返回状态";
public static final String
EXCEPTION_FAILED_TO_LOAD_TSFILES_AFTER_TABLET_CONVERSION_DETAILS_ARG_53647EC5 =
"将 TsFile 转换为 tablet 后加载仍失败。详情：%s";
public static final String
LOG_LOAD_REJECTED_UUID_ARG_REGION_ARG_ENDPOINT_ARG_STATUS_ARG_8ED0679E =
"LOAD 被拒绝：uuid={}，region={}，endpoint={}，status={}";
public static final String
MESSAGE_TARGET_NODE_REJECTED_LOAD_WITHOUT_A_STATUS_OR_A_MESSAGE_F4E4C645 =
"目标节点拒绝 LOAD，且未返回状态或消息";
public static final String LOG_LOAD_COMMAND_REJECTED_UUID_ARG_ENDPOINT_ARG_STATUS_ARG_E8D0031C =
"LOAD command 被拒绝：uuid={}，endpoint={}，status={}";
public static final String
MESSAGE_FAILED_TO_DISPATCH_LOAD_ARG_FOR_REGION_ARG_TO_NODE_ARG_ARG_57EFEC3A =
"分发 LOAD %s（region %s）到节点 %s 失败：%s";
public static final String MESSAGE_FAILED_TO_DISPATCH_LOAD_COMMAND_ARG_TO_NODE_ARG_ARG_53D2BF0B =
"分发 LOAD command %s 到节点 %s 失败：%s";

public static final String RESULT_SET_COLUMN_MEMORY_SHORTAGE_EQUIVALENT =
"本次失败的内存申请超出可用内存，按已记录列的平均内存估算，至少超出相当于 %,d 列的容量。";

Expand Down
55 changes: 38 additions & 17 deletions ...e/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import org.apache.iotdb.db.exception.load.LoadFileException;
import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
Expand Down Expand Up @@ -544,23 +545,43 @@ private Analysis executeTabletConversionOnException(

final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
new LoadTsFileDataTypeConverter(context, isGeneratedByPipe);
final TSStatus status =
loadTsFileStatement.isConvertOnTypeMismatch()
? loadTsFileDataTypeConverter.convertForTreeModel(loadTsFileStatement).orElse(null)
: null;

if (status == null) {
LOGGER.warn(
"Load: Failed to convert to tablets from statement {}. Status is null.",
loadTsFileStatement);
analysis.setFailStatus(
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage()));
} else if (!loadTsFileDataTypeConverter.isSuccessful(status)) {
LOGGER.warn(
"Load: Failed to convert to tablets from statement {}. Status: {}",
loadTsFileStatement,
status);
analysis.setFailStatus(status);
final String filePaths =
loadTsFileStatement.getTsFiles().stream()
.map(File::getPath)
.collect(Collectors.joining(", "));
try {
final TSStatus status =
loadTsFileStatement.isConvertOnTypeMismatch()
? loadTsFileDataTypeConverter.convertForTreeModel(loadTsFileStatement).orElse(null)
: null;

if (status == null) {
final String message =
String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
filePaths,
DataNodeQueryMessages.MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3);
LOGGER.warn(message);
analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, message));
} else if (!loadTsFileDataTypeConverter.isSuccessful(status)) {
final String message =
String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477,
filePaths,
status);
LOGGER.warn(message);
analysis.setFailStatus(status.deepCopy().setMessage(message));
}
} catch (final Exception conversionException) {
final String message =
String.format(
DataNodeQueryMessages.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
filePaths,
conversionException);
LOGGER.warn(message, conversionException);
analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, message));
}

analysis.setFinishQueryAfterAnalyze(true);
Expand Down
57 changes: 49 additions & 8 deletions ...in/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.load.LoadFileException;
import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance;
import org.apache.iotdb.db.queryengine.plan.planner.plan.SubPlan;
Expand Down Expand Up @@ -317,18 +318,39 @@ private void dispatchRemote(
for (final TTsFilePieceReq loadTsFileReq : loadTsFileReqs) {
final TLoadResp loadResp = client.sendTsFilePieceNode(loadTsFileReq);
if (!loadResp.isAccepted()) {
LOGGER.warn(loadResp.message);
throw new FragmentInstanceDispatchException(loadResp.status);
final TSStatus failureStatus =
loadResp.isSetStatus()
? loadResp.getStatus()
: RpcUtils.getStatus(
TSStatusCode.DISPATCH_ERROR,
loadResp.isSetMessage()
? loadResp.getMessage()
: DataNodeQueryMessages
.MESSAGE_TARGET_NODE_REJECTED_LOAD_WITHOUT_A_STATUS_OR_A_MESSAGE_F4E4C645);
LOGGER.warn(
DataNodeQueryMessages
.LOG_LOAD_REJECTED_UUID_ARG_REGION_ARG_ENDPOINT_ARG_STATUS_ARG_8ED0679E,
uuid,
consensusGroupId,
endPoint,
failureStatus);
throw new FragmentInstanceDispatchException(failureStatus);
}
}
}
} catch (final FragmentInstanceDispatchException e) {
throw e;
} catch (Exception e) {
adjustTimeoutIfNecessary(e);

final String exceptionMessage =
String.format(
"failed to dispatch load command %s to node %s because of exception: %s",
uuid, endPoint, e);
DataNodeQueryMessages
.MESSAGE_FAILED_TO_DISPATCH_LOAD_ARG_FOR_REGION_ARG_TO_NODE_ARG_ARG_57EFEC3A,
uuid,
consensusGroupId,
endPoint,
e);
LOGGER.warn(exceptionMessage, e);
throw new FragmentInstanceDispatchException(
new TSStatus()
Expand Down Expand Up @@ -413,16 +435,35 @@ private void dispatchRemote(TLoadCommandReq loadCommandReq, TEndPoint endPoint)

final TLoadResp loadResp = client.sendLoadCommand(loadCommandReq);
if (!loadResp.isAccepted()) {
LOGGER.warn(loadResp.message);
throw new FragmentInstanceDispatchException(loadResp.status);
final TSStatus failureStatus =
loadResp.isSetStatus()
? loadResp.getStatus()
: RpcUtils.getStatus(
TSStatusCode.DISPATCH_ERROR,
loadResp.isSetMessage()
? loadResp.getMessage()
: DataNodeQueryMessages
.MESSAGE_TARGET_NODE_REJECTED_LOAD_WITHOUT_A_STATUS_OR_A_MESSAGE_F4E4C645);
LOGGER.warn(
DataNodeQueryMessages
.LOG_LOAD_COMMAND_REJECTED_UUID_ARG_ENDPOINT_ARG_STATUS_ARG_E8D0031C,
loadCommandReq.uuid,
endPoint,
failureStatus);
throw new FragmentInstanceDispatchException(failureStatus);
}
} catch (final FragmentInstanceDispatchException e) {
throw e;
} catch (Exception e) {
adjustTimeoutIfNecessary(e);

final String exceptionMessage =
String.format(
"failed to dispatch load command %s to node %s because of exception: %s",
loadCommandReq, endPoint, e);
DataNodeQueryMessages
.MESSAGE_FAILED_TO_DISPATCH_LOAD_COMMAND_ARG_TO_NODE_ARG_ARG_53D2BF0B,
loadCommandReq,
endPoint,
e);
LOGGER.warn(exceptionMessage, e);
throw new FragmentInstanceDispatchException(
new TSStatus()
Expand Down
41 changes: 28 additions & 13 deletions ...rc/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.apache.iotdb.db.exception.load.LoadReadOnlyException;
import org.apache.iotdb.db.exception.load.RegionReplicaSetChangedException;
import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.PlanFragmentId;
Expand Down Expand Up @@ -559,6 +560,7 @@ private boolean loadLocally(LoadSingleTsFileNode node) throws IoTDBException {
private void convertFailedTsFilesToTabletsAndRetry() {
final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
new LoadTsFileDataTypeConverter(queryContext, isGeneratedByPipe);
final List<String> failureDetails = new ArrayList<>();

final Iterator<Integer> iterator = failedTsFileNodeIndexes.listIterator();
while (iterator.hasNext()) {
Expand All @@ -579,17 +581,30 @@ private void convertFailedTsFilesToTabletsAndRetry() {
"Load: Successfully converted TsFile {} into tablets and inserted.",
failedNode.getTsFileResource().getTsFilePath());
} else {
LOGGER.warn(
"Load: Failed to convert to tablets from TsFile {}. Status: {}",
failedNode.getTsFileResource().getTsFilePath(),
status);
final String message =
status == null
? String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
filePath,
DataNodeQueryMessages.MESSAGE_TABLET_CONVERSION_RETURNED_NO_STATUS_88BAB5D3)
: String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477,
filePath,
status);
failureDetails.add(message);
LOGGER.warn(message);
}
} catch (final Exception e) {
LOGGER.warn(
"Load: Failed to convert to tablets from TsFile {}. Exception: {}",
failedNode.getTsFileResource().getTsFilePath(),
e.getMessage(),
e);
final String message =
String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD,
filePath,
e);
failureDetails.add(message);
LOGGER.warn(message, e);
}
}

Expand All @@ -600,10 +615,10 @@ private void convertFailedTsFilesToTabletsAndRetry() {
stateMachine.transitionToFinished();
} else {
final String errorMsg =
"Load: failed to load some TsFiles by converting them into tablets. Failed TsFiles: "
+ failedTsFileNodeIndexes.stream()
.map(i -> tsFileNodeList.get(i).getTsFileResource().getTsFilePath())
.collect(Collectors.joining(", "));
String.format(
DataNodeQueryMessages
.EXCEPTION_FAILED_TO_LOAD_TSFILES_AFTER_TABLET_CONVERSION_DETAILS_ARG_53647EC5,
String.join("; ", failureDetails));
LOGGER.warn(errorMsg);
stateMachine.transitionToFailed(new LoadFileException(errorMsg));
}
Expand Down
Loading
Loading