From e1dcba41681c8541261221408444837b87b8e769 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Sun, 8 Mar 2026 01:23:24 +0800 Subject: [PATCH 1/2] Flash: add MPP sql digest logging and update kvproto --- contrib/kvproto | 2 +- dbms/src/Flash/Coprocessor/DAGContext.cpp | 3 ++- dbms/src/Flash/Coprocessor/DAGContext.h | 2 ++ dbms/src/Flash/FlashService.cpp | 5 +++-- dbms/src/Flash/Mpp/MPPTaskStatistics.cpp | 4 +++- dbms/src/Flash/Mpp/MPPTaskStatistics.h | 1 + 6 files changed, 12 insertions(+), 5 deletions(-) diff --git a/contrib/kvproto b/contrib/kvproto index 6db24b6c67f..78644d3dc46 160000 --- a/contrib/kvproto +++ b/contrib/kvproto @@ -1 +1 @@ -Subproject commit 6db24b6c67f30a03694931462d12c2e6982c63e2 +Subproject commit 78644d3dc465994762abd25f767206fbc7b0ba06 diff --git a/dbms/src/Flash/Coprocessor/DAGContext.cpp b/dbms/src/Flash/Coprocessor/DAGContext.cpp index d7c10b1d26b..16cb2f44492 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.cpp +++ b/dbms/src/Flash/Coprocessor/DAGContext.cpp @@ -104,6 +104,7 @@ DAGContext::DAGContext(tipb::DAGRequest & dag_request_, const mpp::TaskMeta & me , resource_group_name(meta_.resource_group_name()) , connection_id(meta_.connection_id()) , connection_alias(meta_.connection_alias()) + , sql_digest(meta_.sql_digest()) { if (dag_request->has_div_precision_increment()) div_precision_increment = dag_request->div_precision_increment(); @@ -483,7 +484,7 @@ UInt64 DAGContext::getReadBytes() const UInt64 read_bytes = 0; for (const auto & [id, sc] : scan_context_map) { - (void)id; // Disable unused variable warnning. + (void)id; // Disable unused variable warning. read_bytes += sc->userReadBytes(); } return read_bytes; diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index 38f02089efe..8fb3d176bbe 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -361,6 +361,7 @@ class DAGContext UInt64 getConnectionID() const { return connection_id; } const String & getConnectionAlias() const { return connection_alias; } + const String & getSQLDigest() const { return sql_digest; } MPPReceiverSetPtr getMPPReceiverSet() const { return mpp_receiver_set; } @@ -544,6 +545,7 @@ class DAGContext UInt64 connection_id; // It's the session alias between mysql client and tidb String connection_alias; + String sql_digest; String query_id_and_cte_id_for_sink; std::unordered_map query_id_and_cte_id_for_sources; diff --git a/dbms/src/Flash/FlashService.cpp b/dbms/src/Flash/FlashService.cpp index 9b07a8c1709..63115ff16b5 100644 --- a/dbms/src/Flash/FlashService.cpp +++ b/dbms/src/Flash/FlashService.cpp @@ -495,11 +495,12 @@ grpc::Status FlashService::DispatchMPPTask( const auto & resource_group = task_meta.resource_group_name(); LOG_INFO( log, - "Handling mpp dispatch request, task: {}, resource_group: {}, conn_id: {}, conn_alias: {}", + "Handling mpp dispatch request, task: {}, resource_group: {}, conn_id: {}, conn_alias: {}, sql_digest: {}", MPPTaskId(task_meta).toString(), resource_group, task_meta.connection_id(), - task_meta.connection_alias()); + task_meta.connection_alias(), + task_meta.sql_digest()); auto check_result = checkGrpcContext(grpc_context); if (!check_result.ok()) return check_result; diff --git a/dbms/src/Flash/Mpp/MPPTaskStatistics.cpp b/dbms/src/Flash/Mpp/MPPTaskStatistics.cpp index 8674cf0dcb1..6d0baf2b5dc 100644 --- a/dbms/src/Flash/Mpp/MPPTaskStatistics.cpp +++ b/dbms/src/Flash/Mpp/MPPTaskStatistics.cpp @@ -77,6 +77,7 @@ void MPPTaskStatistics::initializeExecutorDAG(DAGContext * dag_context_) is_root = dag_context->isRootMPPTask(); sender_executor_id = root_executor.executor_id(); + sql_digest = dag_context->getSQLDigest(); executor_statistics_collector.initialize(dag_context); } @@ -109,7 +110,7 @@ void MPPTaskStatistics::logTracingJson() log, /// don't use info log for initializing status since it does not contains too many information status == INITIALIZING ? Poco::Message::PRIO_DEBUG : Poco::Message::PRIO_INFORMATION, - R"({{"query_tso":{},"task_id":{},"is_root":{},"sender_executor_id":"{}","executors":{},"host":"{}")" + R"({{"query_tso":{},"task_id":{},"is_root":{},"sender_executor_id":"{}","executors":{},"host":"{}","sql_digest":"{}")" R"(,"task_init_timestamp":{},"task_start_timestamp":{},"task_end_timestamp":{})" R"(,"compile_start_timestamp":{},"compile_end_timestamp":{})" R"(,"read_wait_index_start_timestamp":{},"read_wait_index_end_timestamp":{})" @@ -121,6 +122,7 @@ void MPPTaskStatistics::logTracingJson() sender_executor_id, executor_statistics_collector.profilesToJson(), host, + sql_digest, toNanoseconds(task_init_timestamp), toNanoseconds(task_start_timestamp), toNanoseconds(task_end_timestamp), diff --git a/dbms/src/Flash/Mpp/MPPTaskStatistics.h b/dbms/src/Flash/Mpp/MPPTaskStatistics.h index 50db59becbd..34471a30a0a 100644 --- a/dbms/src/Flash/Mpp/MPPTaskStatistics.h +++ b/dbms/src/Flash/Mpp/MPPTaskStatistics.h @@ -89,6 +89,7 @@ class MPPTaskStatistics // executor dag bool is_root = false; String sender_executor_id; + String sql_digest; // resource RUConsumption ru_info{.cpu_ru = 0.0, .cpu_time_ns = 0, .read_ru = 0.0, .read_bytes = 0}; From 3caaa0ad8b3f70b063b8b6c647b6c5c87f21f673 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Sun, 8 Mar 2026 14:23:35 +0800 Subject: [PATCH 2/2] Flash: include connection metadata in MPP task tracing logs --- dbms/src/Flash/Mpp/MPPTaskStatistics.cpp | 7 ++++++- dbms/src/Flash/Mpp/MPPTaskStatistics.h | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dbms/src/Flash/Mpp/MPPTaskStatistics.cpp b/dbms/src/Flash/Mpp/MPPTaskStatistics.cpp index 6d0baf2b5dc..6c205be5fdc 100644 --- a/dbms/src/Flash/Mpp/MPPTaskStatistics.cpp +++ b/dbms/src/Flash/Mpp/MPPTaskStatistics.cpp @@ -77,6 +77,8 @@ void MPPTaskStatistics::initializeExecutorDAG(DAGContext * dag_context_) is_root = dag_context->isRootMPPTask(); sender_executor_id = root_executor.executor_id(); + connection_id = dag_context->getConnectionID(); + connection_alias = dag_context->getConnectionAlias(); sql_digest = dag_context->getSQLDigest(); executor_statistics_collector.initialize(dag_context); } @@ -110,7 +112,8 @@ void MPPTaskStatistics::logTracingJson() log, /// don't use info log for initializing status since it does not contains too many information status == INITIALIZING ? Poco::Message::PRIO_DEBUG : Poco::Message::PRIO_INFORMATION, - R"({{"query_tso":{},"task_id":{},"is_root":{},"sender_executor_id":"{}","executors":{},"host":"{}","sql_digest":"{}")" + R"({{"query_tso":{},"task_id":{},"is_root":{},"sender_executor_id":"{}","executors":{},"host":"{}")" + R"(,"connection_id":{},"connection_alias":"{}","sql_digest":"{}")" R"(,"task_init_timestamp":{},"task_start_timestamp":{},"task_end_timestamp":{})" R"(,"compile_start_timestamp":{},"compile_end_timestamp":{})" R"(,"read_wait_index_start_timestamp":{},"read_wait_index_end_timestamp":{})" @@ -122,6 +125,8 @@ void MPPTaskStatistics::logTracingJson() sender_executor_id, executor_statistics_collector.profilesToJson(), host, + connection_id, + connection_alias, sql_digest, toNanoseconds(task_init_timestamp), toNanoseconds(task_start_timestamp), diff --git a/dbms/src/Flash/Mpp/MPPTaskStatistics.h b/dbms/src/Flash/Mpp/MPPTaskStatistics.h index 34471a30a0a..c7c4b22a48d 100644 --- a/dbms/src/Flash/Mpp/MPPTaskStatistics.h +++ b/dbms/src/Flash/Mpp/MPPTaskStatistics.h @@ -89,6 +89,8 @@ class MPPTaskStatistics // executor dag bool is_root = false; String sender_executor_id; + UInt64 connection_id = 0; + String connection_alias; String sql_digest; // resource