diff --git a/.rat-excludes b/.rat-excludes index 05405443c..6caf109b1 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -33,3 +33,7 @@ NOTICE* docs/** build/apache-maven-*/** src/main/resources/auron-build-info.properties +**/tpch-data-parquet/** +**/tpch-plan-stability/** +**/tpch-queries/** +**/tpch-query-results/** diff --git a/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/customer/part-00000-a1a42661-7a85-42da-b831-f489a5545d61-c000.snappy.parquet b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/customer/part-00000-a1a42661-7a85-42da-b831-f489a5545d61-c000.snappy.parquet new file mode 100644 index 000000000..e1e8de376 Binary files /dev/null and b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/customer/part-00000-a1a42661-7a85-42da-b831-f489a5545d61-c000.snappy.parquet differ diff --git a/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/lineitem/part-00000-6c374e0a-7d76-401b-8458-a8e31f8ab704-c000.snappy.parquet b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/lineitem/part-00000-6c374e0a-7d76-401b-8458-a8e31f8ab704-c000.snappy.parquet new file mode 100644 index 000000000..42d7c0d9f Binary files /dev/null and b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/lineitem/part-00000-6c374e0a-7d76-401b-8458-a8e31f8ab704-c000.snappy.parquet differ diff --git a/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/nation/part-00000-8a852c7b-da96-4425-9ce8-d188f97f3afe-c000.snappy.parquet b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/nation/part-00000-8a852c7b-da96-4425-9ce8-d188f97f3afe-c000.snappy.parquet new file mode 100644 index 000000000..7c9a4404a Binary files /dev/null and b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/nation/part-00000-8a852c7b-da96-4425-9ce8-d188f97f3afe-c000.snappy.parquet differ diff --git a/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/orders/part-00000-41cc94ee-a98d-4103-83b1-0b5bcebe03eb-c000.snappy.parquet b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/orders/part-00000-41cc94ee-a98d-4103-83b1-0b5bcebe03eb-c000.snappy.parquet new file mode 100644 index 000000000..e164929bd Binary files /dev/null and b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/orders/part-00000-41cc94ee-a98d-4103-83b1-0b5bcebe03eb-c000.snappy.parquet differ diff --git a/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/part/part-00000-e664e655-0ada-4e0b-a8aa-213d1e4c3797-c000.snappy.parquet b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/part/part-00000-e664e655-0ada-4e0b-a8aa-213d1e4c3797-c000.snappy.parquet new file mode 100644 index 000000000..4763bc361 Binary files /dev/null and b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/part/part-00000-e664e655-0ada-4e0b-a8aa-213d1e4c3797-c000.snappy.parquet differ diff --git a/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/partsupp/part-00000-2c7e524c-3f91-4228-b003-64ad2c8059a1-c000.snappy.parquet b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/partsupp/part-00000-2c7e524c-3f91-4228-b003-64ad2c8059a1-c000.snappy.parquet new file mode 100644 index 000000000..3362ea327 Binary files /dev/null and b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/partsupp/part-00000-2c7e524c-3f91-4228-b003-64ad2c8059a1-c000.snappy.parquet differ diff --git a/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/region/part-00000-90d7c5d9-46eb-4bf3-8f00-6eee7436761d-c000.snappy.parquet b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/region/part-00000-90d7c5d9-46eb-4bf3-8f00-6eee7436761d-c000.snappy.parquet new file mode 100644 index 000000000..dea986b42 Binary files /dev/null and b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/region/part-00000-90d7c5d9-46eb-4bf3-8f00-6eee7436761d-c000.snappy.parquet differ diff --git a/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/supplier/part-00000-0a763951-8a7b-4d7f-a34a-c97e9b4859f0-c000.snappy.parquet b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/supplier/part-00000-0a763951-8a7b-4d7f-a34a-c97e9b4859f0-c000.snappy.parquet new file mode 100644 index 000000000..b72e647f4 Binary files /dev/null and b/spark-extension-shims-spark/src/test/resources/tpch-data-parquet/supplier/part-00000-0a763951-8a7b-4d7f-a34a-c97e9b4859f0-c000.snappy.parquet differ diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q1.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q1.txt new file mode 100644 index 000000000..948b7d690 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q1.txt @@ -0,0 +1,151 @@ +== Physical Plan == +AdaptiveSparkPlan (26) ++- == Final Plan == + NativeSort (17) + +- InputAdapter (16) + +- AQEShuffleRead (15) + +- ShuffleQueryStage (14), Statistics(sizeInBytes=2.6 KiB, rowCount=4) + +- NativeShuffleExchange (13) + +- NativeProject (12) + +- NativeHashAggregate (11) + +- InputAdapter (10) + +- AQEShuffleRead (9) + +- ShuffleQueryStage (8), Statistics(sizeInBytes=1736.0 B, rowCount=4) + +- NativeShuffleExchange (7) + +- NativeHashAggregate (6) + +- NativeProject (5) + +- NativeProject (4) + +- NativeFilter (3) + +- InputAdapter (2) + +- NativeParquetScan (1) ++- == Initial Plan == + Sort (25) + +- Exchange (24) + +- HashAggregate (23) + +- Exchange (22) + +- HashAggregate (21) + +- Project (20) + +- Filter (19) + +- Scan parquet (18) + + +(18) Scan parquet +Output [7]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6, l_shipdate#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] +ReadSchema: struct + +(2) InputAdapter +Input [7]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6, l_shipdate#7] +Arguments: [#1, #2, #3, #4, #5, #6, #7] + +(3) NativeFilter +Input [7]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7] +Condition : (isnotnull(l_shipdate#7) AND (l_shipdate#7 <= 1998-09-02)) + +(4) NativeProject +Output [6]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6] +Input [7]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7] + +(5) NativeProject +Output [7]: [l_returnflag#5 AS l_returnflag#5, l_linestatus#6 AS l_linestatus#6, l_quantity#1 AS _c2#8, l_extendedprice#2 AS _c3#9, (l_extendedprice#2 * (1 - l_discount#3)) AS _c4#10, ((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)) AS _c5#11, l_discount#3 AS _c6#12] +Input [6]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6] + +(6) NativeHashAggregate +Input [7]: [l_returnflag#5, l_linestatus#6, _c2#8, _c3#9, _c4#10, _c5#11, _c6#12] +Keys [2]: [l_returnflag#5, l_linestatus#6] +Functions [8]: [partial_sum(_c2#8), partial_sum(_c3#9), partial_sum(_c4#10), partial_sum(_c5#11), partial_avg(_c2#8), partial_avg(_c3#9), partial_avg(_c6#12), partial_count(1)] +Aggregate Attributes [15]: [sum#13, isEmpty#14, sum#15, isEmpty#16, sum#17, isEmpty#18, sum#19, isEmpty#20, sum#21, count#22, sum#23, count#24, sum#25, count#26, count#27] +Results [3]: [l_returnflag#5, l_linestatus#6, #28#28] + +(7) NativeShuffleExchange +Input [3]: [l_returnflag#5, l_linestatus#6, #28#28] +Arguments: hashpartitioning(l_returnflag#5, l_linestatus#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(8) ShuffleQueryStage +Output [3]: [l_returnflag#5, l_linestatus#6, #28#28] +Arguments: X + +(9) AQEShuffleRead +Input [3]: [l_returnflag#5, l_linestatus#6, #28#28] +Arguments: coalesced + +(10) InputAdapter +Input [3]: [l_returnflag#5, l_linestatus#6, #28#28] + +(11) NativeHashAggregate +Input [3]: [l_returnflag#5, l_linestatus#6, #28#28] +Keys [2]: [l_returnflag#5, l_linestatus#6] +Functions [8]: [sum(l_quantity#1), sum(l_extendedprice#2), sum((l_extendedprice#2 * (1 - l_discount#3))), sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4))), avg(l_quantity#1), avg(l_extendedprice#2), avg(l_discount#3), count(1)] +Aggregate Attributes [8]: [sum(l_quantity#1)#29, sum(l_extendedprice#2)#30, sum((l_extendedprice#2 * (1 - l_discount#3)))#31, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32, avg(l_quantity#1)#33, avg(l_extendedprice#2)#34, avg(l_discount#3)#35, count(1)#36] +Results [10]: [l_returnflag#5, l_linestatus#6, sum(l_quantity#1)#29, sum(l_extendedprice#2)#30, sum((l_extendedprice#2 * (1 - l_discount#3)))#31, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32, avg(l_quantity#1)#33, avg(l_extendedprice#2)#34, avg(l_discount#3)#35, count(1)#36] + +(12) NativeProject +Output [10]: [l_returnflag#5, l_linestatus#6, sum(l_quantity#1)#29 AS sum_qty#37, sum(l_extendedprice#2)#30 AS sum_base_price#38, sum((l_extendedprice#2 * (1 - l_discount#3)))#31 AS sum_disc_price#39, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32 AS sum_charge#40, avg(l_quantity#1)#33 AS avg_qty#41, avg(l_extendedprice#2)#34 AS avg_price#42, avg(l_discount#3)#35 AS avg_disc#43, count(1)#36 AS count_order#44] +Input [10]: [l_returnflag#5, l_linestatus#6, sum(l_quantity#1)#29, sum(l_extendedprice#2)#30, sum((l_extendedprice#2 * (1 - l_discount#3)))#31, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32, avg(l_quantity#1)#33, avg(l_extendedprice#2)#34, avg(l_discount#3)#35, count(1)#36] + +(13) NativeShuffleExchange +Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44] +Arguments: rangepartitioning(l_returnflag#5 ASC NULLS FIRST, l_linestatus#6 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44] +Arguments: X + +(15) AQEShuffleRead +Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44] +Arguments: coalesced + +(16) InputAdapter +Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44] + +(17) NativeSort +Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44] +Arguments: [l_returnflag#5 ASC NULLS FIRST, l_linestatus#6 ASC NULLS FIRST], true + +(18) Scan parquet +Output [7]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6, l_shipdate#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] +ReadSchema: struct + +(19) Filter +Input [7]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6, l_shipdate#7] +Condition : (isnotnull(l_shipdate#7) AND (l_shipdate#7 <= 1998-09-02)) + +(20) Project +Output [6]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6] +Input [7]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6, l_shipdate#7] + +(21) HashAggregate +Input [6]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_tax#4, l_returnflag#5, l_linestatus#6] +Keys [2]: [l_returnflag#5, l_linestatus#6] +Functions [8]: [partial_sum(l_quantity#1), partial_sum(l_extendedprice#2), partial_sum((l_extendedprice#2 * (1 - l_discount#3))), partial_sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4))), partial_avg(l_quantity#1), partial_avg(l_extendedprice#2), partial_avg(l_discount#3), partial_count(1)] +Aggregate Attributes [15]: [sum#13, isEmpty#14, sum#15, isEmpty#16, sum#17, isEmpty#18, sum#19, isEmpty#20, sum#21, count#22, sum#23, count#24, sum#25, count#26, count#27] +Results [17]: [l_returnflag#5, l_linestatus#6, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, count#59] + +(22) Exchange +Input [17]: [l_returnflag#5, l_linestatus#6, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, count#59] +Arguments: hashpartitioning(l_returnflag#5, l_linestatus#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(23) HashAggregate +Input [17]: [l_returnflag#5, l_linestatus#6, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, count#59] +Keys [2]: [l_returnflag#5, l_linestatus#6] +Functions [8]: [sum(l_quantity#1), sum(l_extendedprice#2), sum((l_extendedprice#2 * (1 - l_discount#3))), sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4))), avg(l_quantity#1), avg(l_extendedprice#2), avg(l_discount#3), count(1)] +Aggregate Attributes [8]: [sum(l_quantity#1)#29, sum(l_extendedprice#2)#30, sum((l_extendedprice#2 * (1 - l_discount#3)))#31, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32, avg(l_quantity#1)#33, avg(l_extendedprice#2)#34, avg(l_discount#3)#35, count(1)#36] +Results [10]: [l_returnflag#5, l_linestatus#6, sum(l_quantity#1)#29 AS sum_qty#37, sum(l_extendedprice#2)#30 AS sum_base_price#38, sum((l_extendedprice#2 * (1 - l_discount#3)))#31 AS sum_disc_price#39, sum(((l_extendedprice#2 * (1 - l_discount#3)) * (1 + l_tax#4)))#32 AS sum_charge#40, avg(l_quantity#1)#33 AS avg_qty#41, avg(l_extendedprice#2)#34 AS avg_price#42, avg(l_discount#3)#35 AS avg_disc#43, count(1)#36 AS count_order#44] + +(24) Exchange +Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44] +Arguments: rangepartitioning(l_returnflag#5 ASC NULLS FIRST, l_linestatus#6 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(25) Sort +Input [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44] +Arguments: [l_returnflag#5 ASC NULLS FIRST, l_linestatus#6 ASC NULLS FIRST], true, 0 + +(26) AdaptiveSparkPlan +Output [10]: [l_returnflag#5, l_linestatus#6, sum_qty#37, sum_base_price#38, sum_disc_price#39, sum_charge#40, avg_qty#41, avg_price#42, avg_disc#43, count_order#44] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q10.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q10.txt new file mode 100644 index 000000000..f15681a24 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q10.txt @@ -0,0 +1,506 @@ +== Physical Plan == +AdaptiveSparkPlan (92) ++- == Final Plan == + NativeTakeOrdered (59) + +- NativeProject (58) + +- NativeHashAggregate (57) + +- InputAdapter (56) + +- AQEShuffleRead (55) + +- ShuffleQueryStage (54), Statistics(sizeInBytes=83.0 KiB, rowCount=399) + +- NativeShuffleExchange (53) + +- NativeHashAggregate (52) + +- NativeProject (51) + +- NativeProject (50) + +- NativeSortMergeJoin Inner (49) + :- NativeSort (40) + : +- InputAdapter (39) + : +- AQEShuffleRead (38) + : +- ShuffleQueryStage (37), Statistics(sizeInBytes=264.6 KiB, rowCount=1.26E+3) + : +- NativeShuffleExchange (36) + : +- NativeProject (35) + : +- NativeSortMergeJoin Inner (34) + : :- NativeSort (24) + : : +- InputAdapter (23) + : : +- AQEShuffleRead (22) + : : +- ShuffleQueryStage (21), Statistics(sizeInBytes=114.5 KiB, rowCount=611) + : : +- NativeShuffleExchange (20) + : : +- NativeProject (19) + : : +- NativeSortMergeJoin Inner (18) + : : :- NativeSort (8) + : : : +- InputAdapter (7) + : : : +- AQEShuffleRead (6) + : : : +- ShuffleQueryStage (5), Statistics(sizeInBytes=286.9 KiB, rowCount=1.50E+3) + : : : +- NativeShuffleExchange (4) + : : : +- NativeFilter (3) + : : : +- InputAdapter (2) + : : : +- NativeParquetScan (1) + : : +- NativeSort (17) + : : +- InputAdapter (16) + : : +- AQEShuffleRead (15) + : : +- ShuffleQueryStage (14), Statistics(sizeInBytes=10.0 KiB, rowCount=611) + : : +- NativeShuffleExchange (13) + : : +- NativeProject (12) + : : +- NativeFilter (11) + : : +- InputAdapter (10) + : : +- NativeParquetScan (9) + : +- NativeSort (33) + : +- InputAdapter (32) + : +- AQEShuffleRead (31) + : +- ShuffleQueryStage (30), Statistics(sizeInBytes=583.3 KiB, rowCount=1.49E+4) + : +- NativeShuffleExchange (29) + : +- NativeProject (28) + : +- NativeFilter (27) + : +- InputAdapter (26) + : +- NativeParquetScan (25) + +- NativeSort (48) + +- InputAdapter (47) + +- AQEShuffleRead (46) + +- ShuffleQueryStage (45), Statistics(sizeInBytes=997.0 B, rowCount=25) + +- NativeShuffleExchange (44) + +- NativeFilter (43) + +- InputAdapter (42) + +- NativeParquetScan (41) ++- == Initial Plan == + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- SortMergeJoin Inner (86) + :- Sort (81) + : +- Exchange (80) + : +- Project (79) + : +- SortMergeJoin Inner (78) + : :- Sort (72) + : : +- Exchange (71) + : : +- Project (70) + : : +- SortMergeJoin Inner (69) + : : :- Sort (63) + : : : +- Exchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- Sort (68) + : : +- Exchange (67) + : : +- Project (66) + : : +- Filter (65) + : : +- Scan parquet (64) + : +- Sort (77) + : +- Exchange (76) + : +- Project (75) + : +- Filter (74) + : +- Scan parquet (73) + +- Sort (85) + +- Exchange (84) + +- Filter (83) + +- Scan parquet (82) + + +(60) Scan parquet +Output [7]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(2) InputAdapter +Input [7]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7] +Arguments: [#8, #2, #3, #9, #5, #6, #7] + +(3) NativeFilter +Input [7]: [#8#1, #2#2, #3#3, #9#4, #5#5, #6#6, #7#7] +Condition : (isnotnull(c_custkey#1) AND isnotnull(c_nationkey#4)) + +(4) NativeShuffleExchange +Input [7]: [#8#1, #2#2, #3#3, #9#4, #5#5, #6#6, #7#7] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [7]: [#8#1, #2#2, #3#3, #9#4, #5#5, #6#6, #7#7] +Arguments: X + +(6) AQEShuffleRead +Input [7]: [#8#1, #2#2, #3#3, #9#4, #5#5, #6#6, #7#7] +Arguments: coalesced + +(7) InputAdapter +Input [7]: [#8#1, #2#2, #3#3, #9#4, #5#5, #6#6, #7#7] + +(8) NativeSort +Input [7]: [#8#1, #2#2, #3#3, #9#4, #5#5, #6#6, #7#7] +Arguments: [c_custkey#1 ASC NULLS FIRST], false + +(64) Scan parquet +Output [3]: [o_orderkey#10, o_custkey#11, o_orderdate#12] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(10) InputAdapter +Input [3]: [o_orderkey#10, o_custkey#11, o_orderdate#12] +Arguments: [#13, #14, #12] + +(11) NativeFilter +Input [3]: [#13#10, #14#11, #12#12] +Condition : ((((isnotnull(o_orderdate#12) AND (o_orderdate#12 >= 1993-10-01)) AND (o_orderdate#12 < 1994-01-01)) AND isnotnull(o_custkey#11)) AND isnotnull(o_orderkey#10)) + +(12) NativeProject +Output [2]: [o_orderkey#10, o_custkey#11] +Input [3]: [#13#10, #14#11, #12#12] + +(13) NativeShuffleExchange +Input [2]: [o_orderkey#10, o_custkey#11] +Arguments: hashpartitioning(o_custkey#11, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [2]: [o_orderkey#10, o_custkey#11] +Arguments: X + +(15) AQEShuffleRead +Input [2]: [o_orderkey#10, o_custkey#11] +Arguments: coalesced + +(16) InputAdapter +Input [2]: [o_orderkey#10, o_custkey#11] + +(17) NativeSort +Input [2]: [o_orderkey#10, o_custkey#11] +Arguments: [o_custkey#11 ASC NULLS FIRST], false + +(18) NativeSortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#11] +Join type: Inner +Join condition: None + +(19) NativeProject +Output [8]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10] +Input [9]: [#8#1, #2#2, #3#3, #9#4, #5#5, #6#6, #7#7, o_orderkey#10, o_custkey#11] + +(20) NativeShuffleExchange +Input [8]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10] +Arguments: hashpartitioning(o_orderkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(21) ShuffleQueryStage +Output [8]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10] +Arguments: X + +(22) AQEShuffleRead +Input [8]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10] +Arguments: coalesced + +(23) InputAdapter +Input [8]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10] + +(24) NativeSort +Input [8]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10] +Arguments: [o_orderkey#10 ASC NULLS FIRST], false + +(73) Scan parquet +Output [4]: [l_orderkey#15, l_extendedprice#16, l_discount#17, l_returnflag#18] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] +ReadSchema: struct + +(26) InputAdapter +Input [4]: [l_orderkey#15, l_extendedprice#16, l_discount#17, l_returnflag#18] +Arguments: [#19, #16, #17, #18] + +(27) NativeFilter +Input [4]: [#19#15, #16#16, #17#17, #18#18] +Condition : ((isnotnull(l_returnflag#18) AND (l_returnflag#18 = R)) AND isnotnull(l_orderkey#15)) + +(28) NativeProject +Output [3]: [l_orderkey#15, l_extendedprice#16, l_discount#17] +Input [4]: [#19#15, #16#16, #17#17, #18#18] + +(29) NativeShuffleExchange +Input [3]: [l_orderkey#15, l_extendedprice#16, l_discount#17] +Arguments: hashpartitioning(l_orderkey#15, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(30) ShuffleQueryStage +Output [3]: [l_orderkey#15, l_extendedprice#16, l_discount#17] +Arguments: X + +(31) AQEShuffleRead +Input [3]: [l_orderkey#15, l_extendedprice#16, l_discount#17] +Arguments: coalesced + +(32) InputAdapter +Input [3]: [l_orderkey#15, l_extendedprice#16, l_discount#17] + +(33) NativeSort +Input [3]: [l_orderkey#15, l_extendedprice#16, l_discount#17] +Arguments: [l_orderkey#15 ASC NULLS FIRST], false + +(34) NativeSortMergeJoin +Left keys [1]: [o_orderkey#10] +Right keys [1]: [l_orderkey#15] +Join type: Inner +Join condition: None + +(35) NativeProject +Output [9]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17] +Input [11]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10, l_orderkey#15, l_extendedprice#16, l_discount#17] + +(36) NativeShuffleExchange +Input [9]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17] +Arguments: hashpartitioning(c_nationkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(37) ShuffleQueryStage +Output [9]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17] +Arguments: X + +(38) AQEShuffleRead +Input [9]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17] +Arguments: coalesced + +(39) InputAdapter +Input [9]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17] + +(40) NativeSort +Input [9]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17] +Arguments: [c_nationkey#4 ASC NULLS FIRST], false + +(82) Scan parquet +Output [2]: [n_nationkey#20, n_name#21] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(42) InputAdapter +Input [2]: [n_nationkey#20, n_name#21] +Arguments: [#22, #21] + +(43) NativeFilter +Input [2]: [#22#20, #21#21] +Condition : isnotnull(n_nationkey#20) + +(44) NativeShuffleExchange +Input [2]: [#22#20, #21#21] +Arguments: hashpartitioning(n_nationkey#20, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(45) ShuffleQueryStage +Output [2]: [#22#20, #21#21] +Arguments: X + +(46) AQEShuffleRead +Input [2]: [#22#20, #21#21] +Arguments: coalesced + +(47) InputAdapter +Input [2]: [#22#20, #21#21] + +(48) NativeSort +Input [2]: [#22#20, #21#21] +Arguments: [n_nationkey#20 ASC NULLS FIRST], false + +(49) NativeSortMergeJoin +Left keys [1]: [c_nationkey#4] +Right keys [1]: [n_nationkey#20] +Join type: Inner +Join condition: None + +(50) NativeProject +Output [9]: [c_custkey#1, c_name#2, c_address#3, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17, n_name#21] +Input [11]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17, #22#20, #21#21] + +(51) NativeProject +Output [8]: [c_custkey#1 AS c_custkey#1, c_name#2 AS c_name#2, c_acctbal#6 AS c_acctbal#6, c_phone#5 AS c_phone#5, n_name#21 AS n_name#21, c_address#3 AS c_address#3, c_comment#7 AS c_comment#7, (l_extendedprice#16 * (1 - l_discount#17)) AS _c7#23] +Input [9]: [c_custkey#1, c_name#2, c_address#3, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17, n_name#21] + +(52) NativeHashAggregate +Input [8]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, _c7#23] +Keys [7]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7] +Functions [1]: [partial_sum(_c7#23)] +Aggregate Attributes [2]: [sum#24, isEmpty#25] +Results [8]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, #26#26] + +(53) NativeShuffleExchange +Input [8]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, #26#26] +Arguments: hashpartitioning(c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(54) ShuffleQueryStage +Output [8]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, #26#26] +Arguments: X + +(55) AQEShuffleRead +Input [8]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, #26#26] +Arguments: coalesced + +(56) InputAdapter +Input [8]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, #26#26] + +(57) NativeHashAggregate +Input [8]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, #26#26] +Keys [7]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7] +Functions [1]: [sum((l_extendedprice#16 * (1 - l_discount#17)))] +Aggregate Attributes [1]: [sum((l_extendedprice#16 * (1 - l_discount#17)))#27] +Results [8]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, sum((l_extendedprice#16 * (1 - l_discount#17)))#27] + +(58) NativeProject +Output [8]: [c_custkey#1, c_name#2, sum((l_extendedprice#16 * (1 - l_discount#17)))#27 AS revenue#28, c_acctbal#6, n_name#21, c_address#3, c_phone#5, c_comment#7] +Input [8]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, sum((l_extendedprice#16 * (1 - l_discount#17)))#27] + +(59) NativeTakeOrdered +Input [8]: [c_custkey#1, c_name#2, revenue#28, c_acctbal#6, n_name#21, c_address#3, c_phone#5, c_comment#7] +Arguments: X, [revenue#28 DESC NULLS LAST] + +(60) Scan parquet +Output [7]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(61) Filter +Input [7]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7] +Condition : (isnotnull(c_custkey#1) AND isnotnull(c_nationkey#4)) + +(62) Exchange +Input [7]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(63) Sort +Input [7]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7] +Arguments: [c_custkey#1 ASC NULLS FIRST], false, 0 + +(64) Scan parquet +Output [3]: [o_orderkey#10, o_custkey#11, o_orderdate#12] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(65) Filter +Input [3]: [o_orderkey#10, o_custkey#11, o_orderdate#12] +Condition : ((((isnotnull(o_orderdate#12) AND (o_orderdate#12 >= 1993-10-01)) AND (o_orderdate#12 < 1994-01-01)) AND isnotnull(o_custkey#11)) AND isnotnull(o_orderkey#10)) + +(66) Project +Output [2]: [o_orderkey#10, o_custkey#11] +Input [3]: [o_orderkey#10, o_custkey#11, o_orderdate#12] + +(67) Exchange +Input [2]: [o_orderkey#10, o_custkey#11] +Arguments: hashpartitioning(o_custkey#11, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(68) Sort +Input [2]: [o_orderkey#10, o_custkey#11] +Arguments: [o_custkey#11 ASC NULLS FIRST], false, 0 + +(69) SortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#11] +Join type: Inner +Join condition: None + +(70) Project +Output [8]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10] +Input [9]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10, o_custkey#11] + +(71) Exchange +Input [8]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10] +Arguments: hashpartitioning(o_orderkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(72) Sort +Input [8]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10] +Arguments: [o_orderkey#10 ASC NULLS FIRST], false, 0 + +(73) Scan parquet +Output [4]: [l_orderkey#15, l_extendedprice#16, l_discount#17, l_returnflag#18] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] +ReadSchema: struct + +(74) Filter +Input [4]: [l_orderkey#15, l_extendedprice#16, l_discount#17, l_returnflag#18] +Condition : ((isnotnull(l_returnflag#18) AND (l_returnflag#18 = R)) AND isnotnull(l_orderkey#15)) + +(75) Project +Output [3]: [l_orderkey#15, l_extendedprice#16, l_discount#17] +Input [4]: [l_orderkey#15, l_extendedprice#16, l_discount#17, l_returnflag#18] + +(76) Exchange +Input [3]: [l_orderkey#15, l_extendedprice#16, l_discount#17] +Arguments: hashpartitioning(l_orderkey#15, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(77) Sort +Input [3]: [l_orderkey#15, l_extendedprice#16, l_discount#17] +Arguments: [l_orderkey#15 ASC NULLS FIRST], false, 0 + +(78) SortMergeJoin +Left keys [1]: [o_orderkey#10] +Right keys [1]: [l_orderkey#15] +Join type: Inner +Join condition: None + +(79) Project +Output [9]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17] +Input [11]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, o_orderkey#10, l_orderkey#15, l_extendedprice#16, l_discount#17] + +(80) Exchange +Input [9]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17] +Arguments: hashpartitioning(c_nationkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=12] + +(81) Sort +Input [9]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17] +Arguments: [c_nationkey#4 ASC NULLS FIRST], false, 0 + +(82) Scan parquet +Output [2]: [n_nationkey#20, n_name#21] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(83) Filter +Input [2]: [n_nationkey#20, n_name#21] +Condition : isnotnull(n_nationkey#20) + +(84) Exchange +Input [2]: [n_nationkey#20, n_name#21] +Arguments: hashpartitioning(n_nationkey#20, 5), ENSURE_REQUIREMENTS, [plan_id=13] + +(85) Sort +Input [2]: [n_nationkey#20, n_name#21] +Arguments: [n_nationkey#20 ASC NULLS FIRST], false, 0 + +(86) SortMergeJoin +Left keys [1]: [c_nationkey#4] +Right keys [1]: [n_nationkey#20] +Join type: Inner +Join condition: None + +(87) Project +Output [9]: [c_custkey#1, c_name#2, c_address#3, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17, n_name#21] +Input [11]: [c_custkey#1, c_name#2, c_address#3, c_nationkey#4, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17, n_nationkey#20, n_name#21] + +(88) HashAggregate +Input [9]: [c_custkey#1, c_name#2, c_address#3, c_phone#5, c_acctbal#6, c_comment#7, l_extendedprice#16, l_discount#17, n_name#21] +Keys [7]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7] +Functions [1]: [partial_sum((l_extendedprice#16 * (1 - l_discount#17)))] +Aggregate Attributes [2]: [sum#24, isEmpty#25] +Results [9]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, sum#29, isEmpty#30] + +(89) Exchange +Input [9]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, sum#29, isEmpty#30] +Arguments: hashpartitioning(c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, 5), ENSURE_REQUIREMENTS, [plan_id=14] + +(90) HashAggregate +Input [9]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7, sum#29, isEmpty#30] +Keys [7]: [c_custkey#1, c_name#2, c_acctbal#6, c_phone#5, n_name#21, c_address#3, c_comment#7] +Functions [1]: [sum((l_extendedprice#16 * (1 - l_discount#17)))] +Aggregate Attributes [1]: [sum((l_extendedprice#16 * (1 - l_discount#17)))#27] +Results [8]: [c_custkey#1, c_name#2, sum((l_extendedprice#16 * (1 - l_discount#17)))#27 AS revenue#28, c_acctbal#6, n_name#21, c_address#3, c_phone#5, c_comment#7] + +(91) TakeOrderedAndProject +Input [8]: [c_custkey#1, c_name#2, revenue#28, c_acctbal#6, n_name#21, c_address#3, c_phone#5, c_comment#7] +Arguments: X, [revenue#28 DESC NULLS LAST], [c_custkey#1, c_name#2, revenue#28, c_acctbal#6, n_name#21, c_address#3, c_phone#5, c_comment#7] + +(92) AdaptiveSparkPlan +Output [8]: [c_custkey#1, c_name#2, revenue#28, c_acctbal#6, n_name#21, c_address#3, c_phone#5, c_comment#7] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q11.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q11.txt new file mode 100644 index 000000000..ee41033f1 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q11.txt @@ -0,0 +1,731 @@ +== Physical Plan == +AdaptiveSparkPlan (74) ++- == Final Plan == + NativeSort (48) + +- ConvertToNative (47) + +- AQEShuffleRead (46) + +- ShuffleQueryStage (45), Statistics(sizeInBytes=14.0 KiB, rowCount=359) + +- Exchange (44) + +- NativeFilter (43) + +- NativeProject (42) + +- NativeHashAggregate (41) + +- InputAdapter (40) + +- AQEShuffleRead (39) + +- ShuffleQueryStage (38), Statistics(sizeInBytes=12.8 KiB, rowCount=374) + +- NativeShuffleExchange (37) + +- NativeHashAggregate (36) + +- NativeProject (35) + +- NativeProject (34) + +- NativeSortMergeJoin Inner (33) + :- NativeSort (23) + : +- InputAdapter (22) + : +- AQEShuffleRead (21) + : +- ShuffleQueryStage (20), Statistics(sizeInBytes=282.0 KiB, rowCount=8.00E+3) + : +- NativeShuffleExchange (19) + : +- NativeProject (18) + : +- NativeSortMergeJoin Inner (17) + : :- NativeSort (8) + : : +- InputAdapter (7) + : : +- AQEShuffleRead (6) + : : +- ShuffleQueryStage (5), Statistics(sizeInBytes=282.0 KiB, rowCount=8.00E+3) + : : +- NativeShuffleExchange (4) + : : +- NativeFilter (3) + : : +- InputAdapter (2) + : : +- NativeParquetScan (1) + : +- NativeSort (16) + : +- InputAdapter (15) + : +- AQEShuffleRead (14) + : +- ShuffleQueryStage (13), Statistics(sizeInBytes=1992.0 B, rowCount=100) + : +- NativeShuffleExchange (12) + : +- NativeFilter (11) + : +- InputAdapter (10) + : +- NativeParquetScan (9) + +- NativeSort (32) + +- InputAdapter (31) + +- AQEShuffleRead (30) + +- ShuffleQueryStage (29), Statistics(sizeInBytes=216.0 B, rowCount=1) + +- NativeShuffleExchange (28) + +- NativeProject (27) + +- NativeFilter (26) + +- InputAdapter (25) + +- NativeParquetScan (24) ++- == Initial Plan == + Sort (73) + +- Exchange (72) + +- Filter (71) + +- HashAggregate (70) + +- Exchange (69) + +- HashAggregate (68) + +- Project (67) + +- SortMergeJoin Inner (66) + :- Sort (60) + : +- Exchange (59) + : +- Project (58) + : +- SortMergeJoin Inner (57) + : :- Sort (52) + : : +- Exchange (51) + : : +- Filter (50) + : : +- Scan parquet (49) + : +- Sort (56) + : +- Exchange (55) + : +- Filter (54) + : +- Scan parquet (53) + +- Sort (65) + +- Exchange (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) + + +(49) Scan parquet +Output [4]: [ps_partkey#1, ps_suppkey#2, ps_availqty#3, ps_supplycost#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(2) InputAdapter +Input [4]: [ps_partkey#1, ps_suppkey#2, ps_availqty#3, ps_supplycost#4] +Arguments: [#5, #6, #3, #4] + +(3) NativeFilter +Input [4]: [#5#1, #6#2, #3#3, #4#4] +Condition : isnotnull(ps_suppkey#2) + +(4) NativeShuffleExchange +Input [4]: [#5#1, #6#2, #3#3, #4#4] +Arguments: hashpartitioning(ps_suppkey#2, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [4]: [#5#1, #6#2, #3#3, #4#4] +Arguments: X + +(6) AQEShuffleRead +Input [4]: [#5#1, #6#2, #3#3, #4#4] +Arguments: coalesced + +(7) InputAdapter +Input [4]: [#5#1, #6#2, #3#3, #4#4] + +(8) NativeSort +Input [4]: [#5#1, #6#2, #3#3, #4#4] +Arguments: [ps_suppkey#2 ASC NULLS FIRST], false + +(53) Scan parquet +Output [2]: [s_suppkey#7, s_nationkey#8] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(10) InputAdapter +Input [2]: [s_suppkey#7, s_nationkey#8] +Arguments: [#9, #10] + +(11) NativeFilter +Input [2]: [#9#7, #10#8] +Condition : (isnotnull(s_suppkey#7) AND isnotnull(s_nationkey#8)) + +(12) NativeShuffleExchange +Input [2]: [#9#7, #10#8] +Arguments: hashpartitioning(s_suppkey#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(13) ShuffleQueryStage +Output [2]: [#9#7, #10#8] +Arguments: X + +(14) AQEShuffleRead +Input [2]: [#9#7, #10#8] +Arguments: coalesced + +(15) InputAdapter +Input [2]: [#9#7, #10#8] + +(16) NativeSort +Input [2]: [#9#7, #10#8] +Arguments: [s_suppkey#7 ASC NULLS FIRST], false + +(17) NativeSortMergeJoin +Left keys [1]: [ps_suppkey#2] +Right keys [1]: [s_suppkey#7] +Join type: Inner +Join condition: None + +(18) NativeProject +Output [4]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8] +Input [6]: [#5#1, #6#2, #3#3, #4#4, #9#7, #10#8] + +(19) NativeShuffleExchange +Input [4]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8] +Arguments: hashpartitioning(s_nationkey#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(20) ShuffleQueryStage +Output [4]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8] +Arguments: X + +(21) AQEShuffleRead +Input [4]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8] +Arguments: coalesced + +(22) InputAdapter +Input [4]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8] + +(23) NativeSort +Input [4]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8] +Arguments: [s_nationkey#8 ASC NULLS FIRST], false + +(61) Scan parquet +Output [2]: [n_nationkey#11, n_name#12] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(25) InputAdapter +Input [2]: [n_nationkey#11, n_name#12] +Arguments: [#13, #12] + +(26) NativeFilter +Input [2]: [#13#11, #12#12] +Condition : ((isnotnull(n_name#12) AND (n_name#12 = GERMANY)) AND isnotnull(n_nationkey#11)) + +(27) NativeProject +Output [1]: [n_nationkey#11] +Input [2]: [#13#11, #12#12] + +(28) NativeShuffleExchange +Input [1]: [n_nationkey#11] +Arguments: hashpartitioning(n_nationkey#11, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(29) ShuffleQueryStage +Output [1]: [n_nationkey#11] +Arguments: X + +(30) AQEShuffleRead +Input [1]: [n_nationkey#11] +Arguments: coalesced + +(31) InputAdapter +Input [1]: [n_nationkey#11] + +(32) NativeSort +Input [1]: [n_nationkey#11] +Arguments: [n_nationkey#11 ASC NULLS FIRST], false + +(33) NativeSortMergeJoin +Left keys [1]: [s_nationkey#8] +Right keys [1]: [n_nationkey#11] +Join type: Inner +Join condition: None + +(34) NativeProject +Output [3]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4] +Input [5]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8, n_nationkey#11] + +(35) NativeProject +Output [2]: [ps_partkey#1 AS ps_partkey#1, (ps_supplycost#4 * cast(ps_availqty#3 as decimal(10,0))) AS _c1#14] +Input [3]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4] + +(36) NativeHashAggregate +Input [2]: [ps_partkey#1, _c1#14] +Keys [1]: [ps_partkey#1] +Functions [1]: [partial_sum(_c1#14)] +Aggregate Attributes [2]: [sum#15, isEmpty#16] +Results [2]: [ps_partkey#1, #17#17] + +(37) NativeShuffleExchange +Input [2]: [ps_partkey#1, #17#17] +Arguments: hashpartitioning(ps_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(38) ShuffleQueryStage +Output [2]: [ps_partkey#1, #17#17] +Arguments: X + +(39) AQEShuffleRead +Input [2]: [ps_partkey#1, #17#17] +Arguments: coalesced + +(40) InputAdapter +Input [2]: [ps_partkey#1, #17#17] + +(41) NativeHashAggregate +Input [2]: [ps_partkey#1, #17#17] +Keys [1]: [ps_partkey#1] +Functions [1]: [sum((ps_supplycost#4 * cast(ps_availqty#3 as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#4 * cast(ps_availqty#3 as decimal(10,0))))#18] +Results [2]: [ps_partkey#1, sum((ps_supplycost#4 * cast(ps_availqty#3 as decimal(10,0))))#18] + +(42) NativeProject +Output [2]: [ps_partkey#1, sum((ps_supplycost#4 * cast(ps_availqty#3 as decimal(10,0))))#18 AS value#19] +Input [2]: [ps_partkey#1, sum((ps_supplycost#4 * cast(ps_availqty#3 as decimal(10,0))))#18] + +(43) NativeFilter +Input [2]: [ps_partkey#1, value#19] +Condition : (isnotnull(value#19) AND (cast(value#19 as decimal(38,6)) > Subquery subquery#20, [id=#21])) + +(44) Exchange +Input [2]: [ps_partkey#1, value#19] +Arguments: rangepartitioning(value#19 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(45) ShuffleQueryStage +Output [2]: [ps_partkey#1, value#19] +Arguments: X + +(46) AQEShuffleRead +Input [2]: [ps_partkey#1, value#19] +Arguments: coalesced + +(47) ConvertToNative +Input [2]: [ps_partkey#1, value#19] + +(48) NativeSort +Input [2]: [ps_partkey#1, value#19] +Arguments: [value#19 DESC NULLS LAST], true + +(49) Scan parquet +Output [4]: [ps_partkey#1, ps_suppkey#2, ps_availqty#3, ps_supplycost#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(50) Filter +Input [4]: [ps_partkey#1, ps_suppkey#2, ps_availqty#3, ps_supplycost#4] +Condition : isnotnull(ps_suppkey#2) + +(51) Exchange +Input [4]: [ps_partkey#1, ps_suppkey#2, ps_availqty#3, ps_supplycost#4] +Arguments: hashpartitioning(ps_suppkey#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(52) Sort +Input [4]: [ps_partkey#1, ps_suppkey#2, ps_availqty#3, ps_supplycost#4] +Arguments: [ps_suppkey#2 ASC NULLS FIRST], false, 0 + +(53) Scan parquet +Output [2]: [s_suppkey#7, s_nationkey#8] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(54) Filter +Input [2]: [s_suppkey#7, s_nationkey#8] +Condition : (isnotnull(s_suppkey#7) AND isnotnull(s_nationkey#8)) + +(55) Exchange +Input [2]: [s_suppkey#7, s_nationkey#8] +Arguments: hashpartitioning(s_suppkey#7, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(56) Sort +Input [2]: [s_suppkey#7, s_nationkey#8] +Arguments: [s_suppkey#7 ASC NULLS FIRST], false, 0 + +(57) SortMergeJoin +Left keys [1]: [ps_suppkey#2] +Right keys [1]: [s_suppkey#7] +Join type: Inner +Join condition: None + +(58) Project +Output [4]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8] +Input [6]: [ps_partkey#1, ps_suppkey#2, ps_availqty#3, ps_supplycost#4, s_suppkey#7, s_nationkey#8] + +(59) Exchange +Input [4]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8] +Arguments: hashpartitioning(s_nationkey#8, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(60) Sort +Input [4]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8] +Arguments: [s_nationkey#8 ASC NULLS FIRST], false, 0 + +(61) Scan parquet +Output [2]: [n_nationkey#11, n_name#12] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(62) Filter +Input [2]: [n_nationkey#11, n_name#12] +Condition : ((isnotnull(n_name#12) AND (n_name#12 = GERMANY)) AND isnotnull(n_nationkey#11)) + +(63) Project +Output [1]: [n_nationkey#11] +Input [2]: [n_nationkey#11, n_name#12] + +(64) Exchange +Input [1]: [n_nationkey#11] +Arguments: hashpartitioning(n_nationkey#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(65) Sort +Input [1]: [n_nationkey#11] +Arguments: [n_nationkey#11 ASC NULLS FIRST], false, 0 + +(66) SortMergeJoin +Left keys [1]: [s_nationkey#8] +Right keys [1]: [n_nationkey#11] +Join type: Inner +Join condition: None + +(67) Project +Output [3]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4] +Input [5]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4, s_nationkey#8, n_nationkey#11] + +(68) HashAggregate +Input [3]: [ps_partkey#1, ps_availqty#3, ps_supplycost#4] +Keys [1]: [ps_partkey#1] +Functions [1]: [partial_sum((ps_supplycost#4 * cast(ps_availqty#3 as decimal(10,0))))] +Aggregate Attributes [2]: [sum#15, isEmpty#16] +Results [3]: [ps_partkey#1, sum#22, isEmpty#23] + +(69) Exchange +Input [3]: [ps_partkey#1, sum#22, isEmpty#23] +Arguments: hashpartitioning(ps_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(70) HashAggregate +Input [3]: [ps_partkey#1, sum#22, isEmpty#23] +Keys [1]: [ps_partkey#1] +Functions [1]: [sum((ps_supplycost#4 * cast(ps_availqty#3 as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#4 * cast(ps_availqty#3 as decimal(10,0))))#18] +Results [2]: [ps_partkey#1, sum((ps_supplycost#4 * cast(ps_availqty#3 as decimal(10,0))))#18 AS value#19] + +(71) Filter +Input [2]: [ps_partkey#1, value#19] +Condition : (isnotnull(value#19) AND (cast(value#19 as decimal(38,6)) > Subquery subquery#20, [id=#21])) + +(72) Exchange +Input [2]: [ps_partkey#1, value#19] +Arguments: rangepartitioning(value#19 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=12] + +(73) Sort +Input [2]: [ps_partkey#1, value#19] +Arguments: [value#19 DESC NULLS LAST], true, 0 + +(74) AdaptiveSparkPlan +Output [2]: [ps_partkey#1, value#19] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 43 Hosting Expression = Subquery subquery#20, [id=#21] +AdaptiveSparkPlan (132) ++- == Final Plan == + NativeProject (109) + +- NativeHashAggregate (108) + +- ShuffleQueryStage (107), Statistics(sizeInBytes=360.0 B, rowCount=1) + +- NativeShuffleExchange (106) + +- NativeHashAggregate (105) + +- NativeProject (104) + +- NativeProject (103) + +- NativeSortMergeJoin Inner (102) + :- NativeSort (95) + : +- InputAdapter (94) + : +- AQEShuffleRead (93) + : +- ShuffleQueryStage (92), Statistics(sizeInBytes=219.3 KiB, rowCount=8.00E+3) + : +- NativeShuffleExchange (91) + : +- NativeProject (90) + : +- NativeSortMergeJoin Inner (89) + : :- NativeSort (82) + : : +- InputAdapter (81) + : : +- AQEShuffleRead (80) + : : +- ShuffleQueryStage (79), Statistics(sizeInBytes=219.3 KiB, rowCount=8.00E+3) + : : +- NativeShuffleExchange (78) + : : +- NativeFilter (77) + : : +- InputAdapter (76) + : : +- NativeParquetScan (75) + : +- NativeSort (88) + : +- InputAdapter (87) + : +- InputAdapter (86) + : +- AQEShuffleRead (85) + : +- ShuffleQueryStage (84), Statistics(sizeInBytes=1992.0 B, rowCount=100) + : +- ReusedExchange (83) + +- NativeSort (101) + +- InputAdapter (100) + +- InputAdapter (99) + +- AQEShuffleRead (98) + +- ShuffleQueryStage (97), Statistics(sizeInBytes=216.0 B, rowCount=1) + +- ReusedExchange (96) ++- == Initial Plan == + HashAggregate (131) + +- Exchange (130) + +- HashAggregate (129) + +- Project (128) + +- SortMergeJoin Inner (127) + :- Sort (121) + : +- Exchange (120) + : +- Project (119) + : +- SortMergeJoin Inner (118) + : :- Sort (113) + : : +- Exchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Sort (117) + : +- Exchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- Sort (126) + +- Exchange (125) + +- Project (124) + +- Filter (123) + +- Scan parquet (122) + + +(110) Scan parquet +Output [3]: [ps_suppkey#24, ps_availqty#25, ps_supplycost#26] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(76) InputAdapter +Input [3]: [ps_suppkey#24, ps_availqty#25, ps_supplycost#26] +Arguments: [#27, #25, #26] + +(77) NativeFilter +Input [3]: [#27#24, #25#25, #26#26] +Condition : isnotnull(ps_suppkey#24) + +(78) NativeShuffleExchange +Input [3]: [#27#24, #25#25, #26#26] +Arguments: hashpartitioning(ps_suppkey#24, 5), ENSURE_REQUIREMENTS, [plan_id=13] + +(79) ShuffleQueryStage +Output [3]: [#27#24, #25#25, #26#26] +Arguments: X + +(80) AQEShuffleRead +Input [3]: [#27#24, #25#25, #26#26] +Arguments: coalesced + +(81) InputAdapter +Input [3]: [#27#24, #25#25, #26#26] + +(82) NativeSort +Input [3]: [#27#24, #25#25, #26#26] +Arguments: [ps_suppkey#24 ASC NULLS FIRST], false + +(83) ReusedExchange [Reuses operator id: 12] +Output [2]: [s_suppkey#28, s_nationkey#29] + +(84) ShuffleQueryStage +Output [2]: [s_suppkey#28, s_nationkey#29] +Arguments: X + +(85) AQEShuffleRead +Input [2]: [s_suppkey#28, s_nationkey#29] +Arguments: coalesced + +(86) InputAdapter +Input [2]: [s_suppkey#28, s_nationkey#29] +Arguments: [#30, #31] + +(87) InputAdapter +Input [2]: [#30#28, #31#29] + +(88) NativeSort +Input [2]: [#30#28, #31#29] +Arguments: [s_suppkey#28 ASC NULLS FIRST], false + +(89) NativeSortMergeJoin +Left keys [1]: [ps_suppkey#24] +Right keys [1]: [s_suppkey#28] +Join type: Inner +Join condition: None + +(90) NativeProject +Output [3]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29] +Input [5]: [#27#24, #25#25, #26#26, #30#28, #31#29] + +(91) NativeShuffleExchange +Input [3]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29] +Arguments: hashpartitioning(s_nationkey#29, 5), ENSURE_REQUIREMENTS, [plan_id=14] + +(92) ShuffleQueryStage +Output [3]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29] +Arguments: X + +(93) AQEShuffleRead +Input [3]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29] +Arguments: coalesced + +(94) InputAdapter +Input [3]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29] + +(95) NativeSort +Input [3]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29] +Arguments: [s_nationkey#29 ASC NULLS FIRST], false + +(96) ReusedExchange [Reuses operator id: 28] +Output [1]: [n_nationkey#32] + +(97) ShuffleQueryStage +Output [1]: [n_nationkey#32] +Arguments: X + +(98) AQEShuffleRead +Input [1]: [n_nationkey#32] +Arguments: coalesced + +(99) InputAdapter +Input [1]: [n_nationkey#32] +Arguments: [#33] + +(100) InputAdapter +Input [1]: [#33#32] + +(101) NativeSort +Input [1]: [#33#32] +Arguments: [n_nationkey#32 ASC NULLS FIRST], false + +(102) NativeSortMergeJoin +Left keys [1]: [s_nationkey#29] +Right keys [1]: [n_nationkey#32] +Join type: Inner +Join condition: None + +(103) NativeProject +Output [2]: [ps_availqty#25, ps_supplycost#26] +Input [4]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29, #33#32] + +(104) NativeProject +Output [1]: [(ps_supplycost#26 * cast(ps_availqty#25 as decimal(10,0))) AS _c0#34] +Input [2]: [ps_availqty#25, ps_supplycost#26] + +(105) NativeHashAggregate +Input [1]: [_c0#34] +Keys: [] +Functions [1]: [partial_sum(_c0#34)] +Aggregate Attributes [2]: [sum#35, isEmpty#36] +Results [1]: [#17#17] + +(106) NativeShuffleExchange +Input [1]: [#17#17] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] + +(107) ShuffleQueryStage +Output [1]: [#17#17] +Arguments: X + +(108) NativeHashAggregate +Input [1]: [#17#17] +Keys: [] +Functions [1]: [sum((ps_supplycost#26 * cast(ps_availqty#25 as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#26 * cast(ps_availqty#25 as decimal(10,0))))#37] +Results [1]: [sum((ps_supplycost#26 * cast(ps_availqty#25 as decimal(10,0))))#37] + +(109) NativeProject +Output [1]: [(sum((ps_supplycost#26 * cast(ps_availqty#25 as decimal(10,0))))#37 * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#38] +Input [1]: [sum((ps_supplycost#26 * cast(ps_availqty#25 as decimal(10,0))))#37] + +(110) Scan parquet +Output [3]: [ps_suppkey#24, ps_availqty#25, ps_supplycost#26] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_suppkey)] +ReadSchema: struct + +(111) Filter +Input [3]: [ps_suppkey#24, ps_availqty#25, ps_supplycost#26] +Condition : isnotnull(ps_suppkey#24) + +(112) Exchange +Input [3]: [ps_suppkey#24, ps_availqty#25, ps_supplycost#26] +Arguments: hashpartitioning(ps_suppkey#24, 5), ENSURE_REQUIREMENTS, [plan_id=16] + +(113) Sort +Input [3]: [ps_suppkey#24, ps_availqty#25, ps_supplycost#26] +Arguments: [ps_suppkey#24 ASC NULLS FIRST], false, 0 + +(114) Scan parquet +Output [2]: [s_suppkey#28, s_nationkey#29] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(115) Filter +Input [2]: [s_suppkey#28, s_nationkey#29] +Condition : (isnotnull(s_suppkey#28) AND isnotnull(s_nationkey#29)) + +(116) Exchange +Input [2]: [s_suppkey#28, s_nationkey#29] +Arguments: hashpartitioning(s_suppkey#28, 5), ENSURE_REQUIREMENTS, [plan_id=17] + +(117) Sort +Input [2]: [s_suppkey#28, s_nationkey#29] +Arguments: [s_suppkey#28 ASC NULLS FIRST], false, 0 + +(118) SortMergeJoin +Left keys [1]: [ps_suppkey#24] +Right keys [1]: [s_suppkey#28] +Join type: Inner +Join condition: None + +(119) Project +Output [3]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29] +Input [5]: [ps_suppkey#24, ps_availqty#25, ps_supplycost#26, s_suppkey#28, s_nationkey#29] + +(120) Exchange +Input [3]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29] +Arguments: hashpartitioning(s_nationkey#29, 5), ENSURE_REQUIREMENTS, [plan_id=18] + +(121) Sort +Input [3]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29] +Arguments: [s_nationkey#29 ASC NULLS FIRST], false, 0 + +(122) Scan parquet +Output [2]: [n_nationkey#32, n_name#39] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] +ReadSchema: struct + +(123) Filter +Input [2]: [n_nationkey#32, n_name#39] +Condition : ((isnotnull(n_name#39) AND (n_name#39 = GERMANY)) AND isnotnull(n_nationkey#32)) + +(124) Project +Output [1]: [n_nationkey#32] +Input [2]: [n_nationkey#32, n_name#39] + +(125) Exchange +Input [1]: [n_nationkey#32] +Arguments: hashpartitioning(n_nationkey#32, 5), ENSURE_REQUIREMENTS, [plan_id=19] + +(126) Sort +Input [1]: [n_nationkey#32] +Arguments: [n_nationkey#32 ASC NULLS FIRST], false, 0 + +(127) SortMergeJoin +Left keys [1]: [s_nationkey#29] +Right keys [1]: [n_nationkey#32] +Join type: Inner +Join condition: None + +(128) Project +Output [2]: [ps_availqty#25, ps_supplycost#26] +Input [4]: [ps_availqty#25, ps_supplycost#26, s_nationkey#29, n_nationkey#32] + +(129) HashAggregate +Input [2]: [ps_availqty#25, ps_supplycost#26] +Keys: [] +Functions [1]: [partial_sum((ps_supplycost#26 * cast(ps_availqty#25 as decimal(10,0))))] +Aggregate Attributes [2]: [sum#35, isEmpty#36] +Results [2]: [sum#40, isEmpty#41] + +(130) Exchange +Input [2]: [sum#40, isEmpty#41] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=20] + +(131) HashAggregate +Input [2]: [sum#40, isEmpty#41] +Keys: [] +Functions [1]: [sum((ps_supplycost#26 * cast(ps_availqty#25 as decimal(10,0))))] +Aggregate Attributes [1]: [sum((ps_supplycost#26 * cast(ps_availqty#25 as decimal(10,0))))#37] +Results [1]: [(sum((ps_supplycost#26 * cast(ps_availqty#25 as decimal(10,0))))#37 * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#38] + +(132) AdaptiveSparkPlan +Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#38] +Arguments: isFinalPlan=true + + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q12.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q12.txt new file mode 100644 index 000000000..67592e581 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q12.txt @@ -0,0 +1,274 @@ +== Physical Plan == +AdaptiveSparkPlan (49) ++- == Final Plan == + NativeSort (32) + +- InputAdapter (31) + +- AQEShuffleRead (30) + +- ShuffleQueryStage (29), Statistics(sizeInBytes=760.0 B, rowCount=2) + +- NativeShuffleExchange (28) + +- NativeProject (27) + +- NativeHashAggregate (26) + +- InputAdapter (25) + +- AQEShuffleRead (24) + +- ShuffleQueryStage (23), Statistics(sizeInBytes=696.0 B, rowCount=2) + +- NativeShuffleExchange (22) + +- NativeHashAggregate (21) + +- NativeProject (20) + +- NativeProject (19) + +- NativeSortMergeJoin Inner (18) + :- NativeSort (8) + : +- InputAdapter (7) + : +- AQEShuffleRead (6) + : +- ShuffleQueryStage (5), Statistics(sizeInBytes=406.0 KiB, rowCount=1.50E+4) + : +- NativeShuffleExchange (4) + : +- NativeFilter (3) + : +- InputAdapter (2) + : +- NativeParquetScan (1) + +- NativeSort (17) + +- InputAdapter (16) + +- AQEShuffleRead (15) + +- ShuffleQueryStage (14), Statistics(sizeInBytes=5.3 KiB, rowCount=307) + +- NativeShuffleExchange (13) + +- NativeProject (12) + +- NativeFilter (11) + +- InputAdapter (10) + +- NativeParquetScan (9) ++- == Initial Plan == + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- SortMergeJoin Inner (42) + :- Sort (36) + : +- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Sort (41) + +- Exchange (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) + + +(33) Scan parquet +Output [2]: [o_orderkey#1, o_orderpriority#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(2) InputAdapter +Input [2]: [o_orderkey#1, o_orderpriority#2] +Arguments: [#3, #2] + +(3) NativeFilter +Input [2]: [#3#1, #2#2] +Condition : isnotnull(o_orderkey#1) + +(4) NativeShuffleExchange +Input [2]: [#3#1, #2#2] +Arguments: hashpartitioning(o_orderkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [2]: [#3#1, #2#2] +Arguments: X + +(6) AQEShuffleRead +Input [2]: [#3#1, #2#2] +Arguments: coalesced + +(7) InputAdapter +Input [2]: [#3#1, #2#2] + +(8) NativeSort +Input [2]: [#3#1, #2#2] +Arguments: [o_orderkey#1 ASC NULLS FIRST], false + +(37) Scan parquet +Output [5]: [l_orderkey#4, l_shipdate#5, l_commitdate#6, l_receiptdate#7, l_shipmode#8] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] +ReadSchema: struct + +(10) InputAdapter +Input [5]: [l_orderkey#4, l_shipdate#5, l_commitdate#6, l_receiptdate#7, l_shipmode#8] +Arguments: [#9, #5, #6, #7, #8] + +(11) NativeFilter +Input [5]: [#9#4, #5#5, #6#6, #7#7, #8#8] +Condition : ((((((((isnotnull(l_commitdate#6) AND isnotnull(l_receiptdate#7)) AND isnotnull(l_shipdate#5)) AND l_shipmode#8 IN (MAIL,SHIP)) AND (l_commitdate#6 < l_receiptdate#7)) AND (l_shipdate#5 < l_commitdate#6)) AND (l_receiptdate#7 >= 1994-01-01)) AND (l_receiptdate#7 < 1995-01-01)) AND isnotnull(l_orderkey#4)) + +(12) NativeProject +Output [2]: [l_orderkey#4, l_shipmode#8] +Input [5]: [#9#4, #5#5, #6#6, #7#7, #8#8] + +(13) NativeShuffleExchange +Input [2]: [l_orderkey#4, l_shipmode#8] +Arguments: hashpartitioning(l_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [2]: [l_orderkey#4, l_shipmode#8] +Arguments: X + +(15) AQEShuffleRead +Input [2]: [l_orderkey#4, l_shipmode#8] +Arguments: coalesced + +(16) InputAdapter +Input [2]: [l_orderkey#4, l_shipmode#8] + +(17) NativeSort +Input [2]: [l_orderkey#4, l_shipmode#8] +Arguments: [l_orderkey#4 ASC NULLS FIRST], false + +(18) NativeSortMergeJoin +Left keys [1]: [o_orderkey#1] +Right keys [1]: [l_orderkey#4] +Join type: Inner +Join condition: None + +(19) NativeProject +Output [2]: [o_orderpriority#2, l_shipmode#8] +Input [4]: [#3#1, #2#2, l_orderkey#4, l_shipmode#8] + +(20) NativeProject +Output [3]: [l_shipmode#8 AS l_shipmode#8, CASE WHEN ((o_orderpriority#2 = 1-URGENT) OR (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END AS _c1#10, CASE WHEN (NOT (o_orderpriority#2 = 1-URGENT) AND NOT (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END AS _c2#11] +Input [2]: [o_orderpriority#2, l_shipmode#8] + +(21) NativeHashAggregate +Input [3]: [l_shipmode#8, _c1#10, _c2#11] +Keys [1]: [l_shipmode#8] +Functions [2]: [partial_sum(_c1#10), partial_sum(_c2#11)] +Aggregate Attributes [2]: [sum#12, sum#13] +Results [2]: [l_shipmode#8, #14#14] + +(22) NativeShuffleExchange +Input [2]: [l_shipmode#8, #14#14] +Arguments: hashpartitioning(l_shipmode#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(23) ShuffleQueryStage +Output [2]: [l_shipmode#8, #14#14] +Arguments: X + +(24) AQEShuffleRead +Input [2]: [l_shipmode#8, #14#14] +Arguments: coalesced + +(25) InputAdapter +Input [2]: [l_shipmode#8, #14#14] + +(26) NativeHashAggregate +Input [2]: [l_shipmode#8, #14#14] +Keys [1]: [l_shipmode#8] +Functions [2]: [sum(CASE WHEN ((o_orderpriority#2 = 1-URGENT) OR (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#2 = 1-URGENT) AND NOT (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#2 = 1-URGENT) OR (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#15, sum(CASE WHEN (NOT (o_orderpriority#2 = 1-URGENT) AND NOT (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#16] +Results [3]: [l_shipmode#8, sum(CASE WHEN ((o_orderpriority#2 = 1-URGENT) OR (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#15, sum(CASE WHEN (NOT (o_orderpriority#2 = 1-URGENT) AND NOT (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#16] + +(27) NativeProject +Output [3]: [l_shipmode#8, sum(CASE WHEN ((o_orderpriority#2 = 1-URGENT) OR (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#15 AS high_line_count#17, sum(CASE WHEN (NOT (o_orderpriority#2 = 1-URGENT) AND NOT (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#16 AS low_line_count#18] +Input [3]: [l_shipmode#8, sum(CASE WHEN ((o_orderpriority#2 = 1-URGENT) OR (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#15, sum(CASE WHEN (NOT (o_orderpriority#2 = 1-URGENT) AND NOT (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#16] + +(28) NativeShuffleExchange +Input [3]: [l_shipmode#8, high_line_count#17, low_line_count#18] +Arguments: rangepartitioning(l_shipmode#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(29) ShuffleQueryStage +Output [3]: [l_shipmode#8, high_line_count#17, low_line_count#18] +Arguments: X + +(30) AQEShuffleRead +Input [3]: [l_shipmode#8, high_line_count#17, low_line_count#18] +Arguments: coalesced + +(31) InputAdapter +Input [3]: [l_shipmode#8, high_line_count#17, low_line_count#18] + +(32) NativeSort +Input [3]: [l_shipmode#8, high_line_count#17, low_line_count#18] +Arguments: [l_shipmode#8 ASC NULLS FIRST], true + +(33) Scan parquet +Output [2]: [o_orderkey#1, o_orderpriority#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(34) Filter +Input [2]: [o_orderkey#1, o_orderpriority#2] +Condition : isnotnull(o_orderkey#1) + +(35) Exchange +Input [2]: [o_orderkey#1, o_orderpriority#2] +Arguments: hashpartitioning(o_orderkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(36) Sort +Input [2]: [o_orderkey#1, o_orderpriority#2] +Arguments: [o_orderkey#1 ASC NULLS FIRST], false, 0 + +(37) Scan parquet +Output [5]: [l_orderkey#4, l_shipdate#5, l_commitdate#6, l_receiptdate#7, l_shipmode#8] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] +ReadSchema: struct + +(38) Filter +Input [5]: [l_orderkey#4, l_shipdate#5, l_commitdate#6, l_receiptdate#7, l_shipmode#8] +Condition : ((((((((isnotnull(l_commitdate#6) AND isnotnull(l_receiptdate#7)) AND isnotnull(l_shipdate#5)) AND l_shipmode#8 IN (MAIL,SHIP)) AND (l_commitdate#6 < l_receiptdate#7)) AND (l_shipdate#5 < l_commitdate#6)) AND (l_receiptdate#7 >= 1994-01-01)) AND (l_receiptdate#7 < 1995-01-01)) AND isnotnull(l_orderkey#4)) + +(39) Project +Output [2]: [l_orderkey#4, l_shipmode#8] +Input [5]: [l_orderkey#4, l_shipdate#5, l_commitdate#6, l_receiptdate#7, l_shipmode#8] + +(40) Exchange +Input [2]: [l_orderkey#4, l_shipmode#8] +Arguments: hashpartitioning(l_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(41) Sort +Input [2]: [l_orderkey#4, l_shipmode#8] +Arguments: [l_orderkey#4 ASC NULLS FIRST], false, 0 + +(42) SortMergeJoin +Left keys [1]: [o_orderkey#1] +Right keys [1]: [l_orderkey#4] +Join type: Inner +Join condition: None + +(43) Project +Output [2]: [o_orderpriority#2, l_shipmode#8] +Input [4]: [o_orderkey#1, o_orderpriority#2, l_orderkey#4, l_shipmode#8] + +(44) HashAggregate +Input [2]: [o_orderpriority#2, l_shipmode#8] +Keys [1]: [l_shipmode#8] +Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#2 = 1-URGENT) OR (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#2 = 1-URGENT) AND NOT (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum#12, sum#13] +Results [3]: [l_shipmode#8, sum#19, sum#20] + +(45) Exchange +Input [3]: [l_shipmode#8, sum#19, sum#20] +Arguments: hashpartitioning(l_shipmode#8, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(46) HashAggregate +Input [3]: [l_shipmode#8, sum#19, sum#20] +Keys [1]: [l_shipmode#8] +Functions [2]: [sum(CASE WHEN ((o_orderpriority#2 = 1-URGENT) OR (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#2 = 1-URGENT) AND NOT (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)] +Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#2 = 1-URGENT) OR (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#15, sum(CASE WHEN (NOT (o_orderpriority#2 = 1-URGENT) AND NOT (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#16] +Results [3]: [l_shipmode#8, sum(CASE WHEN ((o_orderpriority#2 = 1-URGENT) OR (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#15 AS high_line_count#17, sum(CASE WHEN (NOT (o_orderpriority#2 = 1-URGENT) AND NOT (o_orderpriority#2 = 2-HIGH)) THEN 1 ELSE 0 END)#16 AS low_line_count#18] + +(47) Exchange +Input [3]: [l_shipmode#8, high_line_count#17, low_line_count#18] +Arguments: rangepartitioning(l_shipmode#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(48) Sort +Input [3]: [l_shipmode#8, high_line_count#17, low_line_count#18] +Arguments: [l_shipmode#8 ASC NULLS FIRST], true, 0 + +(49) AdaptiveSparkPlan +Output [3]: [l_shipmode#8, high_line_count#17, low_line_count#18] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q13.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q13.txt new file mode 100644 index 000000000..cc6beddf5 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q13.txt @@ -0,0 +1,294 @@ +== Physical Plan == +AdaptiveSparkPlan (51) ++- == Final Plan == + NativeSort (33) + +- InputAdapter (32) + +- AQEShuffleRead (31) + +- ShuffleQueryStage (30), Statistics(sizeInBytes=976.0 B, rowCount=33) + +- NativeShuffleExchange (29) + +- NativeProject (28) + +- NativeHashAggregate (27) + +- InputAdapter (26) + +- AQEShuffleRead (25) + +- ShuffleQueryStage (24), Statistics(sizeInBytes=992.0 B, rowCount=33) + +- NativeShuffleExchange (23) + +- NativeHashAggregate (22) + +- NativeProject (21) + +- NativeHashAggregate (20) + +- NativeHashAggregate (19) + +- NativeProject (18) + +- NativeSortMergeJoin LeftOuter (17) + :- NativeSort (7) + : +- InputAdapter (6) + : +- AQEShuffleRead (5) + : +- ShuffleQueryStage (4), Statistics(sizeInBytes=11.9 KiB, rowCount=1.50E+3) + : +- NativeShuffleExchange (3) + : +- InputAdapter (2) + : +- NativeParquetScan (1) + +- NativeSort (16) + +- InputAdapter (15) + +- AQEShuffleRead (14) + +- ShuffleQueryStage (13), Statistics(sizeInBytes=232.5 KiB, rowCount=1.48E+4) + +- NativeShuffleExchange (12) + +- NativeProject (11) + +- NativeFilter (10) + +- InputAdapter (9) + +- NativeParquetScan (8) ++- == Initial Plan == + Sort (50) + +- Exchange (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- HashAggregate (44) + +- Project (43) + +- SortMergeJoin LeftOuter (42) + :- Sort (36) + : +- Exchange (35) + : +- Scan parquet (34) + +- Sort (41) + +- Exchange (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) + + +(34) Scan parquet +Output [1]: [c_custkey#1] +Batched: true +Location: InMemoryFileIndex [file:/] +ReadSchema: struct + +(2) InputAdapter +Input [1]: [c_custkey#1] +Arguments: [#2] + +(3) NativeShuffleExchange +Input [1]: [#2#1] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(4) ShuffleQueryStage +Output [1]: [#2#1] +Arguments: X + +(5) AQEShuffleRead +Input [1]: [#2#1] +Arguments: coalesced + +(6) InputAdapter +Input [1]: [#2#1] + +(7) NativeSort +Input [1]: [#2#1] +Arguments: [c_custkey#1 ASC NULLS FIRST], false + +(37) Scan parquet +Output [3]: [o_orderkey#3, o_custkey#4, o_comment#5] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] +ReadSchema: struct + +(9) InputAdapter +Input [3]: [o_orderkey#3, o_custkey#4, o_comment#5] +Arguments: [#6, #7, #5] + +(10) NativeFilter +Input [3]: [#6#3, #7#4, #5#5] +Condition : ((isnotnull(o_comment#5) AND NOT o_comment#5 LIKE %special%requests%) AND isnotnull(o_custkey#4)) + +(11) NativeProject +Output [2]: [o_orderkey#3, o_custkey#4] +Input [3]: [#6#3, #7#4, #5#5] + +(12) NativeShuffleExchange +Input [2]: [o_orderkey#3, o_custkey#4] +Arguments: hashpartitioning(o_custkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(13) ShuffleQueryStage +Output [2]: [o_orderkey#3, o_custkey#4] +Arguments: X + +(14) AQEShuffleRead +Input [2]: [o_orderkey#3, o_custkey#4] +Arguments: coalesced + +(15) InputAdapter +Input [2]: [o_orderkey#3, o_custkey#4] + +(16) NativeSort +Input [2]: [o_orderkey#3, o_custkey#4] +Arguments: [o_custkey#4 ASC NULLS FIRST], false + +(17) NativeSortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#4] +Join type: LeftOuter +Join condition: None + +(18) NativeProject +Output [2]: [c_custkey#1, o_orderkey#3] +Input [3]: [#2#1, o_orderkey#3, o_custkey#4] + +(19) NativeHashAggregate +Input [2]: [c_custkey#1, o_orderkey#3] +Keys [1]: [c_custkey#1] +Functions [1]: [partial_count(o_orderkey#3)] +Aggregate Attributes [1]: [count#8] +Results [2]: [c_custkey#1, #9#9] + +(20) NativeHashAggregate +Input [2]: [c_custkey#1, #9#9] +Keys [1]: [c_custkey#1] +Functions [1]: [count(o_orderkey#3)] +Aggregate Attributes [1]: [count(o_orderkey#3)#10] +Results [2]: [c_custkey#1, count(o_orderkey#3)#10] + +(21) NativeProject +Output [1]: [count(o_orderkey#3)#10 AS c_count#11] +Input [2]: [c_custkey#1, count(o_orderkey#3)#10] + +(22) NativeHashAggregate +Input [1]: [c_count#11] +Keys [1]: [c_count#11] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#12] +Results [2]: [c_count#11, #9#9] + +(23) NativeShuffleExchange +Input [2]: [c_count#11, #9#9] +Arguments: hashpartitioning(c_count#11, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(24) ShuffleQueryStage +Output [2]: [c_count#11, #9#9] +Arguments: X + +(25) AQEShuffleRead +Input [2]: [c_count#11, #9#9] +Arguments: coalesced + +(26) InputAdapter +Input [2]: [c_count#11, #9#9] + +(27) NativeHashAggregate +Input [2]: [c_count#11, #9#9] +Keys [1]: [c_count#11] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#13] +Results [2]: [c_count#11, count(1)#13] + +(28) NativeProject +Output [2]: [c_count#11, count(1)#13 AS custdist#14] +Input [2]: [c_count#11, count(1)#13] + +(29) NativeShuffleExchange +Input [2]: [c_count#11, custdist#14] +Arguments: rangepartitioning(custdist#14 DESC NULLS LAST, c_count#11 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(30) ShuffleQueryStage +Output [2]: [c_count#11, custdist#14] +Arguments: X + +(31) AQEShuffleRead +Input [2]: [c_count#11, custdist#14] +Arguments: coalesced + +(32) InputAdapter +Input [2]: [c_count#11, custdist#14] + +(33) NativeSort +Input [2]: [c_count#11, custdist#14] +Arguments: [custdist#14 DESC NULLS LAST, c_count#11 DESC NULLS LAST], true + +(34) Scan parquet +Output [1]: [c_custkey#1] +Batched: true +Location: InMemoryFileIndex [file:/] +ReadSchema: struct + +(35) Exchange +Input [1]: [c_custkey#1] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(36) Sort +Input [1]: [c_custkey#1] +Arguments: [c_custkey#1 ASC NULLS FIRST], false, 0 + +(37) Scan parquet +Output [3]: [o_orderkey#3, o_custkey#4, o_comment#5] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] +ReadSchema: struct + +(38) Filter +Input [3]: [o_orderkey#3, o_custkey#4, o_comment#5] +Condition : ((isnotnull(o_comment#5) AND NOT o_comment#5 LIKE %special%requests%) AND isnotnull(o_custkey#4)) + +(39) Project +Output [2]: [o_orderkey#3, o_custkey#4] +Input [3]: [o_orderkey#3, o_custkey#4, o_comment#5] + +(40) Exchange +Input [2]: [o_orderkey#3, o_custkey#4] +Arguments: hashpartitioning(o_custkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(41) Sort +Input [2]: [o_orderkey#3, o_custkey#4] +Arguments: [o_custkey#4 ASC NULLS FIRST], false, 0 + +(42) SortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#4] +Join type: LeftOuter +Join condition: None + +(43) Project +Output [2]: [c_custkey#1, o_orderkey#3] +Input [3]: [c_custkey#1, o_orderkey#3, o_custkey#4] + +(44) HashAggregate +Input [2]: [c_custkey#1, o_orderkey#3] +Keys [1]: [c_custkey#1] +Functions [1]: [partial_count(o_orderkey#3)] +Aggregate Attributes [1]: [count#8] +Results [2]: [c_custkey#1, count#15] + +(45) HashAggregate +Input [2]: [c_custkey#1, count#15] +Keys [1]: [c_custkey#1] +Functions [1]: [count(o_orderkey#3)] +Aggregate Attributes [1]: [count(o_orderkey#3)#10] +Results [1]: [count(o_orderkey#3)#10 AS c_count#11] + +(46) HashAggregate +Input [1]: [c_count#11] +Keys [1]: [c_count#11] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#12] +Results [2]: [c_count#11, count#16] + +(47) Exchange +Input [2]: [c_count#11, count#16] +Arguments: hashpartitioning(c_count#11, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(48) HashAggregate +Input [2]: [c_count#11, count#16] +Keys [1]: [c_count#11] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#13] +Results [2]: [c_count#11, count(1)#13 AS custdist#14] + +(49) Exchange +Input [2]: [c_count#11, custdist#14] +Arguments: rangepartitioning(custdist#14 DESC NULLS LAST, c_count#11 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(50) Sort +Input [2]: [c_count#11, custdist#14] +Arguments: [custdist#14 DESC NULLS LAST, c_count#11 DESC NULLS LAST], true, 0 + +(51) AdaptiveSparkPlan +Output [2]: [c_count#11, custdist#14] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q14.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q14.txt new file mode 100644 index 000000000..6c815a811 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q14.txt @@ -0,0 +1,231 @@ +== Physical Plan == +AdaptiveSparkPlan (40) ++- == Final Plan == + NativeProject (25) + +- NativeHashAggregate (24) + +- ShuffleQueryStage (23), Statistics(sizeInBytes=360.0 B, rowCount=1) + +- NativeShuffleExchange (22) + +- NativeHashAggregate (21) + +- NativeProject (20) + +- NativeProject (19) + +- NativeSortMergeJoin Inner (18) + :- NativeSort (9) + : +- InputAdapter (8) + : +- AQEShuffleRead (7) + : +- ShuffleQueryStage (6), Statistics(sizeInBytes=28.9 KiB, rowCount=722) + : +- NativeShuffleExchange (5) + : +- NativeProject (4) + : +- NativeFilter (3) + : +- InputAdapter (2) + : +- NativeParquetScan (1) + +- NativeSort (17) + +- InputAdapter (16) + +- AQEShuffleRead (15) + +- ShuffleQueryStage (14), Statistics(sizeInBytes=68.0 KiB, rowCount=2.00E+3) + +- NativeShuffleExchange (13) + +- NativeFilter (12) + +- InputAdapter (11) + +- NativeParquetScan (10) ++- == Initial Plan == + HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- SortMergeJoin Inner (35) + :- Sort (30) + : +- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Sort (34) + +- Exchange (33) + +- Filter (32) + +- Scan parquet (31) + + +(26) Scan parquet +Output [4]: [l_partkey#1, l_extendedprice#2, l_discount#3, l_shipdate#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] +ReadSchema: struct + +(2) InputAdapter +Input [4]: [l_partkey#1, l_extendedprice#2, l_discount#3, l_shipdate#4] +Arguments: [#5, #2, #3, #4] + +(3) NativeFilter +Input [4]: [#5#1, #2#2, #3#3, #4#4] +Condition : (((isnotnull(l_shipdate#4) AND (l_shipdate#4 >= 1995-09-01)) AND (l_shipdate#4 < 1995-10-01)) AND isnotnull(l_partkey#1)) + +(4) NativeProject +Output [3]: [l_partkey#1, l_extendedprice#2, l_discount#3] +Input [4]: [#5#1, #2#2, #3#3, #4#4] + +(5) NativeShuffleExchange +Input [3]: [l_partkey#1, l_extendedprice#2, l_discount#3] +Arguments: hashpartitioning(l_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(6) ShuffleQueryStage +Output [3]: [l_partkey#1, l_extendedprice#2, l_discount#3] +Arguments: X + +(7) AQEShuffleRead +Input [3]: [l_partkey#1, l_extendedprice#2, l_discount#3] +Arguments: coalesced + +(8) InputAdapter +Input [3]: [l_partkey#1, l_extendedprice#2, l_discount#3] + +(9) NativeSort +Input [3]: [l_partkey#1, l_extendedprice#2, l_discount#3] +Arguments: [l_partkey#1 ASC NULLS FIRST], false + +(31) Scan parquet +Output [2]: [p_partkey#6, p_type#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_partkey)] +ReadSchema: struct + +(11) InputAdapter +Input [2]: [p_partkey#6, p_type#7] +Arguments: [#8, #7] + +(12) NativeFilter +Input [2]: [#8#6, #7#7] +Condition : isnotnull(p_partkey#6) + +(13) NativeShuffleExchange +Input [2]: [#8#6, #7#7] +Arguments: hashpartitioning(p_partkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [2]: [#8#6, #7#7] +Arguments: X + +(15) AQEShuffleRead +Input [2]: [#8#6, #7#7] +Arguments: coalesced + +(16) InputAdapter +Input [2]: [#8#6, #7#7] + +(17) NativeSort +Input [2]: [#8#6, #7#7] +Arguments: [p_partkey#6 ASC NULLS FIRST], false + +(18) NativeSortMergeJoin +Left keys [1]: [l_partkey#1] +Right keys [1]: [p_partkey#6] +Join type: Inner +Join condition: None + +(19) NativeProject +Output [3]: [l_extendedprice#2, l_discount#3, p_type#7] +Input [5]: [l_partkey#1, l_extendedprice#2, l_discount#3, #8#6, #7#7] + +(20) NativeProject +Output [2]: [CASE WHEN StartsWith(p_type#7, PROMO) THEN (l_extendedprice#2 * (1 - l_discount#3)) ELSE 0.0000 END AS _c0#9, (l_extendedprice#2 * (1 - l_discount#3)) AS _c1#10] +Input [3]: [l_extendedprice#2, l_discount#3, p_type#7] + +(21) NativeHashAggregate +Input [2]: [_c0#9, _c1#10] +Keys: [] +Functions [2]: [partial_sum(_c0#9), partial_sum(_c1#10)] +Aggregate Attributes [4]: [sum#11, isEmpty#12, sum#13, isEmpty#14] +Results [1]: [#15#15] + +(22) NativeShuffleExchange +Input [1]: [#15#15] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] + +(23) ShuffleQueryStage +Output [1]: [#15#15] +Arguments: X + +(24) NativeHashAggregate +Input [1]: [#15#15] +Keys: [] +Functions [2]: [sum(CASE WHEN StartsWith(p_type#7, PROMO) THEN (l_extendedprice#2 * (1 - l_discount#3)) ELSE 0.0000 END), sum((l_extendedprice#2 * (1 - l_discount#3)))] +Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#7, PROMO) THEN (l_extendedprice#2 * (1 - l_discount#3)) ELSE 0.0000 END)#16, sum((l_extendedprice#2 * (1 - l_discount#3)))#17] +Results [2]: [sum(CASE WHEN StartsWith(p_type#7, PROMO) THEN (l_extendedprice#2 * (1 - l_discount#3)) ELSE 0.0000 END)#16, sum((l_extendedprice#2 * (1 - l_discount#3)))#17] + +(25) NativeProject +Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#7, PROMO) THEN (l_extendedprice#2 * (1 - l_discount#3)) ELSE 0.0000 END)#16) / sum((l_extendedprice#2 * (1 - l_discount#3)))#17) AS promo_revenue#18] +Input [2]: [sum(CASE WHEN StartsWith(p_type#7, PROMO) THEN (l_extendedprice#2 * (1 - l_discount#3)) ELSE 0.0000 END)#16, sum((l_extendedprice#2 * (1 - l_discount#3)))#17] + +(26) Scan parquet +Output [4]: [l_partkey#1, l_extendedprice#2, l_discount#3, l_shipdate#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] +ReadSchema: struct + +(27) Filter +Input [4]: [l_partkey#1, l_extendedprice#2, l_discount#3, l_shipdate#4] +Condition : (((isnotnull(l_shipdate#4) AND (l_shipdate#4 >= 1995-09-01)) AND (l_shipdate#4 < 1995-10-01)) AND isnotnull(l_partkey#1)) + +(28) Project +Output [3]: [l_partkey#1, l_extendedprice#2, l_discount#3] +Input [4]: [l_partkey#1, l_extendedprice#2, l_discount#3, l_shipdate#4] + +(29) Exchange +Input [3]: [l_partkey#1, l_extendedprice#2, l_discount#3] +Arguments: hashpartitioning(l_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(30) Sort +Input [3]: [l_partkey#1, l_extendedprice#2, l_discount#3] +Arguments: [l_partkey#1 ASC NULLS FIRST], false, 0 + +(31) Scan parquet +Output [2]: [p_partkey#6, p_type#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_partkey)] +ReadSchema: struct + +(32) Filter +Input [2]: [p_partkey#6, p_type#7] +Condition : isnotnull(p_partkey#6) + +(33) Exchange +Input [2]: [p_partkey#6, p_type#7] +Arguments: hashpartitioning(p_partkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(34) Sort +Input [2]: [p_partkey#6, p_type#7] +Arguments: [p_partkey#6 ASC NULLS FIRST], false, 0 + +(35) SortMergeJoin +Left keys [1]: [l_partkey#1] +Right keys [1]: [p_partkey#6] +Join type: Inner +Join condition: None + +(36) Project +Output [3]: [l_extendedprice#2, l_discount#3, p_type#7] +Input [5]: [l_partkey#1, l_extendedprice#2, l_discount#3, p_partkey#6, p_type#7] + +(37) HashAggregate +Input [3]: [l_extendedprice#2, l_discount#3, p_type#7] +Keys: [] +Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#7, PROMO) THEN (l_extendedprice#2 * (1 - l_discount#3)) ELSE 0.0000 END), partial_sum((l_extendedprice#2 * (1 - l_discount#3)))] +Aggregate Attributes [4]: [sum#11, isEmpty#12, sum#13, isEmpty#14] +Results [4]: [sum#19, isEmpty#20, sum#21, isEmpty#22] + +(38) Exchange +Input [4]: [sum#19, isEmpty#20, sum#21, isEmpty#22] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] + +(39) HashAggregate +Input [4]: [sum#19, isEmpty#20, sum#21, isEmpty#22] +Keys: [] +Functions [2]: [sum(CASE WHEN StartsWith(p_type#7, PROMO) THEN (l_extendedprice#2 * (1 - l_discount#3)) ELSE 0.0000 END), sum((l_extendedprice#2 * (1 - l_discount#3)))] +Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#7, PROMO) THEN (l_extendedprice#2 * (1 - l_discount#3)) ELSE 0.0000 END)#16, sum((l_extendedprice#2 * (1 - l_discount#3)))#17] +Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#7, PROMO) THEN (l_extendedprice#2 * (1 - l_discount#3)) ELSE 0.0000 END)#16) / sum((l_extendedprice#2 * (1 - l_discount#3)))#17) AS promo_revenue#18] + +(40) AdaptiveSparkPlan +Output [1]: [promo_revenue#18] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q15.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q15.txt new file mode 100644 index 000000000..ff4774394 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q15.txt @@ -0,0 +1,427 @@ +== Physical Plan == +AdaptiveSparkPlan (45) ++- == Final Plan == + InputAdapter (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(sizeInBytes=1256.0 B, rowCount=1) + +- NativeShuffleExchange (25) + +- NativeProject (24) + +- NativeSortMergeJoin Inner (23) + :- NativeSort (8) + : +- InputAdapter (7) + : +- AQEShuffleRead (6) + : +- ShuffleQueryStage (5), Statistics(sizeInBytes=9.6 KiB, rowCount=100) + : +- NativeShuffleExchange (4) + : +- NativeFilter (3) + : +- InputAdapter (2) + : +- NativeParquetScan (1) + +- NativeSort (22) + +- NativeFilter (21) + +- NativeProject (20) + +- NativeHashAggregate (19) + +- InputAdapter (18) + +- AQEShuffleRead (17) + +- ShuffleQueryStage (16), Statistics(sizeInBytes=3.7 KiB, rowCount=100) + +- NativeShuffleExchange (15) + +- NativeHashAggregate (14) + +- NativeProject (13) + +- NativeProject (12) + +- NativeFilter (11) + +- InputAdapter (10) + +- NativeParquetScan (9) ++- == Initial Plan == + Sort (44) + +- Exchange (43) + +- Project (42) + +- SortMergeJoin Inner (41) + :- Sort (32) + : +- Exchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Sort (40) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) + + +(29) Scan parquet +Output [4]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey)] +ReadSchema: struct + +(2) InputAdapter +Input [4]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4] +Arguments: [#5, #2, #3, #4] + +(3) NativeFilter +Input [4]: [#5#1, #2#2, #3#3, #4#4] +Condition : isnotnull(s_suppkey#1) + +(4) NativeShuffleExchange +Input [4]: [#5#1, #2#2, #3#3, #4#4] +Arguments: hashpartitioning(s_suppkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [4]: [#5#1, #2#2, #3#3, #4#4] +Arguments: X + +(6) AQEShuffleRead +Input [4]: [#5#1, #2#2, #3#3, #4#4] +Arguments: coalesced + +(7) InputAdapter +Input [4]: [#5#1, #2#2, #3#3, #4#4] + +(8) NativeSort +Input [4]: [#5#1, #2#2, #3#3, #4#4] +Arguments: [s_suppkey#1 ASC NULLS FIRST], false + +(33) Scan parquet +Output [4]: [l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] +ReadSchema: struct + +(10) InputAdapter +Input [4]: [l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] +Arguments: [#10, #7, #8, #9] + +(11) NativeFilter +Input [4]: [#10#6, #7#7, #8#8, #9#9] +Condition : (((isnotnull(l_shipdate#9) AND (l_shipdate#9 >= 1996-01-01)) AND (l_shipdate#9 < 1996-04-01)) AND isnotnull(l_suppkey#6)) + +(12) NativeProject +Output [3]: [l_suppkey#6, l_extendedprice#7, l_discount#8] +Input [4]: [#10#6, #7#7, #8#8, #9#9] + +(13) NativeProject +Output [2]: [l_suppkey#6 AS l_suppkey#6, (l_extendedprice#7 * (1 - l_discount#8)) AS _c1#11] +Input [3]: [l_suppkey#6, l_extendedprice#7, l_discount#8] + +(14) NativeHashAggregate +Input [2]: [l_suppkey#6, _c1#11] +Keys [1]: [l_suppkey#6] +Functions [1]: [partial_sum(_c1#11)] +Aggregate Attributes [2]: [sum#12, isEmpty#13] +Results [2]: [l_suppkey#6, #14#14] + +(15) NativeShuffleExchange +Input [2]: [l_suppkey#6, #14#14] +Arguments: hashpartitioning(l_suppkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(16) ShuffleQueryStage +Output [2]: [l_suppkey#6, #14#14] +Arguments: X + +(17) AQEShuffleRead +Input [2]: [l_suppkey#6, #14#14] +Arguments: coalesced + +(18) InputAdapter +Input [2]: [l_suppkey#6, #14#14] + +(19) NativeHashAggregate +Input [2]: [l_suppkey#6, #14#14] +Keys [1]: [l_suppkey#6] +Functions [1]: [sum((l_extendedprice#7 * (1 - l_discount#8)))] +Aggregate Attributes [1]: [sum((l_extendedprice#7 * (1 - l_discount#8)))#15] +Results [2]: [l_suppkey#6, sum((l_extendedprice#7 * (1 - l_discount#8)))#15] + +(20) NativeProject +Output [2]: [l_suppkey#6 AS supplier_no#16, sum((l_extendedprice#7 * (1 - l_discount#8)))#15 AS total_revenue#17] +Input [2]: [l_suppkey#6, sum((l_extendedprice#7 * (1 - l_discount#8)))#15] + +(21) NativeFilter +Input [2]: [supplier_no#16, total_revenue#17] +Condition : (isnotnull(total_revenue#17) AND (total_revenue#17 = Subquery subquery#18, [id=#19])) + +(22) NativeSort +Input [2]: [supplier_no#16, total_revenue#17] +Arguments: [supplier_no#16 ASC NULLS FIRST], false + +(23) NativeSortMergeJoin +Left keys [1]: [s_suppkey#1] +Right keys [1]: [supplier_no#16] +Join type: Inner +Join condition: None + +(24) NativeProject +Output [5]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4, total_revenue#17] +Input [6]: [#5#1, #2#2, #3#3, #4#4, supplier_no#16, total_revenue#17] + +(25) NativeShuffleExchange +Input [5]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4, total_revenue#17] +Arguments: rangepartitioning(s_suppkey#1 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(26) ShuffleQueryStage +Output [5]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4, total_revenue#17] +Arguments: X + +(27) AQEShuffleRead +Input [5]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4, total_revenue#17] +Arguments: local + +(28) InputAdapter +Input [5]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4, total_revenue#17] + +(29) Scan parquet +Output [4]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey)] +ReadSchema: struct + +(30) Filter +Input [4]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4] +Condition : isnotnull(s_suppkey#1) + +(31) Exchange +Input [4]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4] +Arguments: hashpartitioning(s_suppkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(32) Sort +Input [4]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4] +Arguments: [s_suppkey#1 ASC NULLS FIRST], false, 0 + +(33) Scan parquet +Output [4]: [l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] +ReadSchema: struct + +(34) Filter +Input [4]: [l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] +Condition : (((isnotnull(l_shipdate#9) AND (l_shipdate#9 >= 1996-01-01)) AND (l_shipdate#9 < 1996-04-01)) AND isnotnull(l_suppkey#6)) + +(35) Project +Output [3]: [l_suppkey#6, l_extendedprice#7, l_discount#8] +Input [4]: [l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] + +(36) HashAggregate +Input [3]: [l_suppkey#6, l_extendedprice#7, l_discount#8] +Keys [1]: [l_suppkey#6] +Functions [1]: [partial_sum((l_extendedprice#7 * (1 - l_discount#8)))] +Aggregate Attributes [2]: [sum#12, isEmpty#13] +Results [3]: [l_suppkey#6, sum#20, isEmpty#21] + +(37) Exchange +Input [3]: [l_suppkey#6, sum#20, isEmpty#21] +Arguments: hashpartitioning(l_suppkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(38) HashAggregate +Input [3]: [l_suppkey#6, sum#20, isEmpty#21] +Keys [1]: [l_suppkey#6] +Functions [1]: [sum((l_extendedprice#7 * (1 - l_discount#8)))] +Aggregate Attributes [1]: [sum((l_extendedprice#7 * (1 - l_discount#8)))#15] +Results [2]: [l_suppkey#6 AS supplier_no#16, sum((l_extendedprice#7 * (1 - l_discount#8)))#15 AS total_revenue#17] + +(39) Filter +Input [2]: [supplier_no#16, total_revenue#17] +Condition : (isnotnull(total_revenue#17) AND (total_revenue#17 = Subquery subquery#18, [id=#19])) + +(40) Sort +Input [2]: [supplier_no#16, total_revenue#17] +Arguments: [supplier_no#16 ASC NULLS FIRST], false, 0 + +(41) SortMergeJoin +Left keys [1]: [s_suppkey#1] +Right keys [1]: [supplier_no#16] +Join type: Inner +Join condition: None + +(42) Project +Output [5]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4, total_revenue#17] +Input [6]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4, supplier_no#16, total_revenue#17] + +(43) Exchange +Input [5]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4, total_revenue#17] +Arguments: rangepartitioning(s_suppkey#1 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(44) Sort +Input [5]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4, total_revenue#17] +Arguments: [s_suppkey#1 ASC NULLS FIRST], true, 0 + +(45) AdaptiveSparkPlan +Output [5]: [s_suppkey#1, s_name#2, s_address#3, s_phone#4, total_revenue#17] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#18, [id=#19] +AdaptiveSparkPlan (72) ++- == Final Plan == + NativeProject (62) + +- NativeHashAggregate (61) + +- ShuffleQueryStage (60), Statistics(sizeInBytes=360.0 B, rowCount=1) + +- NativeShuffleExchange (59) + +- NativeHashAggregate (58) + +- NativeProject (57) + +- NativeHashAggregate (56) + +- InputAdapter (55) + +- AQEShuffleRead (54) + +- ShuffleQueryStage (53), Statistics(sizeInBytes=3.7 KiB, rowCount=100) + +- NativeShuffleExchange (52) + +- NativeHashAggregate (51) + +- NativeProject (50) + +- NativeProject (49) + +- NativeFilter (48) + +- InputAdapter (47) + +- NativeParquetScan (46) ++- == Initial Plan == + HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- HashAggregate (68) + +- Exchange (67) + +- HashAggregate (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) + + +(63) Scan parquet +Output [4]: [l_suppkey#22, l_extendedprice#23, l_discount#24, l_shipdate#25] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] +ReadSchema: struct + +(47) InputAdapter +Input [4]: [l_suppkey#22, l_extendedprice#23, l_discount#24, l_shipdate#25] +Arguments: [#26, #23, #24, #25] + +(48) NativeFilter +Input [4]: [#26#22, #23#23, #24#24, #25#25] +Condition : ((isnotnull(l_shipdate#25) AND (l_shipdate#25 >= 1996-01-01)) AND (l_shipdate#25 < 1996-04-01)) + +(49) NativeProject +Output [3]: [l_suppkey#22, l_extendedprice#23, l_discount#24] +Input [4]: [#26#22, #23#23, #24#24, #25#25] + +(50) NativeProject +Output [2]: [l_suppkey#22 AS l_suppkey#22, (l_extendedprice#23 * (1 - l_discount#24)) AS _c1#27] +Input [3]: [l_suppkey#22, l_extendedprice#23, l_discount#24] + +(51) NativeHashAggregate +Input [2]: [l_suppkey#22, _c1#27] +Keys [1]: [l_suppkey#22] +Functions [1]: [partial_sum(_c1#27)] +Aggregate Attributes [2]: [sum#28, isEmpty#29] +Results [2]: [l_suppkey#22, #14#14] + +(52) NativeShuffleExchange +Input [2]: [l_suppkey#22, #14#14] +Arguments: hashpartitioning(l_suppkey#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(53) ShuffleQueryStage +Output [2]: [l_suppkey#22, #14#14] +Arguments: X + +(54) AQEShuffleRead +Input [2]: [l_suppkey#22, #14#14] +Arguments: coalesced + +(55) InputAdapter +Input [2]: [l_suppkey#22, #14#14] + +(56) NativeHashAggregate +Input [2]: [l_suppkey#22, #14#14] +Keys [1]: [l_suppkey#22] +Functions [1]: [sum((l_extendedprice#23 * (1 - l_discount#24)))] +Aggregate Attributes [1]: [sum((l_extendedprice#23 * (1 - l_discount#24)))#15] +Results [2]: [l_suppkey#22, sum((l_extendedprice#23 * (1 - l_discount#24)))#15] + +(57) NativeProject +Output [1]: [sum((l_extendedprice#23 * (1 - l_discount#24)))#15 AS total_revenue#30] +Input [2]: [l_suppkey#22, sum((l_extendedprice#23 * (1 - l_discount#24)))#15] + +(58) NativeHashAggregate +Input [1]: [total_revenue#30] +Keys: [] +Functions [1]: [partial_max(total_revenue#30)] +Aggregate Attributes [1]: [max#31] +Results [1]: [#14#14] + +(59) NativeShuffleExchange +Input [1]: [#14#14] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] + +(60) ShuffleQueryStage +Output [1]: [#14#14] +Arguments: X + +(61) NativeHashAggregate +Input [1]: [#14#14] +Keys: [] +Functions [1]: [max(total_revenue#30)] +Aggregate Attributes [1]: [max(total_revenue#30)#32] +Results [1]: [max(total_revenue#30)#32] + +(62) NativeProject +Output [1]: [max(total_revenue#30)#32 AS max(total_revenue)#33] +Input [1]: [max(total_revenue#30)#32] + +(63) Scan parquet +Output [4]: [l_suppkey#22, l_extendedprice#23, l_discount#24, l_shipdate#25] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] +ReadSchema: struct + +(64) Filter +Input [4]: [l_suppkey#22, l_extendedprice#23, l_discount#24, l_shipdate#25] +Condition : ((isnotnull(l_shipdate#25) AND (l_shipdate#25 >= 1996-01-01)) AND (l_shipdate#25 < 1996-04-01)) + +(65) Project +Output [3]: [l_suppkey#22, l_extendedprice#23, l_discount#24] +Input [4]: [l_suppkey#22, l_extendedprice#23, l_discount#24, l_shipdate#25] + +(66) HashAggregate +Input [3]: [l_suppkey#22, l_extendedprice#23, l_discount#24] +Keys [1]: [l_suppkey#22] +Functions [1]: [partial_sum((l_extendedprice#23 * (1 - l_discount#24)))] +Aggregate Attributes [2]: [sum#28, isEmpty#29] +Results [3]: [l_suppkey#22, sum#34, isEmpty#35] + +(67) Exchange +Input [3]: [l_suppkey#22, sum#34, isEmpty#35] +Arguments: hashpartitioning(l_suppkey#22, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(68) HashAggregate +Input [3]: [l_suppkey#22, sum#34, isEmpty#35] +Keys [1]: [l_suppkey#22] +Functions [1]: [sum((l_extendedprice#23 * (1 - l_discount#24)))] +Aggregate Attributes [1]: [sum((l_extendedprice#23 * (1 - l_discount#24)))#15] +Results [1]: [sum((l_extendedprice#23 * (1 - l_discount#24)))#15 AS total_revenue#30] + +(69) HashAggregate +Input [1]: [total_revenue#30] +Keys: [] +Functions [1]: [partial_max(total_revenue#30)] +Aggregate Attributes [1]: [max#31] +Results [1]: [max#36] + +(70) Exchange +Input [1]: [max#36] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] + +(71) HashAggregate +Input [1]: [max#36] +Keys: [] +Functions [1]: [max(total_revenue#30)] +Aggregate Attributes [1]: [max(total_revenue#30)#32] +Results [1]: [max(total_revenue#30)#32 AS max(total_revenue)#33] + +(72) AdaptiveSparkPlan +Output [1]: [max(total_revenue)#33] +Arguments: isFinalPlan=true + + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q16.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q16.txt new file mode 100644 index 000000000..ff25b5ff2 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q16.txt @@ -0,0 +1,350 @@ +== Physical Plan == +AdaptiveSparkPlan (61) ++- == Final Plan == + NativeSort (37) + +- InputAdapter (36) + +- AQEShuffleRead (35) + +- ShuffleQueryStage (34), Statistics(sizeInBytes=14.9 KiB, rowCount=296) + +- NativeShuffleExchange (33) + +- NativeProject (32) + +- NativeHashAggregate (31) + +- InputAdapter (30) + +- AQEShuffleRead (29) + +- ShuffleQueryStage (28), Statistics(sizeInBytes=14.3 KiB, rowCount=296) + +- NativeShuffleExchange (27) + +- NativeHashAggregate (26) + +- NativeProject (25) + +- NativeHashAggregate (24) + +- InputAdapter (23) + +- AQEShuffleRead (22) + +- ShuffleQueryStage (21), Statistics(sizeInBytes=62.4 KiB, rowCount=1.19E+3) + +- NativeShuffleExchange (20) + +- NativeHashAggregate (19) + +- NativeProject (18) + +- NativeSortMergeJoin Inner (17) + :- NativeSort (8) + : +- InputAdapter (7) + : +- AQEShuffleRead (6) + : +- ShuffleQueryStage (5), Statistics(sizeInBytes=125.4 KiB, rowCount=8.00E+3) + : +- NativeShuffleExchange (4) + : +- NativeFilter (3) + : +- InputAdapter (2) + : +- NativeParquetScan (1) + +- NativeSort (16) + +- InputAdapter (15) + +- AQEShuffleRead (14) + +- ShuffleQueryStage (13), Statistics(sizeInBytes=14.9 KiB, rowCount=299) + +- NativeShuffleExchange (12) + +- NativeFilter (11) + +- InputAdapter (10) + +- NativeParquetScan (9) ++- == Initial Plan == + Sort (60) + +- Exchange (59) + +- HashAggregate (58) + +- Exchange (57) + +- HashAggregate (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- SortMergeJoin Inner (51) + :- Sort (46) + : +- Exchange (45) + : +- BroadcastHashJoin LeftAnti BuildRight (44) + : :- Filter (39) + : : +- Scan parquet (38) + : +- BroadcastExchange (43) + : +- Project (42) + : +- Filter (41) + : +- Scan parquet (40) + +- Sort (50) + +- Exchange (49) + +- Filter (48) + +- Scan parquet (47) + + +(unknown) Scan parquet +Output [2]: [ps_partkey#1, ps_suppkey#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_partkey)] +ReadSchema: struct + +(2) InputAdapter +Input [2]: [ps_partkey#1, ps_suppkey#2] +Arguments: [#3, #4] + +(3) NativeFilter +Input [2]: [#3#1, #4#2] +Condition : isnotnull(ps_partkey#1) + +(4) NativeShuffleExchange +Input [2]: [#3#1, #4#2] +Arguments: hashpartitioning(ps_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [2]: [#3#1, #4#2] +Arguments: X + +(6) AQEShuffleRead +Input [2]: [#3#1, #4#2] +Arguments: coalesced + +(7) InputAdapter +Input [2]: [#3#1, #4#2] + +(8) NativeSort +Input [2]: [#3#1, #4#2] +Arguments: [ps_partkey#1 ASC NULLS FIRST], false + +(47) Scan parquet +Output [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#9)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +ReadSchema: struct + +(10) InputAdapter +Input [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] +Arguments: [#10, #6, #7, #8] + +(11) NativeFilter +Input [4]: [#10#5, #6#6, #7#7, #8#8] +Condition : (((((isnotnull(p_brand#6) AND isnotnull(p_type#7)) AND NOT (p_brand#6 = Brand#9)) AND NOT StartsWith(p_type#7, MEDIUM POLISHED)) AND p_size#8 IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#5)) + +(12) NativeShuffleExchange +Input [4]: [#10#5, #6#6, #7#7, #8#8] +Arguments: hashpartitioning(p_partkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(13) ShuffleQueryStage +Output [4]: [#10#5, #6#6, #7#7, #8#8] +Arguments: X + +(14) AQEShuffleRead +Input [4]: [#10#5, #6#6, #7#7, #8#8] +Arguments: coalesced + +(15) InputAdapter +Input [4]: [#10#5, #6#6, #7#7, #8#8] + +(16) NativeSort +Input [4]: [#10#5, #6#6, #7#7, #8#8] +Arguments: [p_partkey#5 ASC NULLS FIRST], false + +(17) NativeSortMergeJoin +Left keys [1]: [ps_partkey#1] +Right keys [1]: [p_partkey#5] +Join type: Inner +Join condition: None + +(18) NativeProject +Output [4]: [ps_suppkey#2, p_brand#6, p_type#7, p_size#8] +Input [6]: [#3#1, #4#2, #10#5, #6#6, #7#7, #8#8] + +(19) NativeHashAggregate +Input [4]: [ps_suppkey#2, p_brand#6, p_type#7, p_size#8] +Keys [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Functions: [] +Aggregate Attributes: [] +Results [5]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2, #11#11] + +(20) NativeShuffleExchange +Input [5]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2, #11#11] +Arguments: hashpartitioning(p_brand#6, p_type#7, p_size#8, ps_suppkey#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(21) ShuffleQueryStage +Output [5]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2, #11#11] +Arguments: X + +(22) AQEShuffleRead +Input [5]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2, #11#11] +Arguments: coalesced + +(23) InputAdapter +Input [5]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2, #11#11] + +(24) NativeHashAggregate +Input [5]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2, #11#11] +Keys [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Functions: [] +Aggregate Attributes: [] +Results [5]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2, #11#11] + +(25) NativeProject +Output [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Input [5]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2, #11#11] + +(26) NativeHashAggregate +Input [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Keys [3]: [p_brand#6, p_type#7, p_size#8] +Functions [1]: [partial_count(distinct ps_suppkey#2)] +Aggregate Attributes [1]: [count(ps_suppkey#2)#12] +Results [4]: [p_brand#6, p_type#7, p_size#8, #11#11] + +(27) NativeShuffleExchange +Input [4]: [p_brand#6, p_type#7, p_size#8, #11#11] +Arguments: hashpartitioning(p_brand#6, p_type#7, p_size#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(28) ShuffleQueryStage +Output [4]: [p_brand#6, p_type#7, p_size#8, #11#11] +Arguments: X + +(29) AQEShuffleRead +Input [4]: [p_brand#6, p_type#7, p_size#8, #11#11] +Arguments: coalesced + +(30) InputAdapter +Input [4]: [p_brand#6, p_type#7, p_size#8, #11#11] + +(31) NativeHashAggregate +Input [4]: [p_brand#6, p_type#7, p_size#8, #11#11] +Keys [3]: [p_brand#6, p_type#7, p_size#8] +Functions [1]: [count(distinct ps_suppkey#2)] +Aggregate Attributes [1]: [count(ps_suppkey#2)#12] +Results [4]: [p_brand#6, p_type#7, p_size#8, count(ps_suppkey#2)#12] + +(32) NativeProject +Output [4]: [p_brand#6, p_type#7, p_size#8, count(ps_suppkey#2)#12 AS supplier_cnt#13] +Input [4]: [p_brand#6, p_type#7, p_size#8, count(ps_suppkey#2)#12] + +(33) NativeShuffleExchange +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: rangepartitioning(supplier_cnt#13 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(34) ShuffleQueryStage +Output [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: X + +(35) AQEShuffleRead +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: coalesced + +(36) InputAdapter +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] + +(37) NativeSort +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: [supplier_cnt#13 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST], true + +(38) Scan parquet +Output [2]: [ps_partkey#1, ps_suppkey#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_partkey)] +ReadSchema: struct + +(39) Filter +Input [2]: [ps_partkey#1, ps_suppkey#2] +Condition : isnotnull(ps_partkey#1) + +(40) Scan parquet +Output [2]: [s_suppkey#14, s_comment#15] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_comment)] +ReadSchema: struct + +(41) Filter +Input [2]: [s_suppkey#14, s_comment#15] +Condition : (isnotnull(s_comment#15) AND s_comment#15 LIKE %Customer%Complaints%) + +(42) Project +Output [1]: [s_suppkey#14] +Input [2]: [s_suppkey#14, s_comment#15] + +(43) BroadcastExchange +Input [1]: [s_suppkey#14] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=6] + +(44) BroadcastHashJoin +Left keys [1]: [ps_suppkey#2] +Right keys [1]: [s_suppkey#14] +Join type: LeftAnti +Join condition: None + +(45) Exchange +Input [2]: [ps_partkey#1, ps_suppkey#2] +Arguments: hashpartitioning(ps_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(46) Sort +Input [2]: [ps_partkey#1, ps_suppkey#2] +Arguments: [ps_partkey#1 ASC NULLS FIRST], false, 0 + +(47) Scan parquet +Output [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#9)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +ReadSchema: struct + +(48) Filter +Input [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] +Condition : (((((isnotnull(p_brand#6) AND isnotnull(p_type#7)) AND NOT (p_brand#6 = Brand#9)) AND NOT StartsWith(p_type#7, MEDIUM POLISHED)) AND p_size#8 IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#5)) + +(49) Exchange +Input [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] +Arguments: hashpartitioning(p_partkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(50) Sort +Input [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] +Arguments: [p_partkey#5 ASC NULLS FIRST], false, 0 + +(51) SortMergeJoin +Left keys [1]: [ps_partkey#1] +Right keys [1]: [p_partkey#5] +Join type: Inner +Join condition: None + +(52) Project +Output [4]: [ps_suppkey#2, p_brand#6, p_type#7, p_size#8] +Input [6]: [ps_partkey#1, ps_suppkey#2, p_partkey#5, p_brand#6, p_type#7, p_size#8] + +(53) HashAggregate +Input [4]: [ps_suppkey#2, p_brand#6, p_type#7, p_size#8] +Keys [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] + +(54) Exchange +Input [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Arguments: hashpartitioning(p_brand#6, p_type#7, p_size#8, ps_suppkey#2, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(55) HashAggregate +Input [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Keys [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Functions: [] +Aggregate Attributes: [] +Results [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] + +(56) HashAggregate +Input [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Keys [3]: [p_brand#6, p_type#7, p_size#8] +Functions [1]: [partial_count(distinct ps_suppkey#2)] +Aggregate Attributes [1]: [count(ps_suppkey#2)#12] +Results [4]: [p_brand#6, p_type#7, p_size#8, count#16] + +(57) Exchange +Input [4]: [p_brand#6, p_type#7, p_size#8, count#16] +Arguments: hashpartitioning(p_brand#6, p_type#7, p_size#8, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(58) HashAggregate +Input [4]: [p_brand#6, p_type#7, p_size#8, count#16] +Keys [3]: [p_brand#6, p_type#7, p_size#8] +Functions [1]: [count(distinct ps_suppkey#2)] +Aggregate Attributes [1]: [count(ps_suppkey#2)#12] +Results [4]: [p_brand#6, p_type#7, p_size#8, count(ps_suppkey#2)#12 AS supplier_cnt#13] + +(59) Exchange +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: rangepartitioning(supplier_cnt#13 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(60) Sort +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: [supplier_cnt#13 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST], true, 0 + +(61) AdaptiveSparkPlan +Output [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q17.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q17.txt new file mode 100644 index 000000000..a348bcaeb --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q17.txt @@ -0,0 +1,357 @@ +== Physical Plan == +AdaptiveSparkPlan (61) ++- == Final Plan == + * HashAggregate (37) + +- ShuffleQueryStage (36), Statistics(sizeInBytes=40.0 B, rowCount=1) + +- Exchange (35) + +- * HashAggregate (34) + +- * Project (33) + +- * SortMergeJoin Inner (32) + :- NativeProject (19) + : +- NativeSortMergeJoin Inner (18) + : :- NativeSort (8) + : : +- InputAdapter (7) + : : +- AQEShuffleRead (6) + : : +- ShuffleQueryStage (5), Statistics(sizeInBytes=2.3 MiB, rowCount=6.02E+4) + : : +- NativeShuffleExchange (4) + : : +- NativeFilter (3) + : : +- InputAdapter (2) + : : +- NativeParquetScan (1) + : +- NativeSort (17) + : +- InputAdapter (16) + : +- AQEShuffleRead (15) + : +- ShuffleQueryStage (14), Statistics(sizeInBytes=232.0 B, rowCount=3) + : +- NativeShuffleExchange (13) + : +- NativeProject (12) + : +- NativeFilter (11) + : +- InputAdapter (10) + : +- NativeParquetScan (9) + +- NativeSort (31) + +- NativeFilter (30) + +- NativeProject (29) + +- NativeHashAggregate (28) + +- InputAdapter (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(sizeInBytes=87.9 KiB, rowCount=2.00E+3) + +- NativeShuffleExchange (24) + +- NativeHashAggregate (23) + +- NativeFilter (22) + +- InputAdapter (21) + +- NativeParquetScan (20) ++- == Initial Plan == + HashAggregate (60) + +- Exchange (59) + +- HashAggregate (58) + +- Project (57) + +- SortMergeJoin Inner (56) + :- Project (48) + : +- SortMergeJoin Inner (47) + : :- Sort (41) + : : +- Exchange (40) + : : +- Filter (39) + : : +- Scan parquet (38) + : +- Sort (46) + : +- Exchange (45) + : +- Project (44) + : +- Filter (43) + : +- Scan parquet (42) + +- Sort (55) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) + + +(38) Scan parquet +Output [3]: [l_partkey#1, l_quantity#2, l_extendedprice#3] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] +ReadSchema: struct + +(2) InputAdapter +Input [3]: [l_partkey#1, l_quantity#2, l_extendedprice#3] +Arguments: [#4, #2, #3] + +(3) NativeFilter +Input [3]: [#4#1, #2#2, #3#3] +Condition : (isnotnull(l_partkey#1) AND isnotnull(l_quantity#2)) + +(4) NativeShuffleExchange +Input [3]: [#4#1, #2#2, #3#3] +Arguments: hashpartitioning(l_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [3]: [#4#1, #2#2, #3#3] +Arguments: X + +(6) AQEShuffleRead +Input [3]: [#4#1, #2#2, #3#3] +Arguments: coalesced + +(7) InputAdapter +Input [3]: [#4#1, #2#2, #3#3] + +(8) NativeSort +Input [3]: [#4#1, #2#2, #3#3] +Arguments: [l_partkey#1 ASC NULLS FIRST], false + +(42) Scan parquet +Output [3]: [p_partkey#5, p_brand#6, p_container#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#8), EqualTo(p_container,WRAP BAG), IsNotNull(p_partkey)] +ReadSchema: struct + +(10) InputAdapter +Input [3]: [p_partkey#5, p_brand#6, p_container#7] +Arguments: [#9, #6, #7] + +(11) NativeFilter +Input [3]: [#9#5, #6#6, #7#7] +Condition : ((((isnotnull(p_brand#6) AND isnotnull(p_container#7)) AND (p_brand#6 = Brand#8)) AND (p_container#7 = WRAP BAG)) AND isnotnull(p_partkey#5)) + +(12) NativeProject +Output [1]: [p_partkey#5] +Input [3]: [#9#5, #6#6, #7#7] + +(13) NativeShuffleExchange +Input [1]: [p_partkey#5] +Arguments: hashpartitioning(p_partkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [1]: [p_partkey#5] +Arguments: X + +(15) AQEShuffleRead +Input [1]: [p_partkey#5] +Arguments: coalesced + +(16) InputAdapter +Input [1]: [p_partkey#5] + +(17) NativeSort +Input [1]: [p_partkey#5] +Arguments: [p_partkey#5 ASC NULLS FIRST], false + +(18) NativeSortMergeJoin +Left keys [1]: [l_partkey#1] +Right keys [1]: [p_partkey#5] +Join type: Inner +Join condition: None + +(19) NativeProject +Output [3]: [l_quantity#2, l_extendedprice#3, p_partkey#5] +Input [4]: [#4#1, #2#2, #3#3, p_partkey#5] + +(49) Scan parquet +Output [2]: [l_partkey#10, l_quantity#11] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_partkey)] +ReadSchema: struct + +(21) InputAdapter +Input [2]: [l_partkey#10, l_quantity#11] +Arguments: [#12, #11] + +(22) NativeFilter +Input [2]: [#12#10, #11#11] +Condition : isnotnull(l_partkey#10) + +(23) NativeHashAggregate +Input [2]: [#12#10, #11#11] +Keys [1]: [l_partkey#10] +Functions [1]: [partial_avg(l_quantity#11)] +Aggregate Attributes [2]: [sum#13, count#14] +Results [2]: [l_partkey#10, #15#15] + +(24) NativeShuffleExchange +Input [2]: [l_partkey#10, #15#15] +Arguments: hashpartitioning(l_partkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(25) ShuffleQueryStage +Output [2]: [l_partkey#10, #15#15] +Arguments: X + +(26) AQEShuffleRead +Input [2]: [l_partkey#10, #15#15] +Arguments: coalesced + +(27) InputAdapter +Input [2]: [l_partkey#10, #15#15] + +(28) NativeHashAggregate +Input [2]: [l_partkey#10, #15#15] +Keys [1]: [l_partkey#10] +Functions [1]: [avg(l_quantity#11)] +Aggregate Attributes [1]: [avg(l_quantity#11)#16] +Results [2]: [l_partkey#10, avg(l_quantity#11)#16] + +(29) NativeProject +Output [2]: [(0.2 * avg(l_quantity#11)#16) AS (0.2 * avg(l_quantity))#17, l_partkey#10] +Input [2]: [l_partkey#10, avg(l_quantity#11)#16] + +(30) NativeFilter +Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#10] +Condition : isnotnull((0.2 * avg(l_quantity))#17) + +(31) NativeSort +Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#10] +Arguments: [l_partkey#10 ASC NULLS FIRST], false + +(32) SortMergeJoin [codegen id : 1] +Left keys [1]: [p_partkey#5] +Right keys [1]: [l_partkey#10] +Join type: Inner +Join condition: (cast(l_quantity#2 as decimal(18,7)) < (0.2 * avg(l_quantity))#17) + +(33) Project [codegen id : 1] +Output [1]: [l_extendedprice#3] +Input [5]: [l_quantity#2, l_extendedprice#3, p_partkey#5, (0.2 * avg(l_quantity))#17, l_partkey#10] + +(34) HashAggregate [codegen id : 1] +Input [1]: [l_extendedprice#3] +Keys: [] +Functions [1]: [partial_sum(l_extendedprice#3)] +Aggregate Attributes [2]: [sum#18, isEmpty#19] +Results [2]: [sum#20, isEmpty#21] + +(35) Exchange +Input [2]: [sum#20, isEmpty#21] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] + +(36) ShuffleQueryStage +Output [2]: [sum#20, isEmpty#21] +Arguments: X + +(37) HashAggregate [codegen id : 2] +Input [2]: [sum#20, isEmpty#21] +Keys: [] +Functions [1]: [sum(l_extendedprice#3)] +Aggregate Attributes [1]: [sum(l_extendedprice#3)#22] +Results [1]: [(sum(l_extendedprice#3)#22 / 7.0) AS avg_yearly#23] + +(38) Scan parquet +Output [3]: [l_partkey#1, l_quantity#2, l_extendedprice#3] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] +ReadSchema: struct + +(39) Filter +Input [3]: [l_partkey#1, l_quantity#2, l_extendedprice#3] +Condition : (isnotnull(l_partkey#1) AND isnotnull(l_quantity#2)) + +(40) Exchange +Input [3]: [l_partkey#1, l_quantity#2, l_extendedprice#3] +Arguments: hashpartitioning(l_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(41) Sort +Input [3]: [l_partkey#1, l_quantity#2, l_extendedprice#3] +Arguments: [l_partkey#1 ASC NULLS FIRST], false, 0 + +(42) Scan parquet +Output [3]: [p_partkey#5, p_brand#6, p_container#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#8), EqualTo(p_container,WRAP BAG), IsNotNull(p_partkey)] +ReadSchema: struct + +(43) Filter +Input [3]: [p_partkey#5, p_brand#6, p_container#7] +Condition : ((((isnotnull(p_brand#6) AND isnotnull(p_container#7)) AND (p_brand#6 = Brand#8)) AND (p_container#7 = WRAP BAG)) AND isnotnull(p_partkey#5)) + +(44) Project +Output [1]: [p_partkey#5] +Input [3]: [p_partkey#5, p_brand#6, p_container#7] + +(45) Exchange +Input [1]: [p_partkey#5] +Arguments: hashpartitioning(p_partkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(46) Sort +Input [1]: [p_partkey#5] +Arguments: [p_partkey#5 ASC NULLS FIRST], false, 0 + +(47) SortMergeJoin +Left keys [1]: [l_partkey#1] +Right keys [1]: [p_partkey#5] +Join type: Inner +Join condition: None + +(48) Project +Output [3]: [l_quantity#2, l_extendedprice#3, p_partkey#5] +Input [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, p_partkey#5] + +(49) Scan parquet +Output [2]: [l_partkey#10, l_quantity#11] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_partkey)] +ReadSchema: struct + +(50) Filter +Input [2]: [l_partkey#10, l_quantity#11] +Condition : isnotnull(l_partkey#10) + +(51) HashAggregate +Input [2]: [l_partkey#10, l_quantity#11] +Keys [1]: [l_partkey#10] +Functions [1]: [partial_avg(l_quantity#11)] +Aggregate Attributes [2]: [sum#13, count#14] +Results [3]: [l_partkey#10, sum#24, count#25] + +(52) Exchange +Input [3]: [l_partkey#10, sum#24, count#25] +Arguments: hashpartitioning(l_partkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(53) HashAggregate +Input [3]: [l_partkey#10, sum#24, count#25] +Keys [1]: [l_partkey#10] +Functions [1]: [avg(l_quantity#11)] +Aggregate Attributes [1]: [avg(l_quantity#11)#16] +Results [2]: [(0.2 * avg(l_quantity#11)#16) AS (0.2 * avg(l_quantity))#17, l_partkey#10] + +(54) Filter +Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#10] +Condition : isnotnull((0.2 * avg(l_quantity))#17) + +(55) Sort +Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#10] +Arguments: [l_partkey#10 ASC NULLS FIRST], false, 0 + +(56) SortMergeJoin +Left keys [1]: [p_partkey#5] +Right keys [1]: [l_partkey#10] +Join type: Inner +Join condition: (cast(l_quantity#2 as decimal(18,7)) < (0.2 * avg(l_quantity))#17) + +(57) Project +Output [1]: [l_extendedprice#3] +Input [5]: [l_quantity#2, l_extendedprice#3, p_partkey#5, (0.2 * avg(l_quantity))#17, l_partkey#10] + +(58) HashAggregate +Input [1]: [l_extendedprice#3] +Keys: [] +Functions [1]: [partial_sum(l_extendedprice#3)] +Aggregate Attributes [2]: [sum#18, isEmpty#19] +Results [2]: [sum#20, isEmpty#21] + +(59) Exchange +Input [2]: [sum#20, isEmpty#21] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] + +(60) HashAggregate +Input [2]: [sum#20, isEmpty#21] +Keys: [] +Functions [1]: [sum(l_extendedprice#3)] +Aggregate Attributes [1]: [sum(l_extendedprice#3)#22] +Results [1]: [(sum(l_extendedprice#3)#22 / 7.0) AS avg_yearly#23] + +(61) AdaptiveSparkPlan +Output [1]: [avg_yearly#23] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q18.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q18.txt new file mode 100644 index 000000000..0797df4b5 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q18.txt @@ -0,0 +1,600 @@ +== Physical Plan == +AdaptiveSparkPlan (106) ++- == Final Plan == + NativeTakeOrdered (66) + +- NativeProject (65) + +- NativeHashAggregate (64) + +- NativeHashAggregate (63) + +- NativeProject (62) + +- NativeSortMergeJoin Inner (61) + :- NativeSort (41) + : +- InputAdapter (40) + : +- AQEShuffleRead (39) + : +- ShuffleQueryStage (38), Statistics(sizeInBytes=1116.0 B, rowCount=2) + : +- NativeShuffleExchange (37) + : +- NativeProject (36) + : +- NativeSortMergeJoin Inner (35) + : :- NativeSort (8) + : : +- InputAdapter (7) + : : +- AQEShuffleRead (6) + : : +- ShuffleQueryStage (5), Statistics(sizeInBytes=50.2 KiB, rowCount=1.50E+3) + : : +- NativeShuffleExchange (4) + : : +- NativeFilter (3) + : : +- InputAdapter (2) + : : +- NativeParquetScan (1) + : +- NativeSort (34) + : +- InputAdapter (33) + : +- AQEShuffleRead (32) + : +- ShuffleQueryStage (31), Statistics(sizeInBytes=832.0 B, rowCount=2) + : +- NativeShuffleExchange (30) + : +- NativeSortMergeJoin LeftSemi (29) + : :- NativeSort (16) + : : +- InputAdapter (15) + : : +- AQEShuffleRead (14) + : : +- ShuffleQueryStage (13), Statistics(sizeInBytes=528.8 KiB, rowCount=1.50E+4) + : : +- NativeShuffleExchange (12) + : : +- NativeFilter (11) + : : +- InputAdapter (10) + : : +- NativeParquetScan (9) + : +- NativeSort (28) + : +- NativeProject (27) + : +- NativeFilter (26) + : +- NativeProject (25) + : +- NativeHashAggregate (24) + : +- InputAdapter (23) + : +- AQEShuffleRead (22) + : +- ShuffleQueryStage (21), Statistics(sizeInBytes=560.7 KiB, rowCount=1.50E+4) + : +- NativeShuffleExchange (20) + : +- NativeHashAggregate (19) + : +- InputAdapter (18) + : +- NativeParquetScan (17) + +- NativeSortMergeJoin LeftSemi (60) + :- NativeSort (49) + : +- InputAdapter (48) + : +- AQEShuffleRead (47) + : +- ShuffleQueryStage (46), Statistics(sizeInBytes=1413.0 KiB, rowCount=6.02E+4) + : +- NativeShuffleExchange (45) + : +- NativeFilter (44) + : +- InputAdapter (43) + : +- NativeParquetScan (42) + +- NativeSort (59) + +- NativeProject (58) + +- NativeFilter (57) + +- NativeProject (56) + +- NativeHashAggregate (55) + +- InputAdapter (54) + +- InputAdapter (53) + +- AQEShuffleRead (52) + +- ShuffleQueryStage (51), Statistics(sizeInBytes=560.7 KiB, rowCount=1.50E+4) + +- ReusedExchange (50) ++- == Initial Plan == + TakeOrderedAndProject (105) + +- HashAggregate (104) + +- HashAggregate (103) + +- Project (102) + +- SortMergeJoin Inner (101) + :- Sort (88) + : +- Exchange (87) + : +- Project (86) + : +- SortMergeJoin Inner (85) + : :- Sort (70) + : : +- Exchange (69) + : : +- Filter (68) + : : +- Scan parquet (67) + : +- Sort (84) + : +- Exchange (83) + : +- SortMergeJoin LeftSemi (82) + : :- Sort (74) + : : +- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Sort (81) + : +- Project (80) + : +- Filter (79) + : +- HashAggregate (78) + : +- Exchange (77) + : +- HashAggregate (76) + : +- Scan parquet (75) + +- SortMergeJoin LeftSemi (100) + :- Sort (92) + : +- Exchange (91) + : +- Filter (90) + : +- Scan parquet (89) + +- Sort (99) + +- Project (98) + +- Filter (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Scan parquet (93) + + +(67) Scan parquet +Output [2]: [c_custkey#1, c_name#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_custkey)] +ReadSchema: struct + +(2) InputAdapter +Input [2]: [c_custkey#1, c_name#2] +Arguments: [#3, #2] + +(3) NativeFilter +Input [2]: [#3#1, #2#2] +Condition : isnotnull(c_custkey#1) + +(4) NativeShuffleExchange +Input [2]: [#3#1, #2#2] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [2]: [#3#1, #2#2] +Arguments: X + +(6) AQEShuffleRead +Input [2]: [#3#1, #2#2] +Arguments: coalesced + +(7) InputAdapter +Input [2]: [#3#1, #2#2] + +(8) NativeSort +Input [2]: [#3#1, #2#2] +Arguments: [c_custkey#1 ASC NULLS FIRST], false + +(71) Scan parquet +Output [4]: [o_orderkey#4, o_custkey#5, o_totalprice#6, o_orderdate#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(10) InputAdapter +Input [4]: [o_orderkey#4, o_custkey#5, o_totalprice#6, o_orderdate#7] +Arguments: [#8, #9, #6, #7] + +(11) NativeFilter +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Condition : (isnotnull(o_custkey#5) AND isnotnull(o_orderkey#4)) + +(12) NativeShuffleExchange +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: hashpartitioning(o_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(13) ShuffleQueryStage +Output [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: X + +(14) AQEShuffleRead +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: coalesced + +(15) InputAdapter +Input [4]: [#8#4, #9#5, #6#6, #7#7] + +(16) NativeSort +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: [o_orderkey#4 ASC NULLS FIRST], false + +(75) Scan parquet +Output [2]: [l_orderkey#10, l_quantity#11] +Batched: true +Location: InMemoryFileIndex [file:/] +ReadSchema: struct + +(18) InputAdapter +Input [2]: [l_orderkey#10, l_quantity#11] +Arguments: [#12, #11] + +(19) NativeHashAggregate +Input [2]: [#12#10, #11#11] +Keys [1]: [l_orderkey#10] +Functions [1]: [partial_sum(l_quantity#11)] +Aggregate Attributes [2]: [sum#13, isEmpty#14] +Results [2]: [l_orderkey#10, #15#15] + +(20) NativeShuffleExchange +Input [2]: [l_orderkey#10, #15#15] +Arguments: hashpartitioning(l_orderkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(21) ShuffleQueryStage +Output [2]: [l_orderkey#10, #15#15] +Arguments: X + +(22) AQEShuffleRead +Input [2]: [l_orderkey#10, #15#15] +Arguments: coalesced + +(23) InputAdapter +Input [2]: [l_orderkey#10, #15#15] + +(24) NativeHashAggregate +Input [2]: [l_orderkey#10, #15#15] +Keys [1]: [l_orderkey#10] +Functions [1]: [sum(l_quantity#11)] +Aggregate Attributes [1]: [sum(l_quantity#11)#16] +Results [2]: [l_orderkey#10, sum(l_quantity#11)#16] + +(25) NativeProject +Output [2]: [l_orderkey#10, sum(l_quantity#11)#16 AS sum(l_quantity#17)#18] +Input [2]: [l_orderkey#10, sum(l_quantity#11)#16] + +(26) NativeFilter +Input [2]: [l_orderkey#10, sum(l_quantity#17)#18] +Condition : (isnotnull(sum(l_quantity#17)#18) AND (sum(l_quantity#17)#18 > 300.00)) + +(27) NativeProject +Output [1]: [l_orderkey#10] +Input [2]: [l_orderkey#10, sum(l_quantity#17)#18] + +(28) NativeSort +Input [1]: [l_orderkey#10] +Arguments: [l_orderkey#10 ASC NULLS FIRST], false + +(29) NativeSortMergeJoin +Left keys [1]: [o_orderkey#4] +Right keys [1]: [l_orderkey#10] +Join type: LeftSemi +Join condition: None + +(30) NativeShuffleExchange +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: hashpartitioning(o_custkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(31) ShuffleQueryStage +Output [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: X + +(32) AQEShuffleRead +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: coalesced + +(33) InputAdapter +Input [4]: [#8#4, #9#5, #6#6, #7#7] + +(34) NativeSort +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: [o_custkey#5 ASC NULLS FIRST], false + +(35) NativeSortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#5] +Join type: Inner +Join condition: None + +(36) NativeProject +Output [5]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7] +Input [6]: [#3#1, #2#2, #8#4, #9#5, #6#6, #7#7] + +(37) NativeShuffleExchange +Input [5]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7] +Arguments: hashpartitioning(o_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(38) ShuffleQueryStage +Output [5]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7] +Arguments: X + +(39) AQEShuffleRead +Input [5]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7] +Arguments: coalesced + +(40) InputAdapter +Input [5]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7] + +(41) NativeSort +Input [5]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7] +Arguments: [o_orderkey#4 ASC NULLS FIRST], false + +(89) Scan parquet +Output [2]: [l_orderkey#19, l_quantity#17] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_orderkey)] +ReadSchema: struct + +(43) InputAdapter +Input [2]: [l_orderkey#19, l_quantity#17] +Arguments: [#20, #17] + +(44) NativeFilter +Input [2]: [#20#19, #17#17] +Condition : isnotnull(l_orderkey#19) + +(45) NativeShuffleExchange +Input [2]: [#20#19, #17#17] +Arguments: hashpartitioning(l_orderkey#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(46) ShuffleQueryStage +Output [2]: [#20#19, #17#17] +Arguments: X + +(47) AQEShuffleRead +Input [2]: [#20#19, #17#17] +Arguments: coalesced + +(48) InputAdapter +Input [2]: [#20#19, #17#17] + +(49) NativeSort +Input [2]: [#20#19, #17#17] +Arguments: [l_orderkey#19 ASC NULLS FIRST], false + +(50) ReusedExchange [Reuses operator id: 20] +Output [3]: [l_orderkey#10, sum#21, isEmpty#22] + +(51) ShuffleQueryStage +Output [3]: [l_orderkey#10, sum#21, isEmpty#22] +Arguments: X + +(52) AQEShuffleRead +Input [3]: [l_orderkey#10, sum#21, isEmpty#22] +Arguments: coalesced + +(53) InputAdapter +Input [3]: [l_orderkey#10, sum#21, isEmpty#22] +Arguments: [#12, #21, #22] + +(54) InputAdapter +Input [3]: [#12#10, #21#21, #22#22] + +(55) NativeHashAggregate +Input [3]: [#12#10, #21#21, #22#22] +Keys [1]: [l_orderkey#10] +Functions [1]: [sum(l_quantity#11)] +Aggregate Attributes [1]: [sum(l_quantity#11)#16] +Results [2]: [l_orderkey#10, sum(l_quantity#11)#16] + +(56) NativeProject +Output [2]: [l_orderkey#10, sum(l_quantity#11)#16 AS sum(l_quantity#17)#18] +Input [2]: [l_orderkey#10, sum(l_quantity#11)#16] + +(57) NativeFilter +Input [2]: [l_orderkey#10, sum(l_quantity#17)#18] +Condition : (isnotnull(sum(l_quantity#17)#18) AND (sum(l_quantity#17)#18 > 300.00)) + +(58) NativeProject +Output [1]: [l_orderkey#10] +Input [2]: [l_orderkey#10, sum(l_quantity#17)#18] + +(59) NativeSort +Input [1]: [l_orderkey#10] +Arguments: [l_orderkey#10 ASC NULLS FIRST], false + +(60) NativeSortMergeJoin +Left keys [1]: [l_orderkey#19] +Right keys [1]: [l_orderkey#10] +Join type: LeftSemi +Join condition: None + +(61) NativeSortMergeJoin +Left keys [1]: [o_orderkey#4] +Right keys [1]: [l_orderkey#19] +Join type: Inner +Join condition: None + +(62) NativeProject +Output [6]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7, l_quantity#17] +Input [7]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7, #20#19, #17#17] + +(63) NativeHashAggregate +Input [6]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7, l_quantity#17] +Keys [5]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6] +Functions [1]: [partial_sum(l_quantity#17)] +Aggregate Attributes [2]: [sum#23, isEmpty#24] +Results [6]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, #15#15] + +(64) NativeHashAggregate +Input [6]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, #15#15] +Keys [5]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6] +Functions [1]: [sum(l_quantity#17)] +Aggregate Attributes [1]: [sum(l_quantity#17)#25] +Results [6]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, sum(l_quantity#17)#25] + +(65) NativeProject +Output [6]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, sum(l_quantity#17)#25 AS sum(l_quantity)#26] +Input [6]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, sum(l_quantity#17)#25] + +(66) NativeTakeOrdered +Input [6]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, sum(l_quantity)#26] +Arguments: X, [o_totalprice#6 DESC NULLS LAST, o_orderdate#7 ASC NULLS FIRST] + +(67) Scan parquet +Output [2]: [c_custkey#1, c_name#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_custkey)] +ReadSchema: struct + +(68) Filter +Input [2]: [c_custkey#1, c_name#2] +Condition : isnotnull(c_custkey#1) + +(69) Exchange +Input [2]: [c_custkey#1, c_name#2] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(70) Sort +Input [2]: [c_custkey#1, c_name#2] +Arguments: [c_custkey#1 ASC NULLS FIRST], false, 0 + +(71) Scan parquet +Output [4]: [o_orderkey#4, o_custkey#5, o_totalprice#6, o_orderdate#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(72) Filter +Input [4]: [o_orderkey#4, o_custkey#5, o_totalprice#6, o_orderdate#7] +Condition : (isnotnull(o_custkey#5) AND isnotnull(o_orderkey#4)) + +(73) Exchange +Input [4]: [o_orderkey#4, o_custkey#5, o_totalprice#6, o_orderdate#7] +Arguments: hashpartitioning(o_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(74) Sort +Input [4]: [o_orderkey#4, o_custkey#5, o_totalprice#6, o_orderdate#7] +Arguments: [o_orderkey#4 ASC NULLS FIRST], false, 0 + +(75) Scan parquet +Output [2]: [l_orderkey#10, l_quantity#11] +Batched: true +Location: InMemoryFileIndex [file:/] +ReadSchema: struct + +(76) HashAggregate +Input [2]: [l_orderkey#10, l_quantity#11] +Keys [1]: [l_orderkey#10] +Functions [1]: [partial_sum(l_quantity#11)] +Aggregate Attributes [2]: [sum#13, isEmpty#14] +Results [3]: [l_orderkey#10, sum#21, isEmpty#22] + +(77) Exchange +Input [3]: [l_orderkey#10, sum#21, isEmpty#22] +Arguments: hashpartitioning(l_orderkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(78) HashAggregate +Input [3]: [l_orderkey#10, sum#21, isEmpty#22] +Keys [1]: [l_orderkey#10] +Functions [1]: [sum(l_quantity#11)] +Aggregate Attributes [1]: [sum(l_quantity#11)#16] +Results [2]: [l_orderkey#10, sum(l_quantity#11)#16 AS sum(l_quantity#17)#18] + +(79) Filter +Input [2]: [l_orderkey#10, sum(l_quantity#17)#18] +Condition : (isnotnull(sum(l_quantity#17)#18) AND (sum(l_quantity#17)#18 > 300.00)) + +(80) Project +Output [1]: [l_orderkey#10] +Input [2]: [l_orderkey#10, sum(l_quantity#17)#18] + +(81) Sort +Input [1]: [l_orderkey#10] +Arguments: [l_orderkey#10 ASC NULLS FIRST], false, 0 + +(82) SortMergeJoin +Left keys [1]: [o_orderkey#4] +Right keys [1]: [l_orderkey#10] +Join type: LeftSemi +Join condition: None + +(83) Exchange +Input [4]: [o_orderkey#4, o_custkey#5, o_totalprice#6, o_orderdate#7] +Arguments: hashpartitioning(o_custkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(84) Sort +Input [4]: [o_orderkey#4, o_custkey#5, o_totalprice#6, o_orderdate#7] +Arguments: [o_custkey#5 ASC NULLS FIRST], false, 0 + +(85) SortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#5] +Join type: Inner +Join condition: None + +(86) Project +Output [5]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7] +Input [6]: [c_custkey#1, c_name#2, o_orderkey#4, o_custkey#5, o_totalprice#6, o_orderdate#7] + +(87) Exchange +Input [5]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7] +Arguments: hashpartitioning(o_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(88) Sort +Input [5]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7] +Arguments: [o_orderkey#4 ASC NULLS FIRST], false, 0 + +(89) Scan parquet +Output [2]: [l_orderkey#19, l_quantity#17] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_orderkey)] +ReadSchema: struct + +(90) Filter +Input [2]: [l_orderkey#19, l_quantity#17] +Condition : isnotnull(l_orderkey#19) + +(91) Exchange +Input [2]: [l_orderkey#19, l_quantity#17] +Arguments: hashpartitioning(l_orderkey#19, 5), ENSURE_REQUIREMENTS, [plan_id=12] + +(92) Sort +Input [2]: [l_orderkey#19, l_quantity#17] +Arguments: [l_orderkey#19 ASC NULLS FIRST], false, 0 + +(93) Scan parquet +Output [2]: [l_orderkey#10, l_quantity#11] +Batched: true +Location: InMemoryFileIndex [file:/] +ReadSchema: struct + +(94) HashAggregate +Input [2]: [l_orderkey#10, l_quantity#11] +Keys [1]: [l_orderkey#10] +Functions [1]: [partial_sum(l_quantity#11)] +Aggregate Attributes [2]: [sum#13, isEmpty#14] +Results [3]: [l_orderkey#10, sum#21, isEmpty#22] + +(95) Exchange +Input [3]: [l_orderkey#10, sum#21, isEmpty#22] +Arguments: hashpartitioning(l_orderkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=13] + +(96) HashAggregate +Input [3]: [l_orderkey#10, sum#21, isEmpty#22] +Keys [1]: [l_orderkey#10] +Functions [1]: [sum(l_quantity#11)] +Aggregate Attributes [1]: [sum(l_quantity#11)#16] +Results [2]: [l_orderkey#10, sum(l_quantity#11)#16 AS sum(l_quantity#17)#18] + +(97) Filter +Input [2]: [l_orderkey#10, sum(l_quantity#17)#18] +Condition : (isnotnull(sum(l_quantity#17)#18) AND (sum(l_quantity#17)#18 > 300.00)) + +(98) Project +Output [1]: [l_orderkey#10] +Input [2]: [l_orderkey#10, sum(l_quantity#17)#18] + +(99) Sort +Input [1]: [l_orderkey#10] +Arguments: [l_orderkey#10 ASC NULLS FIRST], false, 0 + +(100) SortMergeJoin +Left keys [1]: [l_orderkey#19] +Right keys [1]: [l_orderkey#10] +Join type: LeftSemi +Join condition: None + +(101) SortMergeJoin +Left keys [1]: [o_orderkey#4] +Right keys [1]: [l_orderkey#19] +Join type: Inner +Join condition: None + +(102) Project +Output [6]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7, l_quantity#17] +Input [7]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7, l_orderkey#19, l_quantity#17] + +(103) HashAggregate +Input [6]: [c_custkey#1, c_name#2, o_orderkey#4, o_totalprice#6, o_orderdate#7, l_quantity#17] +Keys [5]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6] +Functions [1]: [partial_sum(l_quantity#17)] +Aggregate Attributes [2]: [sum#23, isEmpty#24] +Results [7]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, sum#27, isEmpty#28] + +(104) HashAggregate +Input [7]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, sum#27, isEmpty#28] +Keys [5]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6] +Functions [1]: [sum(l_quantity#17)] +Aggregate Attributes [1]: [sum(l_quantity#17)#25] +Results [6]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, sum(l_quantity#17)#25 AS sum(l_quantity)#26] + +(105) TakeOrderedAndProject +Input [6]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, sum(l_quantity)#26] +Arguments: X, [o_totalprice#6 DESC NULLS LAST, o_orderdate#7 ASC NULLS FIRST], [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, sum(l_quantity)#26] + +(106) AdaptiveSparkPlan +Output [6]: [c_name#2, c_custkey#1, o_orderkey#4, o_orderdate#7, o_totalprice#6, sum(l_quantity)#26] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q19.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q19.txt new file mode 100644 index 000000000..d2aff8350 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q19.txt @@ -0,0 +1,221 @@ +== Physical Plan == +AdaptiveSparkPlan (38) ++- == Final Plan == + * HashAggregate (23) + +- ShuffleQueryStage (22), Statistics(sizeInBytes=40.0 B, rowCount=1) + +- Exchange (21) + +- * HashAggregate (20) + +- * Project (19) + +- * SortMergeJoin Inner (18) + :- NativeSort (9) + : +- InputAdapter (8) + : +- AQEShuffleRead (7) + : +- ShuffleQueryStage (6), Statistics(sizeInBytes=66.6 KiB, rowCount=1.20E+3) + : +- NativeShuffleExchange (5) + : +- NativeProject (4) + : +- NativeFilter (3) + : +- InputAdapter (2) + : +- NativeParquetScan (1) + +- NativeSort (17) + +- InputAdapter (16) + +- AQEShuffleRead (15) + +- ShuffleQueryStage (14), Statistics(sizeInBytes=990.0 B, rowCount=5) + +- NativeShuffleExchange (13) + +- NativeFilter (12) + +- InputAdapter (11) + +- NativeParquetScan (10) ++- == Initial Plan == + HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- SortMergeJoin Inner (33) + :- Sort (28) + : +- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Sort (32) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) + + +(24) Scan parquet +Output [6]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, l_shipinstruct#5, l_shipmode#6] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] +ReadSchema: struct + +(2) InputAdapter +Input [6]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, l_shipinstruct#5, l_shipmode#6] +Arguments: [#7, #2, #3, #4, #5, #6] + +(3) NativeFilter +Input [6]: [#7#1, #2#2, #3#3, #4#4, #5#5, #6#6] +Condition : ((((isnotnull(l_shipinstruct#5) AND l_shipmode#6 IN (AIR,AIR REG)) AND (l_shipinstruct#5 = DELIVER IN PERSON)) AND isnotnull(l_partkey#1)) AND ((((l_quantity#2 >= 1.00) AND (l_quantity#2 <= 11.00)) OR ((l_quantity#2 >= 10.00) AND (l_quantity#2 <= 20.00))) OR ((l_quantity#2 >= 20.00) AND (l_quantity#2 <= 30.00)))) + +(4) NativeProject +Output [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4] +Input [6]: [#7#1, #2#2, #3#3, #4#4, #5#5, #6#6] + +(5) NativeShuffleExchange +Input [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4] +Arguments: hashpartitioning(l_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(6) ShuffleQueryStage +Output [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4] +Arguments: X + +(7) AQEShuffleRead +Input [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4] +Arguments: coalesced + +(8) InputAdapter +Input [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4] + +(9) NativeSort +Input [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4] +Arguments: [l_partkey#1 ASC NULLS FIRST], false + +(29) Scan parquet +Output [4]: [p_partkey#8, p_brand#9, p_size#10, p_container#11] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#12),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#13),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#14),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +ReadSchema: struct + +(11) InputAdapter +Input [4]: [p_partkey#8, p_brand#9, p_size#10, p_container#11] +Arguments: [#15, #9, #10, #11] + +(12) NativeFilter +Input [4]: [#15#8, #9#9, #10#10, #11#11] +Condition : (((isnotnull(p_size#10) AND (p_size#10 >= 1)) AND isnotnull(p_partkey#8)) AND (((((p_brand#9 = Brand#12) AND p_container#11 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#10 <= 5)) OR (((p_brand#9 = Brand#13) AND p_container#11 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#10 <= 10))) OR (((p_brand#9 = Brand#14) AND p_container#11 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#10 <= 15)))) + +(13) NativeShuffleExchange +Input [4]: [#15#8, #9#9, #10#10, #11#11] +Arguments: hashpartitioning(p_partkey#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [4]: [#15#8, #9#9, #10#10, #11#11] +Arguments: X + +(15) AQEShuffleRead +Input [4]: [#15#8, #9#9, #10#10, #11#11] +Arguments: coalesced + +(16) InputAdapter +Input [4]: [#15#8, #9#9, #10#10, #11#11] + +(17) NativeSort +Input [4]: [#15#8, #9#9, #10#10, #11#11] +Arguments: [p_partkey#8 ASC NULLS FIRST], false + +(18) SortMergeJoin [codegen id : 1] +Left keys [1]: [l_partkey#1] +Right keys [1]: [p_partkey#8] +Join type: Inner +Join condition: (((((((p_brand#9 = Brand#12) AND p_container#11 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#2 >= 1.00)) AND (l_quantity#2 <= 11.00)) AND (p_size#10 <= 5)) OR (((((p_brand#9 = Brand#13) AND p_container#11 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#2 >= 10.00)) AND (l_quantity#2 <= 20.00)) AND (p_size#10 <= 10))) OR (((((p_brand#9 = Brand#14) AND p_container#11 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#2 >= 20.00)) AND (l_quantity#2 <= 30.00)) AND (p_size#10 <= 15))) + +(19) Project [codegen id : 1] +Output [2]: [l_extendedprice#3, l_discount#4] +Input [8]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, #15#8, #9#9, #10#10, #11#11] + +(20) HashAggregate [codegen id : 1] +Input [2]: [l_extendedprice#3, l_discount#4] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#3 * (1 - l_discount#4)))] +Aggregate Attributes [2]: [sum#16, isEmpty#17] +Results [2]: [sum#18, isEmpty#19] + +(21) Exchange +Input [2]: [sum#18, isEmpty#19] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] + +(22) ShuffleQueryStage +Output [2]: [sum#18, isEmpty#19] +Arguments: X + +(23) HashAggregate [codegen id : 2] +Input [2]: [sum#18, isEmpty#19] +Keys: [] +Functions [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))] +Aggregate Attributes [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#20] +Results [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#20 AS revenue#21] + +(24) Scan parquet +Output [6]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, l_shipinstruct#5, l_shipmode#6] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] +ReadSchema: struct + +(25) Filter +Input [6]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, l_shipinstruct#5, l_shipmode#6] +Condition : ((((isnotnull(l_shipinstruct#5) AND l_shipmode#6 IN (AIR,AIR REG)) AND (l_shipinstruct#5 = DELIVER IN PERSON)) AND isnotnull(l_partkey#1)) AND ((((l_quantity#2 >= 1.00) AND (l_quantity#2 <= 11.00)) OR ((l_quantity#2 >= 10.00) AND (l_quantity#2 <= 20.00))) OR ((l_quantity#2 >= 20.00) AND (l_quantity#2 <= 30.00)))) + +(26) Project +Output [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4] +Input [6]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, l_shipinstruct#5, l_shipmode#6] + +(27) Exchange +Input [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4] +Arguments: hashpartitioning(l_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(28) Sort +Input [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4] +Arguments: [l_partkey#1 ASC NULLS FIRST], false, 0 + +(29) Scan parquet +Output [4]: [p_partkey#8, p_brand#9, p_size#10, p_container#11] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#12),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#13),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#14),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +ReadSchema: struct + +(30) Filter +Input [4]: [p_partkey#8, p_brand#9, p_size#10, p_container#11] +Condition : (((isnotnull(p_size#10) AND (p_size#10 >= 1)) AND isnotnull(p_partkey#8)) AND (((((p_brand#9 = Brand#12) AND p_container#11 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#10 <= 5)) OR (((p_brand#9 = Brand#13) AND p_container#11 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#10 <= 10))) OR (((p_brand#9 = Brand#14) AND p_container#11 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#10 <= 15)))) + +(31) Exchange +Input [4]: [p_partkey#8, p_brand#9, p_size#10, p_container#11] +Arguments: hashpartitioning(p_partkey#8, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(32) Sort +Input [4]: [p_partkey#8, p_brand#9, p_size#10, p_container#11] +Arguments: [p_partkey#8 ASC NULLS FIRST], false, 0 + +(33) SortMergeJoin +Left keys [1]: [l_partkey#1] +Right keys [1]: [p_partkey#8] +Join type: Inner +Join condition: (((((((p_brand#9 = Brand#12) AND p_container#11 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#2 >= 1.00)) AND (l_quantity#2 <= 11.00)) AND (p_size#10 <= 5)) OR (((((p_brand#9 = Brand#13) AND p_container#11 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#2 >= 10.00)) AND (l_quantity#2 <= 20.00)) AND (p_size#10 <= 10))) OR (((((p_brand#9 = Brand#14) AND p_container#11 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#2 >= 20.00)) AND (l_quantity#2 <= 30.00)) AND (p_size#10 <= 15))) + +(34) Project +Output [2]: [l_extendedprice#3, l_discount#4] +Input [8]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, p_partkey#8, p_brand#9, p_size#10, p_container#11] + +(35) HashAggregate +Input [2]: [l_extendedprice#3, l_discount#4] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#3 * (1 - l_discount#4)))] +Aggregate Attributes [2]: [sum#16, isEmpty#17] +Results [2]: [sum#18, isEmpty#19] + +(36) Exchange +Input [2]: [sum#18, isEmpty#19] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] + +(37) HashAggregate +Input [2]: [sum#18, isEmpty#19] +Keys: [] +Functions [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))] +Aggregate Attributes [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#20] +Results [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#20 AS revenue#21] + +(38) AdaptiveSparkPlan +Output [1]: [revenue#21] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q2.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q2.txt new file mode 100644 index 000000000..97a56f9ef --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q2.txt @@ -0,0 +1,1117 @@ +== Physical Plan == +AdaptiveSparkPlan (207) ++- == Final Plan == + NativeTakeOrdered (132) + +- NativeProject (131) + +- NativeSortMergeJoin Inner (130) + :- NativeSort (123) + : +- InputAdapter (122) + : +- AQEShuffleRead (121) + : +- ShuffleQueryStage (120), Statistics(sizeInBytes=2.8 KiB, rowCount=4) + : +- NativeShuffleExchange (119) + : +- NativeProject (118) + : +- NativeSortMergeJoin Inner (117) + : :- NativeSort (108) + : : +- InputAdapter (107) + : : +- AQEShuffleRead (106) + : : +- ShuffleQueryStage (105), Statistics(sizeInBytes=2.5 KiB, rowCount=4) + : : +- NativeShuffleExchange (104) + : : +- NativeProject (103) + : : +- NativeSortMergeJoin Inner (102) + : : :- NativeSort (93) + : : : +- InputAdapter (92) + : : : +- AQEShuffleRead (91) + : : : +- ShuffleQueryStage (90), Statistics(sizeInBytes=748.0 B, rowCount=4) + : : : +- NativeShuffleExchange (89) + : : : +- NativeProject (88) + : : : +- NativeSortMergeJoin Inner (87) + : : : :- NativeSort (24) + : : : : +- InputAdapter (23) + : : : : +- AQEShuffleRead (22) + : : : : +- ShuffleQueryStage (21), Statistics(sizeInBytes=1620.0 B, rowCount=16) + : : : : +- NativeShuffleExchange (20) + : : : : +- NativeProject (19) + : : : : +- NativeSortMergeJoin Inner (18) + : : : : :- NativeSort (9) + : : : : : +- InputAdapter (8) + : : : : : +- AQEShuffleRead (7) + : : : : : +- ShuffleQueryStage (6), Statistics(sizeInBytes=524.0 B, rowCount=4) + : : : : : +- NativeShuffleExchange (5) + : : : : : +- NativeProject (4) + : : : : : +- NativeFilter (3) + : : : : : +- InputAdapter (2) + : : : : : +- NativeParquetScan (1) + : : : : +- NativeSort (17) + : : : : +- InputAdapter (16) + : : : : +- AQEShuffleRead (15) + : : : : +- ShuffleQueryStage (14), Statistics(sizeInBytes=250.6 KiB, rowCount=8.00E+3) + : : : : +- NativeShuffleExchange (13) + : : : : +- NativeFilter (12) + : : : : +- InputAdapter (11) + : : : : +- NativeParquetScan (10) + : : : +- NativeSort (86) + : : : +- InputAdapter (85) + : : : +- AQEShuffleRead (84) + : : : +- ShuffleQueryStage (83), Statistics(sizeInBytes=28.4 KiB, rowCount=1.20E+3) + : : : +- NativeShuffleExchange (82) + : : : +- NativeFilter (81) + : : : +- NativeProject (80) + : : : +- NativeHashAggregate (79) + : : : +- InputAdapter (78) + : : : +- AQEShuffleRead (77) + : : : +- ShuffleQueryStage (76), Statistics(sizeInBytes=46.5 KiB, rowCount=1.20E+3) + : : : +- NativeShuffleExchange (75) + : : : +- NativeHashAggregate (74) + : : : +- NativeProject (73) + : : : +- NativeSortMergeJoin Inner (72) + : : : :- NativeSort (62) + : : : : +- InputAdapter (61) + : : : : +- AQEShuffleRead (60) + : : : : +- ShuffleQueryStage (59), Statistics(sizeInBytes=250.6 KiB, rowCount=8.00E+3) + : : : : +- NativeShuffleExchange (58) + : : : : +- NativeProject (57) + : : : : +- NativeSortMergeJoin Inner (56) + : : : : :- NativeSort (47) + : : : : : +- InputAdapter (46) + : : : : : +- AQEShuffleRead (45) + : : : : : +- ShuffleQueryStage (44), Statistics(sizeInBytes=250.6 KiB, rowCount=8.00E+3) + : : : : : +- NativeShuffleExchange (43) + : : : : : +- NativeProject (42) + : : : : : +- NativeSortMergeJoin Inner (41) + : : : : : :- NativeSort (32) + : : : : : : +- InputAdapter (31) + : : : : : : +- AQEShuffleRead (30) + : : : : : : +- ShuffleQueryStage (29), Statistics(sizeInBytes=250.6 KiB, rowCount=8.00E+3) + : : : : : : +- NativeShuffleExchange (28) + : : : : : : +- NativeFilter (27) + : : : : : : +- InputAdapter (26) + : : : : : : +- NativeParquetScan (25) + : : : : : +- NativeSort (40) + : : : : : +- InputAdapter (39) + : : : : : +- AQEShuffleRead (38) + : : : : : +- ShuffleQueryStage (37), Statistics(sizeInBytes=1992.0 B, rowCount=100) + : : : : : +- NativeShuffleExchange (36) + : : : : : +- NativeFilter (35) + : : : : : +- InputAdapter (34) + : : : : : +- NativeParquetScan (33) + : : : : +- NativeSort (55) + : : : : +- InputAdapter (54) + : : : : +- AQEShuffleRead (53) + : : : : +- ShuffleQueryStage (52), Statistics(sizeInBytes=792.0 B, rowCount=25) + : : : : +- NativeShuffleExchange (51) + : : : : +- NativeFilter (50) + : : : : +- InputAdapter (49) + : : : : +- NativeParquetScan (48) + : : : +- NativeSort (71) + : : : +- InputAdapter (70) + : : : +- AQEShuffleRead (69) + : : : +- ShuffleQueryStage (68), Statistics(sizeInBytes=216.0 B, rowCount=1) + : : : +- NativeShuffleExchange (67) + : : : +- NativeProject (66) + : : : +- NativeFilter (65) + : : : +- InputAdapter (64) + : : : +- NativeParquetScan (63) + : : +- NativeSort (101) + : : +- InputAdapter (100) + : : +- AQEShuffleRead (99) + : : +- ShuffleQueryStage (98), Statistics(sizeInBytes=19.3 KiB, rowCount=100) + : : +- NativeShuffleExchange (97) + : : +- NativeFilter (96) + : : +- InputAdapter (95) + : : +- NativeParquetScan (94) + : +- NativeSort (116) + : +- InputAdapter (115) + : +- AQEShuffleRead (114) + : +- ShuffleQueryStage (113), Statistics(sizeInBytes=1381.0 B, rowCount=25) + : +- NativeShuffleExchange (112) + : +- NativeFilter (111) + : +- InputAdapter (110) + : +- NativeParquetScan (109) + +- NativeSort (129) + +- InputAdapter (128) + +- InputAdapter (127) + +- AQEShuffleRead (126) + +- ShuffleQueryStage (125), Statistics(sizeInBytes=216.0 B, rowCount=1) + +- ReusedExchange (124) ++- == Initial Plan == + TakeOrderedAndProject (206) + +- Project (205) + +- SortMergeJoin Inner (204) + :- Sort (198) + : +- Exchange (197) + : +- Project (196) + : +- SortMergeJoin Inner (195) + : :- Sort (190) + : : +- Exchange (189) + : : +- Project (188) + : : +- SortMergeJoin Inner (187) + : : :- Sort (182) + : : : +- Exchange (181) + : : : +- Project (180) + : : : +- SortMergeJoin Inner (179) + : : : :- Sort (145) + : : : : +- Exchange (144) + : : : : +- Project (143) + : : : : +- SortMergeJoin Inner (142) + : : : : :- Sort (137) + : : : : : +- Exchange (136) + : : : : : +- Project (135) + : : : : : +- Filter (134) + : : : : : +- Scan parquet (133) + : : : : +- Sort (141) + : : : : +- Exchange (140) + : : : : +- Filter (139) + : : : : +- Scan parquet (138) + : : : +- Sort (178) + : : : +- Exchange (177) + : : : +- Filter (176) + : : : +- HashAggregate (175) + : : : +- Exchange (174) + : : : +- HashAggregate (173) + : : : +- Project (172) + : : : +- SortMergeJoin Inner (171) + : : : :- Sort (165) + : : : : +- Exchange (164) + : : : : +- Project (163) + : : : : +- SortMergeJoin Inner (162) + : : : : :- Sort (157) + : : : : : +- Exchange (156) + : : : : : +- Project (155) + : : : : : +- SortMergeJoin Inner (154) + : : : : : :- Sort (149) + : : : : : : +- Exchange (148) + : : : : : : +- Filter (147) + : : : : : : +- Scan parquet (146) + : : : : : +- Sort (153) + : : : : : +- Exchange (152) + : : : : : +- Filter (151) + : : : : : +- Scan parquet (150) + : : : : +- Sort (161) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Sort (170) + : : : +- Exchange (169) + : : : +- Project (168) + : : : +- Filter (167) + : : : +- Scan parquet (166) + : : +- Sort (186) + : : +- Exchange (185) + : : +- Filter (184) + : : +- Scan parquet (183) + : +- Sort (194) + : +- Exchange (193) + : +- Filter (192) + : +- Scan parquet (191) + +- Sort (203) + +- Exchange (202) + +- Project (201) + +- Filter (200) + +- Scan parquet (199) + + +(133) Scan parquet +Output [4]: [p_partkey#1, p_mfgr#2, p_type#3, p_size#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_size), IsNotNull(p_type), EqualTo(p_size,15), StringEndsWith(p_type,BRASS), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) InputAdapter +Input [4]: [p_partkey#1, p_mfgr#2, p_type#3, p_size#4] +Arguments: [#5, #2, #3, #4] + +(3) NativeFilter +Input [4]: [#5#1, #2#2, #3#3, #4#4] +Condition : ((((isnotnull(p_size#4) AND isnotnull(p_type#3)) AND (p_size#4 = 15)) AND EndsWith(p_type#3, BRASS)) AND isnotnull(p_partkey#1)) + +(4) NativeProject +Output [2]: [p_partkey#1, p_mfgr#2] +Input [4]: [#5#1, #2#2, #3#3, #4#4] + +(5) NativeShuffleExchange +Input [2]: [p_partkey#1, p_mfgr#2] +Arguments: hashpartitioning(p_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(6) ShuffleQueryStage +Output [2]: [p_partkey#1, p_mfgr#2] +Arguments: X + +(7) AQEShuffleRead +Input [2]: [p_partkey#1, p_mfgr#2] +Arguments: coalesced + +(8) InputAdapter +Input [2]: [p_partkey#1, p_mfgr#2] + +(9) NativeSort +Input [2]: [p_partkey#1, p_mfgr#2] +Arguments: [p_partkey#1 ASC NULLS FIRST], false + +(138) Scan parquet +Output [3]: [ps_partkey#6, ps_suppkey#7, ps_supplycost#8] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_partkey), IsNotNull(ps_supplycost), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(11) InputAdapter +Input [3]: [ps_partkey#6, ps_suppkey#7, ps_supplycost#8] +Arguments: [#9, #10, #8] + +(12) NativeFilter +Input [3]: [#9#6, #10#7, #8#8] +Condition : ((isnotnull(ps_partkey#6) AND isnotnull(ps_supplycost#8)) AND isnotnull(ps_suppkey#7)) + +(13) NativeShuffleExchange +Input [3]: [#9#6, #10#7, #8#8] +Arguments: hashpartitioning(ps_partkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [3]: [#9#6, #10#7, #8#8] +Arguments: X + +(15) AQEShuffleRead +Input [3]: [#9#6, #10#7, #8#8] +Arguments: coalesced + +(16) InputAdapter +Input [3]: [#9#6, #10#7, #8#8] + +(17) NativeSort +Input [3]: [#9#6, #10#7, #8#8] +Arguments: [ps_partkey#6 ASC NULLS FIRST], false + +(18) NativeSortMergeJoin +Left keys [1]: [p_partkey#1] +Right keys [1]: [ps_partkey#6] +Join type: Inner +Join condition: None + +(19) NativeProject +Output [4]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8] +Input [5]: [p_partkey#1, p_mfgr#2, #9#6, #10#7, #8#8] + +(20) NativeShuffleExchange +Input [4]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8] +Arguments: hashpartitioning(ps_supplycost#8, p_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(21) ShuffleQueryStage +Output [4]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8] +Arguments: X + +(22) AQEShuffleRead +Input [4]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8] +Arguments: coalesced + +(23) InputAdapter +Input [4]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8] + +(24) NativeSort +Input [4]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8] +Arguments: [ps_supplycost#8 ASC NULLS FIRST, p_partkey#1 ASC NULLS FIRST], false + +(146) Scan parquet +Output [3]: [ps_partkey#11, ps_suppkey#12, ps_supplycost#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(26) InputAdapter +Input [3]: [ps_partkey#11, ps_suppkey#12, ps_supplycost#13] +Arguments: [#14, #15, #13] + +(27) NativeFilter +Input [3]: [#14#11, #15#12, #13#13] +Condition : (isnotnull(ps_suppkey#12) AND isnotnull(ps_partkey#11)) + +(28) NativeShuffleExchange +Input [3]: [#14#11, #15#12, #13#13] +Arguments: hashpartitioning(ps_suppkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(29) ShuffleQueryStage +Output [3]: [#14#11, #15#12, #13#13] +Arguments: X + +(30) AQEShuffleRead +Input [3]: [#14#11, #15#12, #13#13] +Arguments: coalesced + +(31) InputAdapter +Input [3]: [#14#11, #15#12, #13#13] + +(32) NativeSort +Input [3]: [#14#11, #15#12, #13#13] +Arguments: [ps_suppkey#12 ASC NULLS FIRST], false + +(150) Scan parquet +Output [2]: [s_suppkey#16, s_nationkey#17] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(34) InputAdapter +Input [2]: [s_suppkey#16, s_nationkey#17] +Arguments: [#18, #19] + +(35) NativeFilter +Input [2]: [#18#16, #19#17] +Condition : (isnotnull(s_suppkey#16) AND isnotnull(s_nationkey#17)) + +(36) NativeShuffleExchange +Input [2]: [#18#16, #19#17] +Arguments: hashpartitioning(s_suppkey#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(37) ShuffleQueryStage +Output [2]: [#18#16, #19#17] +Arguments: X + +(38) AQEShuffleRead +Input [2]: [#18#16, #19#17] +Arguments: coalesced + +(39) InputAdapter +Input [2]: [#18#16, #19#17] + +(40) NativeSort +Input [2]: [#18#16, #19#17] +Arguments: [s_suppkey#16 ASC NULLS FIRST], false + +(41) NativeSortMergeJoin +Left keys [1]: [ps_suppkey#12] +Right keys [1]: [s_suppkey#16] +Join type: Inner +Join condition: None + +(42) NativeProject +Output [3]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17] +Input [5]: [#14#11, #15#12, #13#13, #18#16, #19#17] + +(43) NativeShuffleExchange +Input [3]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17] +Arguments: hashpartitioning(s_nationkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(44) ShuffleQueryStage +Output [3]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17] +Arguments: X + +(45) AQEShuffleRead +Input [3]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17] +Arguments: coalesced + +(46) InputAdapter +Input [3]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17] + +(47) NativeSort +Input [3]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17] +Arguments: [s_nationkey#17 ASC NULLS FIRST], false + +(158) Scan parquet +Output [2]: [n_nationkey#20, n_regionkey#21] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(49) InputAdapter +Input [2]: [n_nationkey#20, n_regionkey#21] +Arguments: [#22, #23] + +(50) NativeFilter +Input [2]: [#22#20, #23#21] +Condition : (isnotnull(n_nationkey#20) AND isnotnull(n_regionkey#21)) + +(51) NativeShuffleExchange +Input [2]: [#22#20, #23#21] +Arguments: hashpartitioning(n_nationkey#20, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(52) ShuffleQueryStage +Output [2]: [#22#20, #23#21] +Arguments: X + +(53) AQEShuffleRead +Input [2]: [#22#20, #23#21] +Arguments: coalesced + +(54) InputAdapter +Input [2]: [#22#20, #23#21] + +(55) NativeSort +Input [2]: [#22#20, #23#21] +Arguments: [n_nationkey#20 ASC NULLS FIRST], false + +(56) NativeSortMergeJoin +Left keys [1]: [s_nationkey#17] +Right keys [1]: [n_nationkey#20] +Join type: Inner +Join condition: None + +(57) NativeProject +Output [3]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21] +Input [5]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17, #22#20, #23#21] + +(58) NativeShuffleExchange +Input [3]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21] +Arguments: hashpartitioning(n_regionkey#21, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(59) ShuffleQueryStage +Output [3]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21] +Arguments: X + +(60) AQEShuffleRead +Input [3]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21] +Arguments: coalesced + +(61) InputAdapter +Input [3]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21] + +(62) NativeSort +Input [3]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21] +Arguments: [n_regionkey#21 ASC NULLS FIRST], false + +(166) Scan parquet +Output [2]: [r_regionkey#24, r_name#25] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,EUROPE), IsNotNull(r_regionkey)] +ReadSchema: struct + +(64) InputAdapter +Input [2]: [r_regionkey#24, r_name#25] +Arguments: [#26, #25] + +(65) NativeFilter +Input [2]: [#26#24, #25#25] +Condition : ((isnotnull(r_name#25) AND (r_name#25 = EUROPE)) AND isnotnull(r_regionkey#24)) + +(66) NativeProject +Output [1]: [r_regionkey#24] +Input [2]: [#26#24, #25#25] + +(67) NativeShuffleExchange +Input [1]: [r_regionkey#24] +Arguments: hashpartitioning(r_regionkey#24, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(68) ShuffleQueryStage +Output [1]: [r_regionkey#24] +Arguments: X + +(69) AQEShuffleRead +Input [1]: [r_regionkey#24] +Arguments: coalesced + +(70) InputAdapter +Input [1]: [r_regionkey#24] + +(71) NativeSort +Input [1]: [r_regionkey#24] +Arguments: [r_regionkey#24 ASC NULLS FIRST], false + +(72) NativeSortMergeJoin +Left keys [1]: [n_regionkey#21] +Right keys [1]: [r_regionkey#24] +Join type: Inner +Join condition: None + +(73) NativeProject +Output [2]: [ps_partkey#11, ps_supplycost#13] +Input [4]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21, r_regionkey#24] + +(74) NativeHashAggregate +Input [2]: [ps_partkey#11, ps_supplycost#13] +Keys [1]: [ps_partkey#11] +Functions [1]: [partial_min(ps_supplycost#13)] +Aggregate Attributes [1]: [min#27] +Results [2]: [ps_partkey#11, #28#28] + +(75) NativeShuffleExchange +Input [2]: [ps_partkey#11, #28#28] +Arguments: hashpartitioning(ps_partkey#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(76) ShuffleQueryStage +Output [2]: [ps_partkey#11, #28#28] +Arguments: X + +(77) AQEShuffleRead +Input [2]: [ps_partkey#11, #28#28] +Arguments: coalesced + +(78) InputAdapter +Input [2]: [ps_partkey#11, #28#28] + +(79) NativeHashAggregate +Input [2]: [ps_partkey#11, #28#28] +Keys [1]: [ps_partkey#11] +Functions [1]: [min(ps_supplycost#13)] +Aggregate Attributes [1]: [min(ps_supplycost#13)#29] +Results [2]: [ps_partkey#11, min(ps_supplycost#13)#29] + +(80) NativeProject +Output [2]: [min(ps_supplycost#13)#29 AS min(ps_supplycost)#30, ps_partkey#11] +Input [2]: [ps_partkey#11, min(ps_supplycost#13)#29] + +(81) NativeFilter +Input [2]: [min(ps_supplycost)#30, ps_partkey#11] +Condition : isnotnull(min(ps_supplycost)#30) + +(82) NativeShuffleExchange +Input [2]: [min(ps_supplycost)#30, ps_partkey#11] +Arguments: hashpartitioning(min(ps_supplycost)#30, ps_partkey#11, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(83) ShuffleQueryStage +Output [2]: [min(ps_supplycost)#30, ps_partkey#11] +Arguments: X + +(84) AQEShuffleRead +Input [2]: [min(ps_supplycost)#30, ps_partkey#11] +Arguments: coalesced + +(85) InputAdapter +Input [2]: [min(ps_supplycost)#30, ps_partkey#11] + +(86) NativeSort +Input [2]: [min(ps_supplycost)#30, ps_partkey#11] +Arguments: [min(ps_supplycost)#30 ASC NULLS FIRST, ps_partkey#11 ASC NULLS FIRST], false + +(87) NativeSortMergeJoin +Left keys [2]: [ps_supplycost#8, p_partkey#1] +Right keys [2]: [min(ps_supplycost)#30, ps_partkey#11] +Join type: Inner +Join condition: None + +(88) NativeProject +Output [3]: [p_partkey#1, p_mfgr#2, ps_suppkey#7] +Input [6]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8, min(ps_supplycost)#30, ps_partkey#11] + +(89) NativeShuffleExchange +Input [3]: [p_partkey#1, p_mfgr#2, ps_suppkey#7] +Arguments: hashpartitioning(ps_suppkey#7, 5), ENSURE_REQUIREMENTS, [plan_id=12] + +(90) ShuffleQueryStage +Output [3]: [p_partkey#1, p_mfgr#2, ps_suppkey#7] +Arguments: X + +(91) AQEShuffleRead +Input [3]: [p_partkey#1, p_mfgr#2, ps_suppkey#7] +Arguments: coalesced + +(92) InputAdapter +Input [3]: [p_partkey#1, p_mfgr#2, ps_suppkey#7] + +(93) NativeSort +Input [3]: [p_partkey#1, p_mfgr#2, ps_suppkey#7] +Arguments: [ps_suppkey#7 ASC NULLS FIRST], false + +(183) Scan parquet +Output [7]: [s_suppkey#31, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(95) InputAdapter +Input [7]: [s_suppkey#31, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Arguments: [#38, #32, #33, #39, #35, #36, #37] + +(96) NativeFilter +Input [7]: [#38#31, #32#32, #33#33, #39#34, #35#35, #36#36, #37#37] +Condition : (isnotnull(s_suppkey#31) AND isnotnull(s_nationkey#34)) + +(97) NativeShuffleExchange +Input [7]: [#38#31, #32#32, #33#33, #39#34, #35#35, #36#36, #37#37] +Arguments: hashpartitioning(s_suppkey#31, 5), ENSURE_REQUIREMENTS, [plan_id=13] + +(98) ShuffleQueryStage +Output [7]: [#38#31, #32#32, #33#33, #39#34, #35#35, #36#36, #37#37] +Arguments: X + +(99) AQEShuffleRead +Input [7]: [#38#31, #32#32, #33#33, #39#34, #35#35, #36#36, #37#37] +Arguments: coalesced + +(100) InputAdapter +Input [7]: [#38#31, #32#32, #33#33, #39#34, #35#35, #36#36, #37#37] + +(101) NativeSort +Input [7]: [#38#31, #32#32, #33#33, #39#34, #35#35, #36#36, #37#37] +Arguments: [s_suppkey#31 ASC NULLS FIRST], false + +(102) NativeSortMergeJoin +Left keys [1]: [ps_suppkey#7] +Right keys [1]: [s_suppkey#31] +Join type: Inner +Join condition: None + +(103) NativeProject +Output [8]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Input [10]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, #38#31, #32#32, #33#33, #39#34, #35#35, #36#36, #37#37] + +(104) NativeShuffleExchange +Input [8]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Arguments: hashpartitioning(s_nationkey#34, 5), ENSURE_REQUIREMENTS, [plan_id=14] + +(105) ShuffleQueryStage +Output [8]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Arguments: X + +(106) AQEShuffleRead +Input [8]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Arguments: coalesced + +(107) InputAdapter +Input [8]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] + +(108) NativeSort +Input [8]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Arguments: [s_nationkey#34 ASC NULLS FIRST], false + +(191) Scan parquet +Output [3]: [n_nationkey#40, n_name#41, n_regionkey#42] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(110) InputAdapter +Input [3]: [n_nationkey#40, n_name#41, n_regionkey#42] +Arguments: [#43, #41, #44] + +(111) NativeFilter +Input [3]: [#43#40, #41#41, #44#42] +Condition : (isnotnull(n_nationkey#40) AND isnotnull(n_regionkey#42)) + +(112) NativeShuffleExchange +Input [3]: [#43#40, #41#41, #44#42] +Arguments: hashpartitioning(n_nationkey#40, 5), ENSURE_REQUIREMENTS, [plan_id=15] + +(113) ShuffleQueryStage +Output [3]: [#43#40, #41#41, #44#42] +Arguments: X + +(114) AQEShuffleRead +Input [3]: [#43#40, #41#41, #44#42] +Arguments: coalesced + +(115) InputAdapter +Input [3]: [#43#40, #41#41, #44#42] + +(116) NativeSort +Input [3]: [#43#40, #41#41, #44#42] +Arguments: [n_nationkey#40 ASC NULLS FIRST], false + +(117) NativeSortMergeJoin +Left keys [1]: [s_nationkey#34] +Right keys [1]: [n_nationkey#40] +Join type: Inner +Join condition: None + +(118) NativeProject +Output [9]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42] +Input [11]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37, #43#40, #41#41, #44#42] + +(119) NativeShuffleExchange +Input [9]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42] +Arguments: hashpartitioning(n_regionkey#42, 5), ENSURE_REQUIREMENTS, [plan_id=16] + +(120) ShuffleQueryStage +Output [9]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42] +Arguments: X + +(121) AQEShuffleRead +Input [9]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42] +Arguments: coalesced + +(122) InputAdapter +Input [9]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42] + +(123) NativeSort +Input [9]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42] +Arguments: [n_regionkey#42 ASC NULLS FIRST], false + +(124) ReusedExchange [Reuses operator id: 67] +Output [1]: [r_regionkey#45] + +(125) ShuffleQueryStage +Output [1]: [r_regionkey#45] +Arguments: X + +(126) AQEShuffleRead +Input [1]: [r_regionkey#45] +Arguments: coalesced + +(127) InputAdapter +Input [1]: [r_regionkey#45] +Arguments: [#46] + +(128) InputAdapter +Input [1]: [#46#45] + +(129) NativeSort +Input [1]: [#46#45] +Arguments: [r_regionkey#45 ASC NULLS FIRST], false + +(130) NativeSortMergeJoin +Left keys [1]: [n_regionkey#42] +Right keys [1]: [r_regionkey#45] +Join type: Inner +Join condition: None + +(131) NativeProject +Output [8]: [s_acctbal#36, s_name#32, n_name#41, p_partkey#1, p_mfgr#2, s_address#33, s_phone#35, s_comment#37] +Input [10]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42, #46#45] + +(132) NativeTakeOrdered +Input [8]: [s_acctbal#36, s_name#32, n_name#41, p_partkey#1, p_mfgr#2, s_address#33, s_phone#35, s_comment#37] +Arguments: X, [s_acctbal#36 DESC NULLS LAST, n_name#41 ASC NULLS FIRST, s_name#32 ASC NULLS FIRST, p_partkey#1 ASC NULLS FIRST] + +(133) Scan parquet +Output [4]: [p_partkey#1, p_mfgr#2, p_type#3, p_size#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_size), IsNotNull(p_type), EqualTo(p_size,15), StringEndsWith(p_type,BRASS), IsNotNull(p_partkey)] +ReadSchema: struct + +(134) Filter +Input [4]: [p_partkey#1, p_mfgr#2, p_type#3, p_size#4] +Condition : ((((isnotnull(p_size#4) AND isnotnull(p_type#3)) AND (p_size#4 = 15)) AND EndsWith(p_type#3, BRASS)) AND isnotnull(p_partkey#1)) + +(135) Project +Output [2]: [p_partkey#1, p_mfgr#2] +Input [4]: [p_partkey#1, p_mfgr#2, p_type#3, p_size#4] + +(136) Exchange +Input [2]: [p_partkey#1, p_mfgr#2] +Arguments: hashpartitioning(p_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=17] + +(137) Sort +Input [2]: [p_partkey#1, p_mfgr#2] +Arguments: [p_partkey#1 ASC NULLS FIRST], false, 0 + +(138) Scan parquet +Output [3]: [ps_partkey#6, ps_suppkey#7, ps_supplycost#8] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_partkey), IsNotNull(ps_supplycost), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(139) Filter +Input [3]: [ps_partkey#6, ps_suppkey#7, ps_supplycost#8] +Condition : ((isnotnull(ps_partkey#6) AND isnotnull(ps_supplycost#8)) AND isnotnull(ps_suppkey#7)) + +(140) Exchange +Input [3]: [ps_partkey#6, ps_suppkey#7, ps_supplycost#8] +Arguments: hashpartitioning(ps_partkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=18] + +(141) Sort +Input [3]: [ps_partkey#6, ps_suppkey#7, ps_supplycost#8] +Arguments: [ps_partkey#6 ASC NULLS FIRST], false, 0 + +(142) SortMergeJoin +Left keys [1]: [p_partkey#1] +Right keys [1]: [ps_partkey#6] +Join type: Inner +Join condition: None + +(143) Project +Output [4]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8] +Input [5]: [p_partkey#1, p_mfgr#2, ps_partkey#6, ps_suppkey#7, ps_supplycost#8] + +(144) Exchange +Input [4]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8] +Arguments: hashpartitioning(ps_supplycost#8, p_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=19] + +(145) Sort +Input [4]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8] +Arguments: [ps_supplycost#8 ASC NULLS FIRST, p_partkey#1 ASC NULLS FIRST], false, 0 + +(146) Scan parquet +Output [3]: [ps_partkey#11, ps_suppkey#12, ps_supplycost#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(147) Filter +Input [3]: [ps_partkey#11, ps_suppkey#12, ps_supplycost#13] +Condition : (isnotnull(ps_suppkey#12) AND isnotnull(ps_partkey#11)) + +(148) Exchange +Input [3]: [ps_partkey#11, ps_suppkey#12, ps_supplycost#13] +Arguments: hashpartitioning(ps_suppkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=20] + +(149) Sort +Input [3]: [ps_partkey#11, ps_suppkey#12, ps_supplycost#13] +Arguments: [ps_suppkey#12 ASC NULLS FIRST], false, 0 + +(150) Scan parquet +Output [2]: [s_suppkey#16, s_nationkey#17] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(151) Filter +Input [2]: [s_suppkey#16, s_nationkey#17] +Condition : (isnotnull(s_suppkey#16) AND isnotnull(s_nationkey#17)) + +(152) Exchange +Input [2]: [s_suppkey#16, s_nationkey#17] +Arguments: hashpartitioning(s_suppkey#16, 5), ENSURE_REQUIREMENTS, [plan_id=21] + +(153) Sort +Input [2]: [s_suppkey#16, s_nationkey#17] +Arguments: [s_suppkey#16 ASC NULLS FIRST], false, 0 + +(154) SortMergeJoin +Left keys [1]: [ps_suppkey#12] +Right keys [1]: [s_suppkey#16] +Join type: Inner +Join condition: None + +(155) Project +Output [3]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17] +Input [5]: [ps_partkey#11, ps_suppkey#12, ps_supplycost#13, s_suppkey#16, s_nationkey#17] + +(156) Exchange +Input [3]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17] +Arguments: hashpartitioning(s_nationkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=22] + +(157) Sort +Input [3]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17] +Arguments: [s_nationkey#17 ASC NULLS FIRST], false, 0 + +(158) Scan parquet +Output [2]: [n_nationkey#20, n_regionkey#21] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(159) Filter +Input [2]: [n_nationkey#20, n_regionkey#21] +Condition : (isnotnull(n_nationkey#20) AND isnotnull(n_regionkey#21)) + +(160) Exchange +Input [2]: [n_nationkey#20, n_regionkey#21] +Arguments: hashpartitioning(n_nationkey#20, 5), ENSURE_REQUIREMENTS, [plan_id=23] + +(161) Sort +Input [2]: [n_nationkey#20, n_regionkey#21] +Arguments: [n_nationkey#20 ASC NULLS FIRST], false, 0 + +(162) SortMergeJoin +Left keys [1]: [s_nationkey#17] +Right keys [1]: [n_nationkey#20] +Join type: Inner +Join condition: None + +(163) Project +Output [3]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21] +Input [5]: [ps_partkey#11, ps_supplycost#13, s_nationkey#17, n_nationkey#20, n_regionkey#21] + +(164) Exchange +Input [3]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21] +Arguments: hashpartitioning(n_regionkey#21, 5), ENSURE_REQUIREMENTS, [plan_id=24] + +(165) Sort +Input [3]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21] +Arguments: [n_regionkey#21 ASC NULLS FIRST], false, 0 + +(166) Scan parquet +Output [2]: [r_regionkey#24, r_name#25] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,EUROPE), IsNotNull(r_regionkey)] +ReadSchema: struct + +(167) Filter +Input [2]: [r_regionkey#24, r_name#25] +Condition : ((isnotnull(r_name#25) AND (r_name#25 = EUROPE)) AND isnotnull(r_regionkey#24)) + +(168) Project +Output [1]: [r_regionkey#24] +Input [2]: [r_regionkey#24, r_name#25] + +(169) Exchange +Input [1]: [r_regionkey#24] +Arguments: hashpartitioning(r_regionkey#24, 5), ENSURE_REQUIREMENTS, [plan_id=25] + +(170) Sort +Input [1]: [r_regionkey#24] +Arguments: [r_regionkey#24 ASC NULLS FIRST], false, 0 + +(171) SortMergeJoin +Left keys [1]: [n_regionkey#21] +Right keys [1]: [r_regionkey#24] +Join type: Inner +Join condition: None + +(172) Project +Output [2]: [ps_partkey#11, ps_supplycost#13] +Input [4]: [ps_partkey#11, ps_supplycost#13, n_regionkey#21, r_regionkey#24] + +(173) HashAggregate +Input [2]: [ps_partkey#11, ps_supplycost#13] +Keys [1]: [ps_partkey#11] +Functions [1]: [partial_min(ps_supplycost#13)] +Aggregate Attributes [1]: [min#27] +Results [2]: [ps_partkey#11, min#47] + +(174) Exchange +Input [2]: [ps_partkey#11, min#47] +Arguments: hashpartitioning(ps_partkey#11, 5), ENSURE_REQUIREMENTS, [plan_id=26] + +(175) HashAggregate +Input [2]: [ps_partkey#11, min#47] +Keys [1]: [ps_partkey#11] +Functions [1]: [min(ps_supplycost#13)] +Aggregate Attributes [1]: [min(ps_supplycost#13)#29] +Results [2]: [min(ps_supplycost#13)#29 AS min(ps_supplycost)#30, ps_partkey#11] + +(176) Filter +Input [2]: [min(ps_supplycost)#30, ps_partkey#11] +Condition : isnotnull(min(ps_supplycost)#30) + +(177) Exchange +Input [2]: [min(ps_supplycost)#30, ps_partkey#11] +Arguments: hashpartitioning(min(ps_supplycost)#30, ps_partkey#11, 5), ENSURE_REQUIREMENTS, [plan_id=27] + +(178) Sort +Input [2]: [min(ps_supplycost)#30, ps_partkey#11] +Arguments: [min(ps_supplycost)#30 ASC NULLS FIRST, ps_partkey#11 ASC NULLS FIRST], false, 0 + +(179) SortMergeJoin +Left keys [2]: [ps_supplycost#8, p_partkey#1] +Right keys [2]: [min(ps_supplycost)#30, ps_partkey#11] +Join type: Inner +Join condition: None + +(180) Project +Output [3]: [p_partkey#1, p_mfgr#2, ps_suppkey#7] +Input [6]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, ps_supplycost#8, min(ps_supplycost)#30, ps_partkey#11] + +(181) Exchange +Input [3]: [p_partkey#1, p_mfgr#2, ps_suppkey#7] +Arguments: hashpartitioning(ps_suppkey#7, 5), ENSURE_REQUIREMENTS, [plan_id=28] + +(182) Sort +Input [3]: [p_partkey#1, p_mfgr#2, ps_suppkey#7] +Arguments: [ps_suppkey#7 ASC NULLS FIRST], false, 0 + +(183) Scan parquet +Output [7]: [s_suppkey#31, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(184) Filter +Input [7]: [s_suppkey#31, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Condition : (isnotnull(s_suppkey#31) AND isnotnull(s_nationkey#34)) + +(185) Exchange +Input [7]: [s_suppkey#31, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Arguments: hashpartitioning(s_suppkey#31, 5), ENSURE_REQUIREMENTS, [plan_id=29] + +(186) Sort +Input [7]: [s_suppkey#31, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Arguments: [s_suppkey#31 ASC NULLS FIRST], false, 0 + +(187) SortMergeJoin +Left keys [1]: [ps_suppkey#7] +Right keys [1]: [s_suppkey#31] +Join type: Inner +Join condition: None + +(188) Project +Output [8]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Input [10]: [p_partkey#1, p_mfgr#2, ps_suppkey#7, s_suppkey#31, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] + +(189) Exchange +Input [8]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Arguments: hashpartitioning(s_nationkey#34, 5), ENSURE_REQUIREMENTS, [plan_id=30] + +(190) Sort +Input [8]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37] +Arguments: [s_nationkey#34 ASC NULLS FIRST], false, 0 + +(191) Scan parquet +Output [3]: [n_nationkey#40, n_name#41, n_regionkey#42] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(192) Filter +Input [3]: [n_nationkey#40, n_name#41, n_regionkey#42] +Condition : (isnotnull(n_nationkey#40) AND isnotnull(n_regionkey#42)) + +(193) Exchange +Input [3]: [n_nationkey#40, n_name#41, n_regionkey#42] +Arguments: hashpartitioning(n_nationkey#40, 5), ENSURE_REQUIREMENTS, [plan_id=31] + +(194) Sort +Input [3]: [n_nationkey#40, n_name#41, n_regionkey#42] +Arguments: [n_nationkey#40 ASC NULLS FIRST], false, 0 + +(195) SortMergeJoin +Left keys [1]: [s_nationkey#34] +Right keys [1]: [n_nationkey#40] +Join type: Inner +Join condition: None + +(196) Project +Output [9]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42] +Input [11]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_nationkey#34, s_phone#35, s_acctbal#36, s_comment#37, n_nationkey#40, n_name#41, n_regionkey#42] + +(197) Exchange +Input [9]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42] +Arguments: hashpartitioning(n_regionkey#42, 5), ENSURE_REQUIREMENTS, [plan_id=32] + +(198) Sort +Input [9]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42] +Arguments: [n_regionkey#42 ASC NULLS FIRST], false, 0 + +(199) Scan parquet +Output [2]: [r_regionkey#45, r_name#48] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,EUROPE), IsNotNull(r_regionkey)] +ReadSchema: struct + +(200) Filter +Input [2]: [r_regionkey#45, r_name#48] +Condition : ((isnotnull(r_name#48) AND (r_name#48 = EUROPE)) AND isnotnull(r_regionkey#45)) + +(201) Project +Output [1]: [r_regionkey#45] +Input [2]: [r_regionkey#45, r_name#48] + +(202) Exchange +Input [1]: [r_regionkey#45] +Arguments: hashpartitioning(r_regionkey#45, 5), ENSURE_REQUIREMENTS, [plan_id=33] + +(203) Sort +Input [1]: [r_regionkey#45] +Arguments: [r_regionkey#45 ASC NULLS FIRST], false, 0 + +(204) SortMergeJoin +Left keys [1]: [n_regionkey#42] +Right keys [1]: [r_regionkey#45] +Join type: Inner +Join condition: None + +(205) Project +Output [8]: [s_acctbal#36, s_name#32, n_name#41, p_partkey#1, p_mfgr#2, s_address#33, s_phone#35, s_comment#37] +Input [10]: [p_partkey#1, p_mfgr#2, s_name#32, s_address#33, s_phone#35, s_acctbal#36, s_comment#37, n_name#41, n_regionkey#42, r_regionkey#45] + +(206) TakeOrderedAndProject +Input [8]: [s_acctbal#36, s_name#32, n_name#41, p_partkey#1, p_mfgr#2, s_address#33, s_phone#35, s_comment#37] +Arguments: X, [s_acctbal#36 DESC NULLS LAST, n_name#41 ASC NULLS FIRST, s_name#32 ASC NULLS FIRST, p_partkey#1 ASC NULLS FIRST], [s_acctbal#36, s_name#32, n_name#41, p_partkey#1, p_mfgr#2, s_address#33, s_phone#35, s_comment#37] + +(207) AdaptiveSparkPlan +Output [8]: [s_acctbal#36, s_name#32, n_name#41, p_partkey#1, p_mfgr#2, s_address#33, s_phone#35, s_comment#37] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q20.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q20.txt new file mode 100644 index 000000000..d11cb0ed5 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q20.txt @@ -0,0 +1,732 @@ +== Physical Plan == +AdaptiveSparkPlan (135) ++- == Final Plan == + InputAdapter (85) + +- AQEShuffleRead (84) + +- ShuffleQueryStage (83), Statistics(sizeInBytes=584.0 B, rowCount=1) + +- NativeShuffleExchange (82) + +- NativeProject (81) + +- NativeSortMergeJoin Inner (80) + :- NativeSort (70) + : +- InputAdapter (69) + : +- AQEShuffleRead (68) + : +- ShuffleQueryStage (67), Statistics(sizeInBytes=2.7 KiB, rowCount=35) + : +- NativeShuffleExchange (66) + : +- NativeProject (65) + : +- NativeSortMergeJoin LeftSemi (64) + : :- NativeSort (8) + : : +- InputAdapter (7) + : : +- AQEShuffleRead (6) + : : +- ShuffleQueryStage (5), Statistics(sizeInBytes=8.2 KiB, rowCount=100) + : : +- NativeShuffleExchange (4) + : : +- NativeFilter (3) + : : +- InputAdapter (2) + : : +- NativeParquetScan (1) + : +- NativeSort (63) + : +- InputAdapter (62) + : +- AQEShuffleRead (61) + : +- ShuffleQueryStage (60), Statistics(sizeInBytes=568.0 B, rowCount=45) + : +- NativeShuffleExchange (59) + : +- ConvertToNative (58) + : +- * Project (57) + : +- * SortMergeJoin Inner (56) + : :- NativeSort (31) + : : +- InputAdapter (30) + : : +- AQEShuffleRead (29) + : : +- ShuffleQueryStage (28), Statistics(sizeInBytes=1856.0 B, rowCount=64) + : : +- NativeShuffleExchange (27) + : : +- NativeSortMergeJoin LeftSemi (26) + : : :- NativeSort (16) + : : : +- InputAdapter (15) + : : : +- AQEShuffleRead (14) + : : : +- ShuffleQueryStage (13), Statistics(sizeInBytes=156.8 KiB, rowCount=8.00E+3) + : : : +- NativeShuffleExchange (12) + : : : +- NativeFilter (11) + : : : +- InputAdapter (10) + : : : +- NativeParquetScan (9) + : : +- NativeSort (25) + : : +- InputAdapter (24) + : : +- AQEShuffleRead (23) + : : +- ShuffleQueryStage (22), Statistics(sizeInBytes=336.0 B, rowCount=16) + : : +- NativeShuffleExchange (21) + : : +- NativeProject (20) + : : +- NativeFilter (19) + : : +- InputAdapter (18) + : : +- NativeParquetScan (17) + : +- NativeSort (55) + : +- InputAdapter (54) + : +- AQEShuffleRead (53) + : +- ShuffleQueryStage (52), Statistics(sizeInBytes=2.0 KiB, rowCount=45) + : +- NativeShuffleExchange (51) + : +- NativeFilter (50) + : +- NativeProject (49) + : +- NativeHashAggregate (48) + : +- NativeHashAggregate (47) + : +- NativeSortMergeJoin LeftSemi (46) + : :- NativeSort (40) + : : +- InputAdapter (39) + : : +- AQEShuffleRead (38) + : : +- ShuffleQueryStage (37), Statistics(sizeInBytes=297.0 KiB, rowCount=9.48E+3) + : : +- NativeShuffleExchange (36) + : : +- NativeProject (35) + : : +- NativeFilter (34) + : : +- InputAdapter (33) + : : +- NativeParquetScan (32) + : +- NativeSort (45) + : +- InputAdapter (44) + : +- AQEShuffleRead (43) + : +- ShuffleQueryStage (42), Statistics(sizeInBytes=336.0 B, rowCount=16) + : +- ReusedExchange (41) + +- NativeSort (79) + +- InputAdapter (78) + +- AQEShuffleRead (77) + +- ShuffleQueryStage (76), Statistics(sizeInBytes=216.0 B, rowCount=1) + +- NativeShuffleExchange (75) + +- NativeProject (74) + +- NativeFilter (73) + +- InputAdapter (72) + +- NativeParquetScan (71) ++- == Initial Plan == + Sort (134) + +- Exchange (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (125) + : +- Exchange (124) + : +- Project (123) + : +- SortMergeJoin LeftSemi (122) + : :- Sort (89) + : : +- Exchange (88) + : : +- Filter (87) + : : +- Scan parquet (86) + : +- Sort (121) + : +- Exchange (120) + : +- Project (119) + : +- SortMergeJoin Inner (118) + : :- Sort (101) + : : +- Exchange (100) + : : +- SortMergeJoin LeftSemi (99) + : : :- Sort (93) + : : : +- Exchange (92) + : : : +- Filter (91) + : : : +- Scan parquet (90) + : : +- Sort (98) + : : +- Exchange (97) + : : +- Project (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Sort (117) + : +- Exchange (116) + : +- Filter (115) + : +- HashAggregate (114) + : +- HashAggregate (113) + : +- SortMergeJoin LeftSemi (112) + : :- Sort (106) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Sort (111) + : +- Exchange (110) + : +- Project (109) + : +- Filter (108) + : +- Scan parquet (107) + +- Sort (130) + +- Exchange (129) + +- Project (128) + +- Filter (127) + +- Scan parquet (126) + + +(86) Scan parquet +Output [4]: [s_suppkey#1, s_name#2, s_address#3, s_nationkey#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) InputAdapter +Input [4]: [s_suppkey#1, s_name#2, s_address#3, s_nationkey#4] +Arguments: [#5, #2, #3, #6] + +(3) NativeFilter +Input [4]: [#5#1, #2#2, #3#3, #6#4] +Condition : isnotnull(s_nationkey#4) + +(4) NativeShuffleExchange +Input [4]: [#5#1, #2#2, #3#3, #6#4] +Arguments: hashpartitioning(s_suppkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [4]: [#5#1, #2#2, #3#3, #6#4] +Arguments: X + +(6) AQEShuffleRead +Input [4]: [#5#1, #2#2, #3#3, #6#4] +Arguments: coalesced + +(7) InputAdapter +Input [4]: [#5#1, #2#2, #3#3, #6#4] + +(8) NativeSort +Input [4]: [#5#1, #2#2, #3#3, #6#4] +Arguments: [s_suppkey#1 ASC NULLS FIRST], false + +(90) Scan parquet +Output [3]: [ps_partkey#7, ps_suppkey#8, ps_availqty#9] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(10) InputAdapter +Input [3]: [ps_partkey#7, ps_suppkey#8, ps_availqty#9] +Arguments: [#10, #11, #9] + +(11) NativeFilter +Input [3]: [#10#7, #11#8, #9#9] +Condition : ((isnotnull(ps_availqty#9) AND isnotnull(ps_partkey#7)) AND isnotnull(ps_suppkey#8)) + +(12) NativeShuffleExchange +Input [3]: [#10#7, #11#8, #9#9] +Arguments: hashpartitioning(ps_partkey#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(13) ShuffleQueryStage +Output [3]: [#10#7, #11#8, #9#9] +Arguments: X + +(14) AQEShuffleRead +Input [3]: [#10#7, #11#8, #9#9] +Arguments: coalesced + +(15) InputAdapter +Input [3]: [#10#7, #11#8, #9#9] + +(16) NativeSort +Input [3]: [#10#7, #11#8, #9#9] +Arguments: [ps_partkey#7 ASC NULLS FIRST], false + +(94) Scan parquet +Output [2]: [p_partkey#12, p_name#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(18) InputAdapter +Input [2]: [p_partkey#12, p_name#13] +Arguments: [#14, #13] + +(19) NativeFilter +Input [2]: [#14#12, #13#13] +Condition : (isnotnull(p_name#13) AND StartsWith(p_name#13, forest)) + +(20) NativeProject +Output [1]: [p_partkey#12] +Input [2]: [#14#12, #13#13] + +(21) NativeShuffleExchange +Input [1]: [p_partkey#12] +Arguments: hashpartitioning(p_partkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(22) ShuffleQueryStage +Output [1]: [p_partkey#12] +Arguments: X + +(23) AQEShuffleRead +Input [1]: [p_partkey#12] +Arguments: coalesced + +(24) InputAdapter +Input [1]: [p_partkey#12] + +(25) NativeSort +Input [1]: [p_partkey#12] +Arguments: [p_partkey#12 ASC NULLS FIRST], false + +(26) NativeSortMergeJoin +Left keys [1]: [ps_partkey#7] +Right keys [1]: [p_partkey#12] +Join type: LeftSemi +Join condition: None + +(27) NativeShuffleExchange +Input [3]: [#10#7, #11#8, #9#9] +Arguments: hashpartitioning(ps_partkey#7, ps_suppkey#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(28) ShuffleQueryStage +Output [3]: [#10#7, #11#8, #9#9] +Arguments: X + +(29) AQEShuffleRead +Input [3]: [#10#7, #11#8, #9#9] +Arguments: coalesced + +(30) InputAdapter +Input [3]: [#10#7, #11#8, #9#9] + +(31) NativeSort +Input [3]: [#10#7, #11#8, #9#9] +Arguments: [ps_partkey#7 ASC NULLS FIRST, ps_suppkey#8 ASC NULLS FIRST], false + +(102) Scan parquet +Output [4]: [l_partkey#15, l_suppkey#16, l_quantity#17, l_shipdate#18] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(33) InputAdapter +Input [4]: [l_partkey#15, l_suppkey#16, l_quantity#17, l_shipdate#18] +Arguments: [#19, #20, #17, #18] + +(34) NativeFilter +Input [4]: [#19#15, #20#16, #17#17, #18#18] +Condition : ((((isnotnull(l_shipdate#18) AND (l_shipdate#18 >= 1994-01-01)) AND (l_shipdate#18 < 1995-01-01)) AND isnotnull(l_partkey#15)) AND isnotnull(l_suppkey#16)) + +(35) NativeProject +Output [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] +Input [4]: [#19#15, #20#16, #17#17, #18#18] + +(36) NativeShuffleExchange +Input [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] +Arguments: hashpartitioning(l_partkey#15, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(37) ShuffleQueryStage +Output [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] +Arguments: X + +(38) AQEShuffleRead +Input [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] +Arguments: coalesced + +(39) InputAdapter +Input [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] + +(40) NativeSort +Input [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] +Arguments: [l_partkey#15 ASC NULLS FIRST], false + +(41) ReusedExchange [Reuses operator id: 21] +Output [1]: [p_partkey#12] + +(42) ShuffleQueryStage +Output [1]: [p_partkey#12] +Arguments: X + +(43) AQEShuffleRead +Input [1]: [p_partkey#12] +Arguments: coalesced + +(44) InputAdapter +Input [1]: [p_partkey#12] + +(45) NativeSort +Input [1]: [p_partkey#12] +Arguments: [p_partkey#12 ASC NULLS FIRST], false + +(46) NativeSortMergeJoin +Left keys [1]: [l_partkey#15] +Right keys [1]: [p_partkey#12] +Join type: LeftSemi +Join condition: None + +(47) NativeHashAggregate +Input [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] +Keys [2]: [l_partkey#15, l_suppkey#16] +Functions [1]: [partial_sum(l_quantity#17)] +Aggregate Attributes [2]: [sum#21, isEmpty#22] +Results [3]: [l_partkey#15, l_suppkey#16, #23#23] + +(48) NativeHashAggregate +Input [3]: [l_partkey#15, l_suppkey#16, #23#23] +Keys [2]: [l_partkey#15, l_suppkey#16] +Functions [1]: [sum(l_quantity#17)] +Aggregate Attributes [1]: [sum(l_quantity#17)#24] +Results [3]: [l_partkey#15, l_suppkey#16, sum(l_quantity#17)#24] + +(49) NativeProject +Output [3]: [(0.5 * sum(l_quantity#17)#24) AS (0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] +Input [3]: [l_partkey#15, l_suppkey#16, sum(l_quantity#17)#24] + +(50) NativeFilter +Input [3]: [(0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] +Condition : isnotnull((0.5 * sum(l_quantity))#25) + +(51) NativeShuffleExchange +Input [3]: [(0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] +Arguments: hashpartitioning(l_partkey#15, l_suppkey#16, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(52) ShuffleQueryStage +Output [3]: [(0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] +Arguments: X + +(53) AQEShuffleRead +Input [3]: [(0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] +Arguments: coalesced + +(54) InputAdapter +Input [3]: [(0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] + +(55) NativeSort +Input [3]: [(0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] +Arguments: [l_partkey#15 ASC NULLS FIRST, l_suppkey#16 ASC NULLS FIRST], false + +(56) SortMergeJoin [codegen id : 1] +Left keys [2]: [ps_partkey#7, ps_suppkey#8] +Right keys [2]: [l_partkey#15, l_suppkey#16] +Join type: Inner +Join condition: (cast(ps_availqty#9 as decimal(24,3)) > (0.5 * sum(l_quantity))#25) + +(57) Project [codegen id : 1] +Output [1]: [ps_suppkey#8] +Input [6]: [#10#7, #11#8, #9#9, (0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] + +(58) ConvertToNative +Input [1]: [ps_suppkey#8] + +(59) NativeShuffleExchange +Input [1]: [ps_suppkey#8] +Arguments: hashpartitioning(ps_suppkey#8, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(60) ShuffleQueryStage +Output [1]: [ps_suppkey#8] +Arguments: X + +(61) AQEShuffleRead +Input [1]: [ps_suppkey#8] +Arguments: coalesced + +(62) InputAdapter +Input [1]: [ps_suppkey#8] + +(63) NativeSort +Input [1]: [ps_suppkey#8] +Arguments: [ps_suppkey#8 ASC NULLS FIRST], false + +(64) NativeSortMergeJoin +Left keys [1]: [s_suppkey#1] +Right keys [1]: [ps_suppkey#8] +Join type: LeftSemi +Join condition: None + +(65) NativeProject +Output [3]: [s_name#2, s_address#3, s_nationkey#4] +Input [4]: [#5#1, #2#2, #3#3, #6#4] + +(66) NativeShuffleExchange +Input [3]: [s_name#2, s_address#3, s_nationkey#4] +Arguments: hashpartitioning(s_nationkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(67) ShuffleQueryStage +Output [3]: [s_name#2, s_address#3, s_nationkey#4] +Arguments: X + +(68) AQEShuffleRead +Input [3]: [s_name#2, s_address#3, s_nationkey#4] +Arguments: coalesced + +(69) InputAdapter +Input [3]: [s_name#2, s_address#3, s_nationkey#4] + +(70) NativeSort +Input [3]: [s_name#2, s_address#3, s_nationkey#4] +Arguments: [s_nationkey#4 ASC NULLS FIRST], false + +(126) Scan parquet +Output [2]: [n_nationkey#26, n_name#27] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(72) InputAdapter +Input [2]: [n_nationkey#26, n_name#27] +Arguments: [#28, #27] + +(73) NativeFilter +Input [2]: [#28#26, #27#27] +Condition : ((isnotnull(n_name#27) AND (n_name#27 = CANADA)) AND isnotnull(n_nationkey#26)) + +(74) NativeProject +Output [1]: [n_nationkey#26] +Input [2]: [#28#26, #27#27] + +(75) NativeShuffleExchange +Input [1]: [n_nationkey#26] +Arguments: hashpartitioning(n_nationkey#26, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(76) ShuffleQueryStage +Output [1]: [n_nationkey#26] +Arguments: X + +(77) AQEShuffleRead +Input [1]: [n_nationkey#26] +Arguments: coalesced + +(78) InputAdapter +Input [1]: [n_nationkey#26] + +(79) NativeSort +Input [1]: [n_nationkey#26] +Arguments: [n_nationkey#26 ASC NULLS FIRST], false + +(80) NativeSortMergeJoin +Left keys [1]: [s_nationkey#4] +Right keys [1]: [n_nationkey#26] +Join type: Inner +Join condition: None + +(81) NativeProject +Output [2]: [s_name#2, s_address#3] +Input [4]: [s_name#2, s_address#3, s_nationkey#4, n_nationkey#26] + +(82) NativeShuffleExchange +Input [2]: [s_name#2, s_address#3] +Arguments: rangepartitioning(s_name#2 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(83) ShuffleQueryStage +Output [2]: [s_name#2, s_address#3] +Arguments: X + +(84) AQEShuffleRead +Input [2]: [s_name#2, s_address#3] +Arguments: local + +(85) InputAdapter +Input [2]: [s_name#2, s_address#3] + +(86) Scan parquet +Output [4]: [s_suppkey#1, s_name#2, s_address#3, s_nationkey#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_nationkey)] +ReadSchema: struct + +(87) Filter +Input [4]: [s_suppkey#1, s_name#2, s_address#3, s_nationkey#4] +Condition : isnotnull(s_nationkey#4) + +(88) Exchange +Input [4]: [s_suppkey#1, s_name#2, s_address#3, s_nationkey#4] +Arguments: hashpartitioning(s_suppkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(89) Sort +Input [4]: [s_suppkey#1, s_name#2, s_address#3, s_nationkey#4] +Arguments: [s_suppkey#1 ASC NULLS FIRST], false, 0 + +(90) Scan parquet +Output [3]: [ps_partkey#7, ps_suppkey#8, ps_availqty#9] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] +ReadSchema: struct + +(91) Filter +Input [3]: [ps_partkey#7, ps_suppkey#8, ps_availqty#9] +Condition : ((isnotnull(ps_availqty#9) AND isnotnull(ps_partkey#7)) AND isnotnull(ps_suppkey#8)) + +(92) Exchange +Input [3]: [ps_partkey#7, ps_suppkey#8, ps_availqty#9] +Arguments: hashpartitioning(ps_partkey#7, 5), ENSURE_REQUIREMENTS, [plan_id=12] + +(93) Sort +Input [3]: [ps_partkey#7, ps_suppkey#8, ps_availqty#9] +Arguments: [ps_partkey#7 ASC NULLS FIRST], false, 0 + +(94) Scan parquet +Output [2]: [p_partkey#12, p_name#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(95) Filter +Input [2]: [p_partkey#12, p_name#13] +Condition : (isnotnull(p_name#13) AND StartsWith(p_name#13, forest)) + +(96) Project +Output [1]: [p_partkey#12] +Input [2]: [p_partkey#12, p_name#13] + +(97) Exchange +Input [1]: [p_partkey#12] +Arguments: hashpartitioning(p_partkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=13] + +(98) Sort +Input [1]: [p_partkey#12] +Arguments: [p_partkey#12 ASC NULLS FIRST], false, 0 + +(99) SortMergeJoin +Left keys [1]: [ps_partkey#7] +Right keys [1]: [p_partkey#12] +Join type: LeftSemi +Join condition: None + +(100) Exchange +Input [3]: [ps_partkey#7, ps_suppkey#8, ps_availqty#9] +Arguments: hashpartitioning(ps_partkey#7, ps_suppkey#8, 5), ENSURE_REQUIREMENTS, [plan_id=14] + +(101) Sort +Input [3]: [ps_partkey#7, ps_suppkey#8, ps_availqty#9] +Arguments: [ps_partkey#7 ASC NULLS FIRST, ps_suppkey#8 ASC NULLS FIRST], false, 0 + +(102) Scan parquet +Output [4]: [l_partkey#15, l_suppkey#16, l_quantity#17, l_shipdate#18] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(103) Filter +Input [4]: [l_partkey#15, l_suppkey#16, l_quantity#17, l_shipdate#18] +Condition : ((((isnotnull(l_shipdate#18) AND (l_shipdate#18 >= 1994-01-01)) AND (l_shipdate#18 < 1995-01-01)) AND isnotnull(l_partkey#15)) AND isnotnull(l_suppkey#16)) + +(104) Project +Output [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] +Input [4]: [l_partkey#15, l_suppkey#16, l_quantity#17, l_shipdate#18] + +(105) Exchange +Input [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] +Arguments: hashpartitioning(l_partkey#15, 5), ENSURE_REQUIREMENTS, [plan_id=15] + +(106) Sort +Input [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] +Arguments: [l_partkey#15 ASC NULLS FIRST], false, 0 + +(107) Scan parquet +Output [2]: [p_partkey#12, p_name#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] +ReadSchema: struct + +(108) Filter +Input [2]: [p_partkey#12, p_name#13] +Condition : (isnotnull(p_name#13) AND StartsWith(p_name#13, forest)) + +(109) Project +Output [1]: [p_partkey#12] +Input [2]: [p_partkey#12, p_name#13] + +(110) Exchange +Input [1]: [p_partkey#12] +Arguments: hashpartitioning(p_partkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=16] + +(111) Sort +Input [1]: [p_partkey#12] +Arguments: [p_partkey#12 ASC NULLS FIRST], false, 0 + +(112) SortMergeJoin +Left keys [1]: [l_partkey#15] +Right keys [1]: [p_partkey#12] +Join type: LeftSemi +Join condition: None + +(113) HashAggregate +Input [3]: [l_partkey#15, l_suppkey#16, l_quantity#17] +Keys [2]: [l_partkey#15, l_suppkey#16] +Functions [1]: [partial_sum(l_quantity#17)] +Aggregate Attributes [2]: [sum#21, isEmpty#22] +Results [4]: [l_partkey#15, l_suppkey#16, sum#29, isEmpty#30] + +(114) HashAggregate +Input [4]: [l_partkey#15, l_suppkey#16, sum#29, isEmpty#30] +Keys [2]: [l_partkey#15, l_suppkey#16] +Functions [1]: [sum(l_quantity#17)] +Aggregate Attributes [1]: [sum(l_quantity#17)#24] +Results [3]: [(0.5 * sum(l_quantity#17)#24) AS (0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] + +(115) Filter +Input [3]: [(0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] +Condition : isnotnull((0.5 * sum(l_quantity))#25) + +(116) Exchange +Input [3]: [(0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] +Arguments: hashpartitioning(l_partkey#15, l_suppkey#16, 5), ENSURE_REQUIREMENTS, [plan_id=17] + +(117) Sort +Input [3]: [(0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] +Arguments: [l_partkey#15 ASC NULLS FIRST, l_suppkey#16 ASC NULLS FIRST], false, 0 + +(118) SortMergeJoin +Left keys [2]: [ps_partkey#7, ps_suppkey#8] +Right keys [2]: [l_partkey#15, l_suppkey#16] +Join type: Inner +Join condition: (cast(ps_availqty#9 as decimal(24,3)) > (0.5 * sum(l_quantity))#25) + +(119) Project +Output [1]: [ps_suppkey#8] +Input [6]: [ps_partkey#7, ps_suppkey#8, ps_availqty#9, (0.5 * sum(l_quantity))#25, l_partkey#15, l_suppkey#16] + +(120) Exchange +Input [1]: [ps_suppkey#8] +Arguments: hashpartitioning(ps_suppkey#8, 5), ENSURE_REQUIREMENTS, [plan_id=18] + +(121) Sort +Input [1]: [ps_suppkey#8] +Arguments: [ps_suppkey#8 ASC NULLS FIRST], false, 0 + +(122) SortMergeJoin +Left keys [1]: [s_suppkey#1] +Right keys [1]: [ps_suppkey#8] +Join type: LeftSemi +Join condition: None + +(123) Project +Output [3]: [s_name#2, s_address#3, s_nationkey#4] +Input [4]: [s_suppkey#1, s_name#2, s_address#3, s_nationkey#4] + +(124) Exchange +Input [3]: [s_name#2, s_address#3, s_nationkey#4] +Arguments: hashpartitioning(s_nationkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=19] + +(125) Sort +Input [3]: [s_name#2, s_address#3, s_nationkey#4] +Arguments: [s_nationkey#4 ASC NULLS FIRST], false, 0 + +(126) Scan parquet +Output [2]: [n_nationkey#26, n_name#27] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(127) Filter +Input [2]: [n_nationkey#26, n_name#27] +Condition : ((isnotnull(n_name#27) AND (n_name#27 = CANADA)) AND isnotnull(n_nationkey#26)) + +(128) Project +Output [1]: [n_nationkey#26] +Input [2]: [n_nationkey#26, n_name#27] + +(129) Exchange +Input [1]: [n_nationkey#26] +Arguments: hashpartitioning(n_nationkey#26, 5), ENSURE_REQUIREMENTS, [plan_id=20] + +(130) Sort +Input [1]: [n_nationkey#26] +Arguments: [n_nationkey#26 ASC NULLS FIRST], false, 0 + +(131) SortMergeJoin +Left keys [1]: [s_nationkey#4] +Right keys [1]: [n_nationkey#26] +Join type: Inner +Join condition: None + +(132) Project +Output [2]: [s_name#2, s_address#3] +Input [4]: [s_name#2, s_address#3, s_nationkey#4, n_nationkey#26] + +(133) Exchange +Input [2]: [s_name#2, s_address#3] +Arguments: rangepartitioning(s_name#2 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=21] + +(134) Sort +Input [2]: [s_name#2, s_address#3] +Arguments: [s_name#2 ASC NULLS FIRST], true, 0 + +(135) AdaptiveSparkPlan +Output [2]: [s_name#2, s_address#3] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q21.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q21.txt new file mode 100644 index 000000000..236bbfef3 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q21.txt @@ -0,0 +1,700 @@ +== Physical Plan == +AdaptiveSparkPlan (128) ++- == Final Plan == + NativeProject (82) + +- NativeHashAggregate (81) + +- InputAdapter (80) + +- AQEShuffleRead (79) + +- ShuffleQueryStage (78), Statistics(sizeInBytes=696.0 B, rowCount=1) + +- NativeShuffleExchange (77) + +- NativeHashAggregate (76) + +- NativeProject (75) + +- NativeSortMergeJoin Inner (74) + :- NativeSort (64) + : +- InputAdapter (63) + : +- AQEShuffleRead (62) + : +- ShuffleQueryStage (61), Statistics(sizeInBytes=31.4 KiB, rowCount=1.06E+3) + : +- NativeShuffleExchange (60) + : +- NativeProject (59) + : +- NativeSortMergeJoin Inner (58) + : :- NativeSort (48) + : : +- InputAdapter (47) + : : +- AQEShuffleRead (46) + : : +- ShuffleQueryStage (45), Statistics(sizeInBytes=79.4 KiB, rowCount=2.12E+3) + : : +- NativeShuffleExchange (44) + : : +- NativeProject (43) + : : +- NativeSortMergeJoin Inner (42) + : : :- NativeSort (8) + : : : +- InputAdapter (7) + : : : +- AQEShuffleRead (6) + : : : +- ShuffleQueryStage (5), Statistics(sizeInBytes=4.7 KiB, rowCount=100) + : : : +- NativeShuffleExchange (4) + : : : +- NativeFilter (3) + : : : +- InputAdapter (2) + : : : +- NativeParquetScan (1) + : : +- NativeSort (41) + : : +- InputAdapter (40) + : : +- AQEShuffleRead (39) + : : +- ShuffleQueryStage (38), Statistics(sizeInBytes=33.6 KiB, rowCount=2.12E+3) + : : +- NativeShuffleExchange (37) + : : +- ConvertToNative (36) + : : +- * SortMergeJoin LeftAnti (35) + : : :- * SortMergeJoin LeftSemi (25) + : : : :- NativeSort (17) + : : : : +- InputAdapter (16) + : : : : +- AQEShuffleRead (15) + : : : : +- ShuffleQueryStage (14), Statistics(sizeInBytes=594.8 KiB, rowCount=3.79E+4) + : : : : +- NativeShuffleExchange (13) + : : : : +- NativeProject (12) + : : : : +- NativeFilter (11) + : : : : +- InputAdapter (10) + : : : : +- NativeParquetScan (9) + : : : +- NativeSort (24) + : : : +- InputAdapter (23) + : : : +- AQEShuffleRead (22) + : : : +- ShuffleQueryStage (21), Statistics(sizeInBytes=942.9 KiB, rowCount=6.02E+4) + : : : +- NativeShuffleExchange (20) + : : : +- InputAdapter (19) + : : : +- NativeParquetScan (18) + : : +- NativeSort (34) + : : +- InputAdapter (33) + : : +- AQEShuffleRead (32) + : : +- ShuffleQueryStage (31), Statistics(sizeInBytes=594.8 KiB, rowCount=3.79E+4) + : : +- NativeShuffleExchange (30) + : : +- NativeProject (29) + : : +- NativeFilter (28) + : : +- InputAdapter (27) + : : +- NativeParquetScan (26) + : +- NativeSort (57) + : +- InputAdapter (56) + : +- AQEShuffleRead (55) + : +- ShuffleQueryStage (54), Statistics(sizeInBytes=57.5 KiB, rowCount=7.30E+3) + : +- NativeShuffleExchange (53) + : +- NativeProject (52) + : +- NativeFilter (51) + : +- InputAdapter (50) + : +- NativeParquetScan (49) + +- NativeSort (73) + +- InputAdapter (72) + +- AQEShuffleRead (71) + +- ShuffleQueryStage (70), Statistics(sizeInBytes=216.0 B, rowCount=1) + +- NativeShuffleExchange (69) + +- NativeProject (68) + +- NativeFilter (67) + +- InputAdapter (66) + +- NativeParquetScan (65) ++- == Initial Plan == + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- SortMergeJoin Inner (122) + :- Sort (116) + : +- Exchange (115) + : +- Project (114) + : +- SortMergeJoin Inner (113) + : :- Sort (107) + : : +- Exchange (106) + : : +- Project (105) + : : +- SortMergeJoin Inner (104) + : : :- Sort (86) + : : : +- Exchange (85) + : : : +- Filter (84) + : : : +- Scan parquet (83) + : : +- Sort (103) + : : +- Exchange (102) + : : +- SortMergeJoin LeftAnti (101) + : : :- SortMergeJoin LeftSemi (95) + : : : :- Sort (91) + : : : : +- Exchange (90) + : : : : +- Project (89) + : : : : +- Filter (88) + : : : : +- Scan parquet (87) + : : : +- Sort (94) + : : : +- Exchange (93) + : : : +- Scan parquet (92) + : : +- Sort (100) + : : +- Exchange (99) + : : +- Project (98) + : : +- Filter (97) + : : +- Scan parquet (96) + : +- Sort (112) + : +- Exchange (111) + : +- Project (110) + : +- Filter (109) + : +- Scan parquet (108) + +- Sort (121) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) + + +(83) Scan parquet +Output [3]: [s_suppkey#1, s_name#2, s_nationkey#3] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) InputAdapter +Input [3]: [s_suppkey#1, s_name#2, s_nationkey#3] +Arguments: [#4, #2, #5] + +(3) NativeFilter +Input [3]: [#4#1, #2#2, #5#3] +Condition : (isnotnull(s_suppkey#1) AND isnotnull(s_nationkey#3)) + +(4) NativeShuffleExchange +Input [3]: [#4#1, #2#2, #5#3] +Arguments: hashpartitioning(s_suppkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [3]: [#4#1, #2#2, #5#3] +Arguments: X + +(6) AQEShuffleRead +Input [3]: [#4#1, #2#2, #5#3] +Arguments: coalesced + +(7) InputAdapter +Input [3]: [#4#1, #2#2, #5#3] + +(8) NativeSort +Input [3]: [#4#1, #2#2, #5#3] +Arguments: [s_suppkey#1 ASC NULLS FIRST], false + +(87) Scan parquet +Output [4]: [l_orderkey#6, l_suppkey#7, l_commitdate#8, l_receiptdate#9] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(10) InputAdapter +Input [4]: [l_orderkey#6, l_suppkey#7, l_commitdate#8, l_receiptdate#9] +Arguments: [#10, #11, #8, #9] + +(11) NativeFilter +Input [4]: [#10#6, #11#7, #8#8, #9#9] +Condition : ((((isnotnull(l_receiptdate#9) AND isnotnull(l_commitdate#8)) AND (l_receiptdate#9 > l_commitdate#8)) AND isnotnull(l_suppkey#7)) AND isnotnull(l_orderkey#6)) + +(12) NativeProject +Output [2]: [l_orderkey#6, l_suppkey#7] +Input [4]: [#10#6, #11#7, #8#8, #9#9] + +(13) NativeShuffleExchange +Input [2]: [l_orderkey#6, l_suppkey#7] +Arguments: hashpartitioning(l_orderkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [2]: [l_orderkey#6, l_suppkey#7] +Arguments: X + +(15) AQEShuffleRead +Input [2]: [l_orderkey#6, l_suppkey#7] +Arguments: coalesced + +(16) InputAdapter +Input [2]: [l_orderkey#6, l_suppkey#7] + +(17) NativeSort +Input [2]: [l_orderkey#6, l_suppkey#7] +Arguments: [l_orderkey#6 ASC NULLS FIRST], false + +(92) Scan parquet +Output [2]: [l_orderkey#12, l_suppkey#13] +Batched: true +Location: InMemoryFileIndex [file:/] +ReadSchema: struct + +(19) InputAdapter +Input [2]: [l_orderkey#12, l_suppkey#13] +Arguments: [#14, #15] + +(20) NativeShuffleExchange +Input [2]: [#14#12, #15#13] +Arguments: hashpartitioning(l_orderkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(21) ShuffleQueryStage +Output [2]: [#14#12, #15#13] +Arguments: X + +(22) AQEShuffleRead +Input [2]: [#14#12, #15#13] +Arguments: coalesced + +(23) InputAdapter +Input [2]: [#14#12, #15#13] + +(24) NativeSort +Input [2]: [#14#12, #15#13] +Arguments: [l_orderkey#12 ASC NULLS FIRST], false + +(25) SortMergeJoin [codegen id : 1] +Left keys [1]: [l_orderkey#6] +Right keys [1]: [l_orderkey#12] +Join type: LeftSemi +Join condition: NOT (l_suppkey#13 = l_suppkey#7) + +(96) Scan parquet +Output [4]: [l_orderkey#16, l_suppkey#17, l_commitdate#18, l_receiptdate#19] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] +ReadSchema: struct + +(27) InputAdapter +Input [4]: [l_orderkey#16, l_suppkey#17, l_commitdate#18, l_receiptdate#19] +Arguments: [#20, #21, #18, #19] + +(28) NativeFilter +Input [4]: [#20#16, #21#17, #18#18, #19#19] +Condition : ((isnotnull(l_receiptdate#19) AND isnotnull(l_commitdate#18)) AND (l_receiptdate#19 > l_commitdate#18)) + +(29) NativeProject +Output [2]: [l_orderkey#16, l_suppkey#17] +Input [4]: [#20#16, #21#17, #18#18, #19#19] + +(30) NativeShuffleExchange +Input [2]: [l_orderkey#16, l_suppkey#17] +Arguments: hashpartitioning(l_orderkey#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(31) ShuffleQueryStage +Output [2]: [l_orderkey#16, l_suppkey#17] +Arguments: X + +(32) AQEShuffleRead +Input [2]: [l_orderkey#16, l_suppkey#17] +Arguments: coalesced + +(33) InputAdapter +Input [2]: [l_orderkey#16, l_suppkey#17] + +(34) NativeSort +Input [2]: [l_orderkey#16, l_suppkey#17] +Arguments: [l_orderkey#16 ASC NULLS FIRST], false + +(35) SortMergeJoin [codegen id : 2] +Left keys [1]: [l_orderkey#6] +Right keys [1]: [l_orderkey#16] +Join type: LeftAnti +Join condition: NOT (l_suppkey#17 = l_suppkey#7) + +(36) ConvertToNative +Input [2]: [l_orderkey#6, l_suppkey#7] + +(37) NativeShuffleExchange +Input [2]: [l_orderkey#6, l_suppkey#7] +Arguments: hashpartitioning(l_suppkey#7, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(38) ShuffleQueryStage +Output [2]: [l_orderkey#6, l_suppkey#7] +Arguments: X + +(39) AQEShuffleRead +Input [2]: [l_orderkey#6, l_suppkey#7] +Arguments: coalesced + +(40) InputAdapter +Input [2]: [l_orderkey#6, l_suppkey#7] + +(41) NativeSort +Input [2]: [l_orderkey#6, l_suppkey#7] +Arguments: [l_suppkey#7 ASC NULLS FIRST], false + +(42) NativeSortMergeJoin +Left keys [1]: [s_suppkey#1] +Right keys [1]: [l_suppkey#7] +Join type: Inner +Join condition: None + +(43) NativeProject +Output [3]: [s_name#2, s_nationkey#3, l_orderkey#6] +Input [5]: [#4#1, #2#2, #5#3, l_orderkey#6, l_suppkey#7] + +(44) NativeShuffleExchange +Input [3]: [s_name#2, s_nationkey#3, l_orderkey#6] +Arguments: hashpartitioning(l_orderkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(45) ShuffleQueryStage +Output [3]: [s_name#2, s_nationkey#3, l_orderkey#6] +Arguments: X + +(46) AQEShuffleRead +Input [3]: [s_name#2, s_nationkey#3, l_orderkey#6] +Arguments: coalesced + +(47) InputAdapter +Input [3]: [s_name#2, s_nationkey#3, l_orderkey#6] + +(48) NativeSort +Input [3]: [s_name#2, s_nationkey#3, l_orderkey#6] +Arguments: [l_orderkey#6 ASC NULLS FIRST], false + +(108) Scan parquet +Output [2]: [o_orderkey#22, o_orderstatus#23] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] +ReadSchema: struct + +(50) InputAdapter +Input [2]: [o_orderkey#22, o_orderstatus#23] +Arguments: [#24, #23] + +(51) NativeFilter +Input [2]: [#24#22, #23#23] +Condition : ((isnotnull(o_orderstatus#23) AND (o_orderstatus#23 = F)) AND isnotnull(o_orderkey#22)) + +(52) NativeProject +Output [1]: [o_orderkey#22] +Input [2]: [#24#22, #23#23] + +(53) NativeShuffleExchange +Input [1]: [o_orderkey#22] +Arguments: hashpartitioning(o_orderkey#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(54) ShuffleQueryStage +Output [1]: [o_orderkey#22] +Arguments: X + +(55) AQEShuffleRead +Input [1]: [o_orderkey#22] +Arguments: coalesced + +(56) InputAdapter +Input [1]: [o_orderkey#22] + +(57) NativeSort +Input [1]: [o_orderkey#22] +Arguments: [o_orderkey#22 ASC NULLS FIRST], false + +(58) NativeSortMergeJoin +Left keys [1]: [l_orderkey#6] +Right keys [1]: [o_orderkey#22] +Join type: Inner +Join condition: None + +(59) NativeProject +Output [2]: [s_name#2, s_nationkey#3] +Input [4]: [s_name#2, s_nationkey#3, l_orderkey#6, o_orderkey#22] + +(60) NativeShuffleExchange +Input [2]: [s_name#2, s_nationkey#3] +Arguments: hashpartitioning(s_nationkey#3, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(61) ShuffleQueryStage +Output [2]: [s_name#2, s_nationkey#3] +Arguments: X + +(62) AQEShuffleRead +Input [2]: [s_name#2, s_nationkey#3] +Arguments: coalesced + +(63) InputAdapter +Input [2]: [s_name#2, s_nationkey#3] + +(64) NativeSort +Input [2]: [s_name#2, s_nationkey#3] +Arguments: [s_nationkey#3 ASC NULLS FIRST], false + +(117) Scan parquet +Output [2]: [n_nationkey#25, n_name#26] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(66) InputAdapter +Input [2]: [n_nationkey#25, n_name#26] +Arguments: [#27, #26] + +(67) NativeFilter +Input [2]: [#27#25, #26#26] +Condition : ((isnotnull(n_name#26) AND (n_name#26 = SAUDI ARABIA)) AND isnotnull(n_nationkey#25)) + +(68) NativeProject +Output [1]: [n_nationkey#25] +Input [2]: [#27#25, #26#26] + +(69) NativeShuffleExchange +Input [1]: [n_nationkey#25] +Arguments: hashpartitioning(n_nationkey#25, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(70) ShuffleQueryStage +Output [1]: [n_nationkey#25] +Arguments: X + +(71) AQEShuffleRead +Input [1]: [n_nationkey#25] +Arguments: coalesced + +(72) InputAdapter +Input [1]: [n_nationkey#25] + +(73) NativeSort +Input [1]: [n_nationkey#25] +Arguments: [n_nationkey#25 ASC NULLS FIRST], false + +(74) NativeSortMergeJoin +Left keys [1]: [s_nationkey#3] +Right keys [1]: [n_nationkey#25] +Join type: Inner +Join condition: None + +(75) NativeProject +Output [1]: [s_name#2] +Input [3]: [s_name#2, s_nationkey#3, n_nationkey#25] + +(76) NativeHashAggregate +Input [1]: [s_name#2] +Keys [1]: [s_name#2] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#28] +Results [2]: [s_name#2, #29#29] + +(77) NativeShuffleExchange +Input [2]: [s_name#2, #29#29] +Arguments: hashpartitioning(s_name#2, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(78) ShuffleQueryStage +Output [2]: [s_name#2, #29#29] +Arguments: X + +(79) AQEShuffleRead +Input [2]: [s_name#2, #29#29] +Arguments: coalesced + +(80) InputAdapter +Input [2]: [s_name#2, #29#29] + +(81) NativeHashAggregate +Input [2]: [s_name#2, #29#29] +Keys [1]: [s_name#2] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#30] +Results [2]: [s_name#2, count(1)#30] + +(82) NativeProject +Output [2]: [s_name#2, count(1)#30 AS numwait#31] +Input [2]: [s_name#2, count(1)#30] + +(83) Scan parquet +Output [3]: [s_suppkey#1, s_name#2, s_nationkey#3] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(84) Filter +Input [3]: [s_suppkey#1, s_name#2, s_nationkey#3] +Condition : (isnotnull(s_suppkey#1) AND isnotnull(s_nationkey#3)) + +(85) Exchange +Input [3]: [s_suppkey#1, s_name#2, s_nationkey#3] +Arguments: hashpartitioning(s_suppkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(86) Sort +Input [3]: [s_suppkey#1, s_name#2, s_nationkey#3] +Arguments: [s_suppkey#1 ASC NULLS FIRST], false, 0 + +(87) Scan parquet +Output [4]: [l_orderkey#6, l_suppkey#7, l_commitdate#8, l_receiptdate#9] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(88) Filter +Input [4]: [l_orderkey#6, l_suppkey#7, l_commitdate#8, l_receiptdate#9] +Condition : ((((isnotnull(l_receiptdate#9) AND isnotnull(l_commitdate#8)) AND (l_receiptdate#9 > l_commitdate#8)) AND isnotnull(l_suppkey#7)) AND isnotnull(l_orderkey#6)) + +(89) Project +Output [2]: [l_orderkey#6, l_suppkey#7] +Input [4]: [l_orderkey#6, l_suppkey#7, l_commitdate#8, l_receiptdate#9] + +(90) Exchange +Input [2]: [l_orderkey#6, l_suppkey#7] +Arguments: hashpartitioning(l_orderkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=12] + +(91) Sort +Input [2]: [l_orderkey#6, l_suppkey#7] +Arguments: [l_orderkey#6 ASC NULLS FIRST], false, 0 + +(92) Scan parquet +Output [2]: [l_orderkey#12, l_suppkey#13] +Batched: true +Location: InMemoryFileIndex [file:/] +ReadSchema: struct + +(93) Exchange +Input [2]: [l_orderkey#12, l_suppkey#13] +Arguments: hashpartitioning(l_orderkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=13] + +(94) Sort +Input [2]: [l_orderkey#12, l_suppkey#13] +Arguments: [l_orderkey#12 ASC NULLS FIRST], false, 0 + +(95) SortMergeJoin +Left keys [1]: [l_orderkey#6] +Right keys [1]: [l_orderkey#12] +Join type: LeftSemi +Join condition: NOT (l_suppkey#13 = l_suppkey#7) + +(96) Scan parquet +Output [4]: [l_orderkey#16, l_suppkey#17, l_commitdate#18, l_receiptdate#19] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] +ReadSchema: struct + +(97) Filter +Input [4]: [l_orderkey#16, l_suppkey#17, l_commitdate#18, l_receiptdate#19] +Condition : ((isnotnull(l_receiptdate#19) AND isnotnull(l_commitdate#18)) AND (l_receiptdate#19 > l_commitdate#18)) + +(98) Project +Output [2]: [l_orderkey#16, l_suppkey#17] +Input [4]: [l_orderkey#16, l_suppkey#17, l_commitdate#18, l_receiptdate#19] + +(99) Exchange +Input [2]: [l_orderkey#16, l_suppkey#17] +Arguments: hashpartitioning(l_orderkey#16, 5), ENSURE_REQUIREMENTS, [plan_id=14] + +(100) Sort +Input [2]: [l_orderkey#16, l_suppkey#17] +Arguments: [l_orderkey#16 ASC NULLS FIRST], false, 0 + +(101) SortMergeJoin +Left keys [1]: [l_orderkey#6] +Right keys [1]: [l_orderkey#16] +Join type: LeftAnti +Join condition: NOT (l_suppkey#17 = l_suppkey#7) + +(102) Exchange +Input [2]: [l_orderkey#6, l_suppkey#7] +Arguments: hashpartitioning(l_suppkey#7, 5), ENSURE_REQUIREMENTS, [plan_id=15] + +(103) Sort +Input [2]: [l_orderkey#6, l_suppkey#7] +Arguments: [l_suppkey#7 ASC NULLS FIRST], false, 0 + +(104) SortMergeJoin +Left keys [1]: [s_suppkey#1] +Right keys [1]: [l_suppkey#7] +Join type: Inner +Join condition: None + +(105) Project +Output [3]: [s_name#2, s_nationkey#3, l_orderkey#6] +Input [5]: [s_suppkey#1, s_name#2, s_nationkey#3, l_orderkey#6, l_suppkey#7] + +(106) Exchange +Input [3]: [s_name#2, s_nationkey#3, l_orderkey#6] +Arguments: hashpartitioning(l_orderkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=16] + +(107) Sort +Input [3]: [s_name#2, s_nationkey#3, l_orderkey#6] +Arguments: [l_orderkey#6 ASC NULLS FIRST], false, 0 + +(108) Scan parquet +Output [2]: [o_orderkey#22, o_orderstatus#23] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] +ReadSchema: struct + +(109) Filter +Input [2]: [o_orderkey#22, o_orderstatus#23] +Condition : ((isnotnull(o_orderstatus#23) AND (o_orderstatus#23 = F)) AND isnotnull(o_orderkey#22)) + +(110) Project +Output [1]: [o_orderkey#22] +Input [2]: [o_orderkey#22, o_orderstatus#23] + +(111) Exchange +Input [1]: [o_orderkey#22] +Arguments: hashpartitioning(o_orderkey#22, 5), ENSURE_REQUIREMENTS, [plan_id=17] + +(112) Sort +Input [1]: [o_orderkey#22] +Arguments: [o_orderkey#22 ASC NULLS FIRST], false, 0 + +(113) SortMergeJoin +Left keys [1]: [l_orderkey#6] +Right keys [1]: [o_orderkey#22] +Join type: Inner +Join condition: None + +(114) Project +Output [2]: [s_name#2, s_nationkey#3] +Input [4]: [s_name#2, s_nationkey#3, l_orderkey#6, o_orderkey#22] + +(115) Exchange +Input [2]: [s_name#2, s_nationkey#3] +Arguments: hashpartitioning(s_nationkey#3, 5), ENSURE_REQUIREMENTS, [plan_id=18] + +(116) Sort +Input [2]: [s_name#2, s_nationkey#3] +Arguments: [s_nationkey#3 ASC NULLS FIRST], false, 0 + +(117) Scan parquet +Output [2]: [n_nationkey#25, n_name#26] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] +ReadSchema: struct + +(118) Filter +Input [2]: [n_nationkey#25, n_name#26] +Condition : ((isnotnull(n_name#26) AND (n_name#26 = SAUDI ARABIA)) AND isnotnull(n_nationkey#25)) + +(119) Project +Output [1]: [n_nationkey#25] +Input [2]: [n_nationkey#25, n_name#26] + +(120) Exchange +Input [1]: [n_nationkey#25] +Arguments: hashpartitioning(n_nationkey#25, 5), ENSURE_REQUIREMENTS, [plan_id=19] + +(121) Sort +Input [1]: [n_nationkey#25] +Arguments: [n_nationkey#25 ASC NULLS FIRST], false, 0 + +(122) SortMergeJoin +Left keys [1]: [s_nationkey#3] +Right keys [1]: [n_nationkey#25] +Join type: Inner +Join condition: None + +(123) Project +Output [1]: [s_name#2] +Input [3]: [s_name#2, s_nationkey#3, n_nationkey#25] + +(124) HashAggregate +Input [1]: [s_name#2] +Keys [1]: [s_name#2] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#28] +Results [2]: [s_name#2, count#32] + +(125) Exchange +Input [2]: [s_name#2, count#32] +Arguments: hashpartitioning(s_name#2, 5), ENSURE_REQUIREMENTS, [plan_id=20] + +(126) HashAggregate +Input [2]: [s_name#2, count#32] +Keys [1]: [s_name#2] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#30] +Results [2]: [s_name#2, count(1)#30 AS numwait#31] + +(127) TakeOrderedAndProject +Input [2]: [s_name#2, numwait#31] +Arguments: X, [numwait#31 DESC NULLS LAST, s_name#2 ASC NULLS FIRST], [s_name#2, numwait#31] + +(128) AdaptiveSparkPlan +Output [2]: [s_name#2, numwait#31] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q22.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q22.txt new file mode 100644 index 000000000..fa3c4b9d1 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q22.txt @@ -0,0 +1,353 @@ +== Physical Plan == +AdaptiveSparkPlan (44) ++- == Final Plan == + NativeSort (29) + +- InputAdapter (28) + +- AQEShuffleRead (27) + +- ShuffleQueryStage (26), Statistics(sizeInBytes=896.0 B, rowCount=7) + +- NativeShuffleExchange (25) + +- NativeProject (24) + +- NativeHashAggregate (23) + +- InputAdapter (22) + +- AQEShuffleRead (21) + +- ShuffleQueryStage (20), Statistics(sizeInBytes=760.0 B, rowCount=7) + +- NativeShuffleExchange (19) + +- NativeHashAggregate (18) + +- NativeProject (17) + +- NativeSortMergeJoin LeftAnti (16) + :- NativeSort (8) + : +- InputAdapter (7) + : +- AQEShuffleRead (6) + : +- ShuffleQueryStage (5), Statistics(sizeInBytes=8.6 KiB, rowCount=190) + : +- NativeShuffleExchange (4) + : +- NativeFilter (3) + : +- InputAdapter (2) + : +- NativeParquetScan (1) + +- NativeSort (15) + +- InputAdapter (14) + +- AQEShuffleRead (13) + +- ShuffleQueryStage (12), Statistics(sizeInBytes=117.6 KiB, rowCount=1.50E+4) + +- NativeShuffleExchange (11) + +- InputAdapter (10) + +- NativeParquetScan (9) ++- == Initial Plan == + Sort (43) + +- Exchange (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- SortMergeJoin LeftAnti (37) + :- Sort (33) + : +- Exchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Sort (36) + +- Exchange (35) + +- Scan parquet (34) + + +(30) Scan parquet +Output [3]: [c_custkey#1, c_phone#2, c_acctbal#3] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_acctbal)] +ReadSchema: struct + +(2) InputAdapter +Input [3]: [c_custkey#1, c_phone#2, c_acctbal#3] +Arguments: [#4, #2, #3] + +(3) NativeFilter +Input [3]: [#4#1, #2#2, #3#3] +Condition : ((isnotnull(c_acctbal#3) AND substring(c_phone#2, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#3 as decimal(16,6)) > Subquery subquery#5, [id=#6])) + +(4) NativeShuffleExchange +Input [3]: [#4#1, #2#2, #3#3] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [3]: [#4#1, #2#2, #3#3] +Arguments: X + +(6) AQEShuffleRead +Input [3]: [#4#1, #2#2, #3#3] +Arguments: coalesced + +(7) InputAdapter +Input [3]: [#4#1, #2#2, #3#3] + +(8) NativeSort +Input [3]: [#4#1, #2#2, #3#3] +Arguments: [c_custkey#1 ASC NULLS FIRST], false + +(34) Scan parquet +Output [1]: [o_custkey#7] +Batched: true +Location: InMemoryFileIndex [file:/] +ReadSchema: struct + +(10) InputAdapter +Input [1]: [o_custkey#7] +Arguments: [#8] + +(11) NativeShuffleExchange +Input [1]: [#8#7] +Arguments: hashpartitioning(o_custkey#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(12) ShuffleQueryStage +Output [1]: [#8#7] +Arguments: X + +(13) AQEShuffleRead +Input [1]: [#8#7] +Arguments: coalesced + +(14) InputAdapter +Input [1]: [#8#7] + +(15) NativeSort +Input [1]: [#8#7] +Arguments: [o_custkey#7 ASC NULLS FIRST], false + +(16) NativeSortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#7] +Join type: LeftAnti +Join condition: None + +(17) NativeProject +Output [2]: [substring(c_phone#2, 1, 2) AS cntrycode#9, c_acctbal#3] +Input [3]: [#4#1, #2#2, #3#3] + +(18) NativeHashAggregate +Input [2]: [cntrycode#9, c_acctbal#3] +Keys [1]: [cntrycode#9] +Functions [2]: [partial_count(1), partial_sum(c_acctbal#3)] +Aggregate Attributes [3]: [count#10, sum#11, isEmpty#12] +Results [2]: [cntrycode#9, #13#13] + +(19) NativeShuffleExchange +Input [2]: [cntrycode#9, #13#13] +Arguments: hashpartitioning(cntrycode#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(20) ShuffleQueryStage +Output [2]: [cntrycode#9, #13#13] +Arguments: X + +(21) AQEShuffleRead +Input [2]: [cntrycode#9, #13#13] +Arguments: coalesced + +(22) InputAdapter +Input [2]: [cntrycode#9, #13#13] + +(23) NativeHashAggregate +Input [2]: [cntrycode#9, #13#13] +Keys [1]: [cntrycode#9] +Functions [2]: [count(1), sum(c_acctbal#3)] +Aggregate Attributes [2]: [count(1)#14, sum(c_acctbal#3)#15] +Results [3]: [cntrycode#9, count(1)#14, sum(c_acctbal#3)#15] + +(24) NativeProject +Output [3]: [cntrycode#9, count(1)#14 AS numcust#16, sum(c_acctbal#3)#15 AS totacctbal#17] +Input [3]: [cntrycode#9, count(1)#14, sum(c_acctbal#3)#15] + +(25) NativeShuffleExchange +Input [3]: [cntrycode#9, numcust#16, totacctbal#17] +Arguments: rangepartitioning(cntrycode#9 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(26) ShuffleQueryStage +Output [3]: [cntrycode#9, numcust#16, totacctbal#17] +Arguments: X + +(27) AQEShuffleRead +Input [3]: [cntrycode#9, numcust#16, totacctbal#17] +Arguments: coalesced + +(28) InputAdapter +Input [3]: [cntrycode#9, numcust#16, totacctbal#17] + +(29) NativeSort +Input [3]: [cntrycode#9, numcust#16, totacctbal#17] +Arguments: [cntrycode#9 ASC NULLS FIRST], true + +(30) Scan parquet +Output [3]: [c_custkey#1, c_phone#2, c_acctbal#3] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_acctbal)] +ReadSchema: struct + +(31) Filter +Input [3]: [c_custkey#1, c_phone#2, c_acctbal#3] +Condition : ((isnotnull(c_acctbal#3) AND substring(c_phone#2, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#3 as decimal(16,6)) > Subquery subquery#5, [id=#6])) + +(32) Exchange +Input [3]: [c_custkey#1, c_phone#2, c_acctbal#3] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(33) Sort +Input [3]: [c_custkey#1, c_phone#2, c_acctbal#3] +Arguments: [c_custkey#1 ASC NULLS FIRST], false, 0 + +(34) Scan parquet +Output [1]: [o_custkey#7] +Batched: true +Location: InMemoryFileIndex [file:/] +ReadSchema: struct + +(35) Exchange +Input [1]: [o_custkey#7] +Arguments: hashpartitioning(o_custkey#7, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(36) Sort +Input [1]: [o_custkey#7] +Arguments: [o_custkey#7 ASC NULLS FIRST], false, 0 + +(37) SortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#7] +Join type: LeftAnti +Join condition: None + +(38) Project +Output [2]: [substring(c_phone#2, 1, 2) AS cntrycode#9, c_acctbal#3] +Input [3]: [c_custkey#1, c_phone#2, c_acctbal#3] + +(39) HashAggregate +Input [2]: [cntrycode#9, c_acctbal#3] +Keys [1]: [cntrycode#9] +Functions [2]: [partial_count(1), partial_sum(c_acctbal#3)] +Aggregate Attributes [3]: [count#10, sum#11, isEmpty#12] +Results [4]: [cntrycode#9, count#18, sum#19, isEmpty#20] + +(40) Exchange +Input [4]: [cntrycode#9, count#18, sum#19, isEmpty#20] +Arguments: hashpartitioning(cntrycode#9, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(41) HashAggregate +Input [4]: [cntrycode#9, count#18, sum#19, isEmpty#20] +Keys [1]: [cntrycode#9] +Functions [2]: [count(1), sum(c_acctbal#3)] +Aggregate Attributes [2]: [count(1)#14, sum(c_acctbal#3)#15] +Results [3]: [cntrycode#9, count(1)#14 AS numcust#16, sum(c_acctbal#3)#15 AS totacctbal#17] + +(42) Exchange +Input [3]: [cntrycode#9, numcust#16, totacctbal#17] +Arguments: rangepartitioning(cntrycode#9 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(43) Sort +Input [3]: [cntrycode#9, numcust#16, totacctbal#17] +Arguments: [cntrycode#9 ASC NULLS FIRST], true, 0 + +(44) AdaptiveSparkPlan +Output [3]: [cntrycode#9, numcust#16, totacctbal#17] +Arguments: isFinalPlan=true + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery subquery#5, [id=#6] +AdaptiveSparkPlan (60) ++- == Final Plan == + NativeProject (53) + +- NativeHashAggregate (52) + +- ShuffleQueryStage (51), Statistics(sizeInBytes=360.0 B, rowCount=1) + +- NativeShuffleExchange (50) + +- NativeHashAggregate (49) + +- NativeProject (48) + +- NativeFilter (47) + +- InputAdapter (46) + +- NativeParquetScan (45) ++- == Initial Plan == + HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) + + +(54) Scan parquet +Output [2]: [c_phone#21, c_acctbal#22] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] +ReadSchema: struct + +(46) InputAdapter +Input [2]: [c_phone#21, c_acctbal#22] +Arguments: [#21, #22] + +(47) NativeFilter +Input [2]: [#21#21, #22#22] +Condition : ((isnotnull(c_acctbal#22) AND (c_acctbal#22 > 0.00)) AND substring(c_phone#21, 1, 2) IN (13,31,23,29,30,18,17)) + +(48) NativeProject +Output [1]: [c_acctbal#22] +Input [2]: [#21#21, #22#22] + +(49) NativeHashAggregate +Input [1]: [c_acctbal#22] +Keys: [] +Functions [1]: [partial_avg(c_acctbal#22)] +Aggregate Attributes [2]: [sum#23, count#24] +Results [1]: [#13#13] + +(50) NativeShuffleExchange +Input [1]: [#13#13] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] + +(51) ShuffleQueryStage +Output [1]: [#13#13] +Arguments: X + +(52) NativeHashAggregate +Input [1]: [#13#13] +Keys: [] +Functions [1]: [avg(c_acctbal#22)] +Aggregate Attributes [1]: [avg(c_acctbal#22)#25] +Results [1]: [avg(c_acctbal#22)#25] + +(53) NativeProject +Output [1]: [avg(c_acctbal#22)#25 AS avg(c_acctbal)#26] +Input [1]: [avg(c_acctbal#22)#25] + +(54) Scan parquet +Output [2]: [c_phone#21, c_acctbal#22] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] +ReadSchema: struct + +(55) Filter +Input [2]: [c_phone#21, c_acctbal#22] +Condition : ((isnotnull(c_acctbal#22) AND (c_acctbal#22 > 0.00)) AND substring(c_phone#21, 1, 2) IN (13,31,23,29,30,18,17)) + +(56) Project +Output [1]: [c_acctbal#22] +Input [2]: [c_phone#21, c_acctbal#22] + +(57) HashAggregate +Input [1]: [c_acctbal#22] +Keys: [] +Functions [1]: [partial_avg(c_acctbal#22)] +Aggregate Attributes [2]: [sum#23, count#24] +Results [2]: [sum#27, count#28] + +(58) Exchange +Input [2]: [sum#27, count#28] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] + +(59) HashAggregate +Input [2]: [sum#27, count#28] +Keys: [] +Functions [1]: [avg(c_acctbal#22)] +Aggregate Attributes [1]: [avg(c_acctbal#22)#25] +Results [1]: [avg(c_acctbal#22)#25 AS avg(c_acctbal)#26] + +(60) AdaptiveSparkPlan +Output [1]: [avg(c_acctbal)#26] +Arguments: isFinalPlan=true + + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q3.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q3.txt new file mode 100644 index 000000000..f0c82a1e1 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q3.txt @@ -0,0 +1,359 @@ +== Physical Plan == +AdaptiveSparkPlan (64) ++- == Final Plan == + NativeTakeOrdered (40) + +- NativeProject (39) + +- NativeHashAggregate (38) + +- NativeHashAggregate (37) + +- NativeProject (36) + +- NativeProject (35) + +- NativeSortMergeJoin Inner (34) + :- NativeSort (24) + : +- InputAdapter (23) + : +- AQEShuffleRead (22) + : +- ShuffleQueryStage (21), Statistics(sizeInBytes=28.6 KiB, rowCount=1.80E+3) + : +- NativeShuffleExchange (20) + : +- NativeProject (19) + : +- NativeSortMergeJoin Inner (18) + : :- NativeSort (9) + : : +- InputAdapter (8) + : : +- AQEShuffleRead (7) + : : +- ShuffleQueryStage (6), Statistics(sizeInBytes=2.8 KiB, rowCount=337) + : : +- NativeShuffleExchange (5) + : : +- NativeProject (4) + : : +- NativeFilter (3) + : : +- InputAdapter (2) + : : +- NativeParquetScan (1) + : +- NativeSort (17) + : +- InputAdapter (16) + : +- AQEShuffleRead (15) + : +- ShuffleQueryStage (14), Statistics(sizeInBytes=172.3 KiB, rowCount=7.29E+3) + : +- NativeShuffleExchange (13) + : +- NativeFilter (12) + : +- InputAdapter (11) + : +- NativeParquetScan (10) + +- NativeSort (33) + +- InputAdapter (32) + +- AQEShuffleRead (31) + +- ShuffleQueryStage (30), Statistics(sizeInBytes=1264.1 KiB, rowCount=3.23E+4) + +- NativeShuffleExchange (29) + +- NativeProject (28) + +- NativeFilter (27) + +- InputAdapter (26) + +- NativeParquetScan (25) ++- == Initial Plan == + TakeOrderedAndProject (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Project (60) + +- SortMergeJoin Inner (59) + :- Sort (53) + : +- Exchange (52) + : +- Project (51) + : +- SortMergeJoin Inner (50) + : :- Sort (45) + : : +- Exchange (44) + : : +- Project (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Sort (49) + : +- Exchange (48) + : +- Filter (47) + : +- Scan parquet (46) + +- Sort (58) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) + + +(41) Scan parquet +Output [2]: [c_custkey#1, c_mktsegment#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] +ReadSchema: struct + +(2) InputAdapter +Input [2]: [c_custkey#1, c_mktsegment#2] +Arguments: [#3, #2] + +(3) NativeFilter +Input [2]: [#3#1, #2#2] +Condition : ((isnotnull(c_mktsegment#2) AND (c_mktsegment#2 = BUILDING)) AND isnotnull(c_custkey#1)) + +(4) NativeProject +Output [1]: [c_custkey#1] +Input [2]: [#3#1, #2#2] + +(5) NativeShuffleExchange +Input [1]: [c_custkey#1] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(6) ShuffleQueryStage +Output [1]: [c_custkey#1] +Arguments: X + +(7) AQEShuffleRead +Input [1]: [c_custkey#1] +Arguments: coalesced + +(8) InputAdapter +Input [1]: [c_custkey#1] + +(9) NativeSort +Input [1]: [c_custkey#1] +Arguments: [c_custkey#1 ASC NULLS FIRST], false + +(46) Scan parquet +Output [4]: [o_orderkey#4, o_custkey#5, o_orderdate#6, o_shippriority#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(11) InputAdapter +Input [4]: [o_orderkey#4, o_custkey#5, o_orderdate#6, o_shippriority#7] +Arguments: [#8, #9, #6, #7] + +(12) NativeFilter +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Condition : (((isnotnull(o_orderdate#6) AND (o_orderdate#6 < 1995-03-15)) AND isnotnull(o_custkey#5)) AND isnotnull(o_orderkey#4)) + +(13) NativeShuffleExchange +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: hashpartitioning(o_custkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: X + +(15) AQEShuffleRead +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: coalesced + +(16) InputAdapter +Input [4]: [#8#4, #9#5, #6#6, #7#7] + +(17) NativeSort +Input [4]: [#8#4, #9#5, #6#6, #7#7] +Arguments: [o_custkey#5 ASC NULLS FIRST], false + +(18) NativeSortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#5] +Join type: Inner +Join condition: None + +(19) NativeProject +Output [3]: [o_orderkey#4, o_orderdate#6, o_shippriority#7] +Input [5]: [c_custkey#1, #8#4, #9#5, #6#6, #7#7] + +(20) NativeShuffleExchange +Input [3]: [o_orderkey#4, o_orderdate#6, o_shippriority#7] +Arguments: hashpartitioning(o_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(21) ShuffleQueryStage +Output [3]: [o_orderkey#4, o_orderdate#6, o_shippriority#7] +Arguments: X + +(22) AQEShuffleRead +Input [3]: [o_orderkey#4, o_orderdate#6, o_shippriority#7] +Arguments: coalesced + +(23) InputAdapter +Input [3]: [o_orderkey#4, o_orderdate#6, o_shippriority#7] + +(24) NativeSort +Input [3]: [o_orderkey#4, o_orderdate#6, o_shippriority#7] +Arguments: [o_orderkey#4 ASC NULLS FIRST], false + +(54) Scan parquet +Output [4]: [l_orderkey#10, l_extendedprice#11, l_discount#12, l_shipdate#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] +ReadSchema: struct + +(26) InputAdapter +Input [4]: [l_orderkey#10, l_extendedprice#11, l_discount#12, l_shipdate#13] +Arguments: [#14, #11, #12, #13] + +(27) NativeFilter +Input [4]: [#14#10, #11#11, #12#12, #13#13] +Condition : ((isnotnull(l_shipdate#13) AND (l_shipdate#13 > 1995-03-15)) AND isnotnull(l_orderkey#10)) + +(28) NativeProject +Output [3]: [l_orderkey#10, l_extendedprice#11, l_discount#12] +Input [4]: [#14#10, #11#11, #12#12, #13#13] + +(29) NativeShuffleExchange +Input [3]: [l_orderkey#10, l_extendedprice#11, l_discount#12] +Arguments: hashpartitioning(l_orderkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(30) ShuffleQueryStage +Output [3]: [l_orderkey#10, l_extendedprice#11, l_discount#12] +Arguments: X + +(31) AQEShuffleRead +Input [3]: [l_orderkey#10, l_extendedprice#11, l_discount#12] +Arguments: coalesced + +(32) InputAdapter +Input [3]: [l_orderkey#10, l_extendedprice#11, l_discount#12] + +(33) NativeSort +Input [3]: [l_orderkey#10, l_extendedprice#11, l_discount#12] +Arguments: [l_orderkey#10 ASC NULLS FIRST], false + +(34) NativeSortMergeJoin +Left keys [1]: [o_orderkey#4] +Right keys [1]: [l_orderkey#10] +Join type: Inner +Join condition: None + +(35) NativeProject +Output [5]: [o_orderdate#6, o_shippriority#7, l_orderkey#10, l_extendedprice#11, l_discount#12] +Input [6]: [o_orderkey#4, o_orderdate#6, o_shippriority#7, l_orderkey#10, l_extendedprice#11, l_discount#12] + +(36) NativeProject +Output [4]: [l_orderkey#10 AS l_orderkey#10, o_orderdate#6 AS o_orderdate#6, o_shippriority#7 AS o_shippriority#7, (l_extendedprice#11 * (1 - l_discount#12)) AS _c3#15] +Input [5]: [o_orderdate#6, o_shippriority#7, l_orderkey#10, l_extendedprice#11, l_discount#12] + +(37) NativeHashAggregate +Input [4]: [l_orderkey#10, o_orderdate#6, o_shippriority#7, _c3#15] +Keys [3]: [l_orderkey#10, o_orderdate#6, o_shippriority#7] +Functions [1]: [partial_sum(_c3#15)] +Aggregate Attributes [2]: [sum#16, isEmpty#17] +Results [4]: [l_orderkey#10, o_orderdate#6, o_shippriority#7, #18#18] + +(38) NativeHashAggregate +Input [4]: [l_orderkey#10, o_orderdate#6, o_shippriority#7, #18#18] +Keys [3]: [l_orderkey#10, o_orderdate#6, o_shippriority#7] +Functions [1]: [sum((l_extendedprice#11 * (1 - l_discount#12)))] +Aggregate Attributes [1]: [sum((l_extendedprice#11 * (1 - l_discount#12)))#19] +Results [4]: [l_orderkey#10, o_orderdate#6, o_shippriority#7, sum((l_extendedprice#11 * (1 - l_discount#12)))#19] + +(39) NativeProject +Output [4]: [l_orderkey#10, sum((l_extendedprice#11 * (1 - l_discount#12)))#19 AS revenue#20, o_orderdate#6, o_shippriority#7] +Input [4]: [l_orderkey#10, o_orderdate#6, o_shippriority#7, sum((l_extendedprice#11 * (1 - l_discount#12)))#19] + +(40) NativeTakeOrdered +Input [4]: [l_orderkey#10, revenue#20, o_orderdate#6, o_shippriority#7] +Arguments: X, [revenue#20 DESC NULLS LAST, o_orderdate#6 ASC NULLS FIRST] + +(41) Scan parquet +Output [2]: [c_custkey#1, c_mktsegment#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] +ReadSchema: struct + +(42) Filter +Input [2]: [c_custkey#1, c_mktsegment#2] +Condition : ((isnotnull(c_mktsegment#2) AND (c_mktsegment#2 = BUILDING)) AND isnotnull(c_custkey#1)) + +(43) Project +Output [1]: [c_custkey#1] +Input [2]: [c_custkey#1, c_mktsegment#2] + +(44) Exchange +Input [1]: [c_custkey#1] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(45) Sort +Input [1]: [c_custkey#1] +Arguments: [c_custkey#1 ASC NULLS FIRST], false, 0 + +(46) Scan parquet +Output [4]: [o_orderkey#4, o_custkey#5, o_orderdate#6, o_shippriority#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(47) Filter +Input [4]: [o_orderkey#4, o_custkey#5, o_orderdate#6, o_shippriority#7] +Condition : (((isnotnull(o_orderdate#6) AND (o_orderdate#6 < 1995-03-15)) AND isnotnull(o_custkey#5)) AND isnotnull(o_orderkey#4)) + +(48) Exchange +Input [4]: [o_orderkey#4, o_custkey#5, o_orderdate#6, o_shippriority#7] +Arguments: hashpartitioning(o_custkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(49) Sort +Input [4]: [o_orderkey#4, o_custkey#5, o_orderdate#6, o_shippriority#7] +Arguments: [o_custkey#5 ASC NULLS FIRST], false, 0 + +(50) SortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#5] +Join type: Inner +Join condition: None + +(51) Project +Output [3]: [o_orderkey#4, o_orderdate#6, o_shippriority#7] +Input [5]: [c_custkey#1, o_orderkey#4, o_custkey#5, o_orderdate#6, o_shippriority#7] + +(52) Exchange +Input [3]: [o_orderkey#4, o_orderdate#6, o_shippriority#7] +Arguments: hashpartitioning(o_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(53) Sort +Input [3]: [o_orderkey#4, o_orderdate#6, o_shippriority#7] +Arguments: [o_orderkey#4 ASC NULLS FIRST], false, 0 + +(54) Scan parquet +Output [4]: [l_orderkey#10, l_extendedprice#11, l_discount#12, l_shipdate#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] +ReadSchema: struct + +(55) Filter +Input [4]: [l_orderkey#10, l_extendedprice#11, l_discount#12, l_shipdate#13] +Condition : ((isnotnull(l_shipdate#13) AND (l_shipdate#13 > 1995-03-15)) AND isnotnull(l_orderkey#10)) + +(56) Project +Output [3]: [l_orderkey#10, l_extendedprice#11, l_discount#12] +Input [4]: [l_orderkey#10, l_extendedprice#11, l_discount#12, l_shipdate#13] + +(57) Exchange +Input [3]: [l_orderkey#10, l_extendedprice#11, l_discount#12] +Arguments: hashpartitioning(l_orderkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(58) Sort +Input [3]: [l_orderkey#10, l_extendedprice#11, l_discount#12] +Arguments: [l_orderkey#10 ASC NULLS FIRST], false, 0 + +(59) SortMergeJoin +Left keys [1]: [o_orderkey#4] +Right keys [1]: [l_orderkey#10] +Join type: Inner +Join condition: None + +(60) Project +Output [5]: [o_orderdate#6, o_shippriority#7, l_orderkey#10, l_extendedprice#11, l_discount#12] +Input [6]: [o_orderkey#4, o_orderdate#6, o_shippriority#7, l_orderkey#10, l_extendedprice#11, l_discount#12] + +(61) HashAggregate +Input [5]: [o_orderdate#6, o_shippriority#7, l_orderkey#10, l_extendedprice#11, l_discount#12] +Keys [3]: [l_orderkey#10, o_orderdate#6, o_shippriority#7] +Functions [1]: [partial_sum((l_extendedprice#11 * (1 - l_discount#12)))] +Aggregate Attributes [2]: [sum#16, isEmpty#17] +Results [5]: [l_orderkey#10, o_orderdate#6, o_shippriority#7, sum#21, isEmpty#22] + +(62) HashAggregate +Input [5]: [l_orderkey#10, o_orderdate#6, o_shippriority#7, sum#21, isEmpty#22] +Keys [3]: [l_orderkey#10, o_orderdate#6, o_shippriority#7] +Functions [1]: [sum((l_extendedprice#11 * (1 - l_discount#12)))] +Aggregate Attributes [1]: [sum((l_extendedprice#11 * (1 - l_discount#12)))#19] +Results [4]: [l_orderkey#10, sum((l_extendedprice#11 * (1 - l_discount#12)))#19 AS revenue#20, o_orderdate#6, o_shippriority#7] + +(63) TakeOrderedAndProject +Input [4]: [l_orderkey#10, revenue#20, o_orderdate#6, o_shippriority#7] +Arguments: X, [revenue#20 DESC NULLS LAST, o_orderdate#6 ASC NULLS FIRST], [l_orderkey#10, revenue#20, o_orderdate#6, o_shippriority#7] + +(64) AdaptiveSparkPlan +Output [4]: [l_orderkey#10, revenue#20, o_orderdate#6, o_shippriority#7] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q4.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q4.txt new file mode 100644 index 000000000..98338c527 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q4.txt @@ -0,0 +1,279 @@ +== Physical Plan == +AdaptiveSparkPlan (50) ++- == Final Plan == + NativeSort (32) + +- InputAdapter (31) + +- AQEShuffleRead (30) + +- ShuffleQueryStage (29), Statistics(sizeInBytes=584.0 B, rowCount=5) + +- NativeShuffleExchange (28) + +- NativeProject (27) + +- NativeHashAggregate (26) + +- InputAdapter (25) + +- AQEShuffleRead (24) + +- ShuffleQueryStage (23), Statistics(sizeInBytes=696.0 B, rowCount=5) + +- NativeShuffleExchange (22) + +- NativeHashAggregate (21) + +- NativeProject (20) + +- NativeSortMergeJoin LeftSemi (19) + :- NativeSort (9) + : +- InputAdapter (8) + : +- AQEShuffleRead (7) + : +- ShuffleQueryStage (6), Statistics(sizeInBytes=12.0 KiB, rowCount=582) + : +- NativeShuffleExchange (5) + : +- NativeProject (4) + : +- NativeFilter (3) + : +- InputAdapter (2) + : +- NativeParquetScan (1) + +- NativeSort (18) + +- InputAdapter (17) + +- AQEShuffleRead (16) + +- ShuffleQueryStage (15), Statistics(sizeInBytes=297.5 KiB, rowCount=3.79E+4) + +- NativeShuffleExchange (14) + +- NativeProject (13) + +- NativeFilter (12) + +- InputAdapter (11) + +- NativeParquetScan (10) ++- == Initial Plan == + Sort (49) + +- Exchange (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- SortMergeJoin LeftSemi (43) + :- Sort (37) + : +- Exchange (36) + : +- Project (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Sort (42) + +- Exchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) + + +(33) Scan parquet +Output [3]: [o_orderkey#1, o_orderdate#2, o_orderpriority#3] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] +ReadSchema: struct + +(2) InputAdapter +Input [3]: [o_orderkey#1, o_orderdate#2, o_orderpriority#3] +Arguments: [#4, #2, #3] + +(3) NativeFilter +Input [3]: [#4#1, #2#2, #3#3] +Condition : ((isnotnull(o_orderdate#2) AND (o_orderdate#2 >= 1993-07-01)) AND (o_orderdate#2 < 1993-10-01)) + +(4) NativeProject +Output [2]: [o_orderkey#1, o_orderpriority#3] +Input [3]: [#4#1, #2#2, #3#3] + +(5) NativeShuffleExchange +Input [2]: [o_orderkey#1, o_orderpriority#3] +Arguments: hashpartitioning(o_orderkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(6) ShuffleQueryStage +Output [2]: [o_orderkey#1, o_orderpriority#3] +Arguments: X + +(7) AQEShuffleRead +Input [2]: [o_orderkey#1, o_orderpriority#3] +Arguments: coalesced + +(8) InputAdapter +Input [2]: [o_orderkey#1, o_orderpriority#3] + +(9) NativeSort +Input [2]: [o_orderkey#1, o_orderpriority#3] +Arguments: [o_orderkey#1 ASC NULLS FIRST], false + +(38) Scan parquet +Output [3]: [l_orderkey#5, l_commitdate#6, l_receiptdate#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] +ReadSchema: struct + +(11) InputAdapter +Input [3]: [l_orderkey#5, l_commitdate#6, l_receiptdate#7] +Arguments: [#8, #6, #7] + +(12) NativeFilter +Input [3]: [#8#5, #6#6, #7#7] +Condition : ((isnotnull(l_commitdate#6) AND isnotnull(l_receiptdate#7)) AND (l_commitdate#6 < l_receiptdate#7)) + +(13) NativeProject +Output [1]: [l_orderkey#5] +Input [3]: [#8#5, #6#6, #7#7] + +(14) NativeShuffleExchange +Input [1]: [l_orderkey#5] +Arguments: hashpartitioning(l_orderkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(15) ShuffleQueryStage +Output [1]: [l_orderkey#5] +Arguments: X + +(16) AQEShuffleRead +Input [1]: [l_orderkey#5] +Arguments: coalesced + +(17) InputAdapter +Input [1]: [l_orderkey#5] + +(18) NativeSort +Input [1]: [l_orderkey#5] +Arguments: [l_orderkey#5 ASC NULLS FIRST], false + +(19) NativeSortMergeJoin +Left keys [1]: [o_orderkey#1] +Right keys [1]: [l_orderkey#5] +Join type: LeftSemi +Join condition: None + +(20) NativeProject +Output [1]: [o_orderpriority#3] +Input [2]: [o_orderkey#1, o_orderpriority#3] + +(21) NativeHashAggregate +Input [1]: [o_orderpriority#3] +Keys [1]: [o_orderpriority#3] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#9] +Results [2]: [o_orderpriority#3, #10#10] + +(22) NativeShuffleExchange +Input [2]: [o_orderpriority#3, #10#10] +Arguments: hashpartitioning(o_orderpriority#3, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(23) ShuffleQueryStage +Output [2]: [o_orderpriority#3, #10#10] +Arguments: X + +(24) AQEShuffleRead +Input [2]: [o_orderpriority#3, #10#10] +Arguments: coalesced + +(25) InputAdapter +Input [2]: [o_orderpriority#3, #10#10] + +(26) NativeHashAggregate +Input [2]: [o_orderpriority#3, #10#10] +Keys [1]: [o_orderpriority#3] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#11] +Results [2]: [o_orderpriority#3, count(1)#11] + +(27) NativeProject +Output [2]: [o_orderpriority#3, count(1)#11 AS order_count#12] +Input [2]: [o_orderpriority#3, count(1)#11] + +(28) NativeShuffleExchange +Input [2]: [o_orderpriority#3, order_count#12] +Arguments: rangepartitioning(o_orderpriority#3 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(29) ShuffleQueryStage +Output [2]: [o_orderpriority#3, order_count#12] +Arguments: X + +(30) AQEShuffleRead +Input [2]: [o_orderpriority#3, order_count#12] +Arguments: coalesced + +(31) InputAdapter +Input [2]: [o_orderpriority#3, order_count#12] + +(32) NativeSort +Input [2]: [o_orderpriority#3, order_count#12] +Arguments: [o_orderpriority#3 ASC NULLS FIRST], true + +(33) Scan parquet +Output [3]: [o_orderkey#1, o_orderdate#2, o_orderpriority#3] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] +ReadSchema: struct + +(34) Filter +Input [3]: [o_orderkey#1, o_orderdate#2, o_orderpriority#3] +Condition : ((isnotnull(o_orderdate#2) AND (o_orderdate#2 >= 1993-07-01)) AND (o_orderdate#2 < 1993-10-01)) + +(35) Project +Output [2]: [o_orderkey#1, o_orderpriority#3] +Input [3]: [o_orderkey#1, o_orderdate#2, o_orderpriority#3] + +(36) Exchange +Input [2]: [o_orderkey#1, o_orderpriority#3] +Arguments: hashpartitioning(o_orderkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(37) Sort +Input [2]: [o_orderkey#1, o_orderpriority#3] +Arguments: [o_orderkey#1 ASC NULLS FIRST], false, 0 + +(38) Scan parquet +Output [3]: [l_orderkey#5, l_commitdate#6, l_receiptdate#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] +ReadSchema: struct + +(39) Filter +Input [3]: [l_orderkey#5, l_commitdate#6, l_receiptdate#7] +Condition : ((isnotnull(l_commitdate#6) AND isnotnull(l_receiptdate#7)) AND (l_commitdate#6 < l_receiptdate#7)) + +(40) Project +Output [1]: [l_orderkey#5] +Input [3]: [l_orderkey#5, l_commitdate#6, l_receiptdate#7] + +(41) Exchange +Input [1]: [l_orderkey#5] +Arguments: hashpartitioning(l_orderkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(42) Sort +Input [1]: [l_orderkey#5] +Arguments: [l_orderkey#5 ASC NULLS FIRST], false, 0 + +(43) SortMergeJoin +Left keys [1]: [o_orderkey#1] +Right keys [1]: [l_orderkey#5] +Join type: LeftSemi +Join condition: None + +(44) Project +Output [1]: [o_orderpriority#3] +Input [2]: [o_orderkey#1, o_orderpriority#3] + +(45) HashAggregate +Input [1]: [o_orderpriority#3] +Keys [1]: [o_orderpriority#3] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#9] +Results [2]: [o_orderpriority#3, count#13] + +(46) Exchange +Input [2]: [o_orderpriority#3, count#13] +Arguments: hashpartitioning(o_orderpriority#3, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(47) HashAggregate +Input [2]: [o_orderpriority#3, count#13] +Keys [1]: [o_orderpriority#3] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#11] +Results [2]: [o_orderpriority#3, count(1)#11 AS order_count#12] + +(48) Exchange +Input [2]: [o_orderpriority#3, order_count#12] +Arguments: rangepartitioning(o_orderpriority#3 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(49) Sort +Input [2]: [o_orderpriority#3, order_count#12] +Arguments: [o_orderpriority#3 ASC NULLS FIRST], true, 0 + +(50) AdaptiveSparkPlan +Output [2]: [o_orderpriority#3, order_count#12] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q5.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q5.txt new file mode 100644 index 000000000..76a74dd8c --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q5.txt @@ -0,0 +1,776 @@ +== Physical Plan == +AdaptiveSparkPlan (143) ++- == Final Plan == + NativeSort (93) + +- ConvertToNative (92) + +- AQEShuffleRead (91) + +- ShuffleQueryStage (90), Statistics(sizeInBytes=248.0 B, rowCount=5) + +- Exchange (89) + +- NativeProject (88) + +- NativeHashAggregate (87) + +- InputAdapter (86) + +- AQEShuffleRead (85) + +- ShuffleQueryStage (84), Statistics(sizeInBytes=760.0 B, rowCount=5) + +- NativeShuffleExchange (83) + +- NativeHashAggregate (82) + +- NativeProject (81) + +- NativeProject (80) + +- NativeSortMergeJoin Inner (79) + :- NativeSort (69) + : +- InputAdapter (68) + : +- AQEShuffleRead (67) + : +- ShuffleQueryStage (66), Statistics(sizeInBytes=19.8 KiB, rowCount=382) + : +- NativeShuffleExchange (65) + : +- NativeProject (64) + : +- NativeSortMergeJoin Inner (63) + : :- NativeSort (54) + : : +- InputAdapter (53) + : : +- AQEShuffleRead (52) + : : +- ShuffleQueryStage (51), Statistics(sizeInBytes=15.5 KiB, rowCount=382) + : : +- NativeShuffleExchange (50) + : : +- NativeProject (49) + : : +- NativeSortMergeJoin Inner (48) + : : :- NativeSort (39) + : : : +- InputAdapter (38) + : : : +- AQEShuffleRead (37) + : : : +- ShuffleQueryStage (36), Statistics(sizeInBytes=436.0 KiB, rowCount=9.28E+3) + : : : +- NativeShuffleExchange (35) + : : : +- NativeProject (34) + : : : +- NativeSortMergeJoin Inner (33) + : : : :- NativeSort (24) + : : : : +- InputAdapter (23) + : : : : +- AQEShuffleRead (22) + : : : : +- ShuffleQueryStage (21), Statistics(sizeInBytes=36.4 KiB, rowCount=2.30E+3) + : : : : +- NativeShuffleExchange (20) + : : : : +- NativeProject (19) + : : : : +- NativeSortMergeJoin Inner (18) + : : : : :- NativeSort (8) + : : : : : +- InputAdapter (7) + : : : : : +- AQEShuffleRead (6) + : : : : : +- ShuffleQueryStage (5), Statistics(sizeInBytes=23.8 KiB, rowCount=1.50E+3) + : : : : : +- NativeShuffleExchange (4) + : : : : : +- NativeFilter (3) + : : : : : +- InputAdapter (2) + : : : : : +- NativeParquetScan (1) + : : : : +- NativeSort (17) + : : : : +- InputAdapter (16) + : : : : +- AQEShuffleRead (15) + : : : : +- ShuffleQueryStage (14), Statistics(sizeInBytes=36.4 KiB, rowCount=2.30E+3) + : : : : +- NativeShuffleExchange (13) + : : : : +- NativeProject (12) + : : : : +- NativeFilter (11) + : : : : +- InputAdapter (10) + : : : : +- NativeParquetScan (9) + : : : +- NativeSort (32) + : : : +- InputAdapter (31) + : : : +- AQEShuffleRead (30) + : : : +- ShuffleQueryStage (29), Statistics(sizeInBytes=2.8 MiB, rowCount=6.02E+4) + : : : +- NativeShuffleExchange (28) + : : : +- NativeFilter (27) + : : : +- InputAdapter (26) + : : : +- NativeParquetScan (25) + : : +- NativeSort (47) + : : +- InputAdapter (46) + : : +- AQEShuffleRead (45) + : : +- ShuffleQueryStage (44), Statistics(sizeInBytes=1992.0 B, rowCount=100) + : : +- NativeShuffleExchange (43) + : : +- NativeFilter (42) + : : +- InputAdapter (41) + : : +- NativeParquetScan (40) + : +- NativeSort (62) + : +- InputAdapter (61) + : +- AQEShuffleRead (60) + : +- ShuffleQueryStage (59), Statistics(sizeInBytes=1381.0 B, rowCount=25) + : +- NativeShuffleExchange (58) + : +- NativeFilter (57) + : +- InputAdapter (56) + : +- NativeParquetScan (55) + +- NativeSort (78) + +- InputAdapter (77) + +- AQEShuffleRead (76) + +- ShuffleQueryStage (75), Statistics(sizeInBytes=216.0 B, rowCount=1) + +- NativeShuffleExchange (74) + +- NativeProject (73) + +- NativeFilter (72) + +- InputAdapter (71) + +- NativeParquetScan (70) ++- == Initial Plan == + Sort (142) + +- Exchange (141) + +- HashAggregate (140) + +- Exchange (139) + +- HashAggregate (138) + +- Project (137) + +- SortMergeJoin Inner (136) + :- Sort (130) + : +- Exchange (129) + : +- Project (128) + : +- SortMergeJoin Inner (127) + : :- Sort (122) + : : +- Exchange (121) + : : +- Project (120) + : : +- SortMergeJoin Inner (119) + : : :- Sort (114) + : : : +- Exchange (113) + : : : +- Project (112) + : : : +- SortMergeJoin Inner (111) + : : : :- Sort (106) + : : : : +- Exchange (105) + : : : : +- Project (104) + : : : : +- SortMergeJoin Inner (103) + : : : : :- Sort (97) + : : : : : +- Exchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- Sort (102) + : : : : +- Exchange (101) + : : : : +- Project (100) + : : : : +- Filter (99) + : : : : +- Scan parquet (98) + : : : +- Sort (110) + : : : +- Exchange (109) + : : : +- Filter (108) + : : : +- Scan parquet (107) + : : +- Sort (118) + : : +- Exchange (117) + : : +- Filter (116) + : : +- Scan parquet (115) + : +- Sort (126) + : +- Exchange (125) + : +- Filter (124) + : +- Scan parquet (123) + +- Sort (135) + +- Exchange (134) + +- Project (133) + +- Filter (132) + +- Scan parquet (131) + + +(94) Scan parquet +Output [2]: [c_custkey#1, c_nationkey#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(2) InputAdapter +Input [2]: [c_custkey#1, c_nationkey#2] +Arguments: [#3, #4] + +(3) NativeFilter +Input [2]: [#3#1, #4#2] +Condition : (isnotnull(c_custkey#1) AND isnotnull(c_nationkey#2)) + +(4) NativeShuffleExchange +Input [2]: [#3#1, #4#2] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [2]: [#3#1, #4#2] +Arguments: X + +(6) AQEShuffleRead +Input [2]: [#3#1, #4#2] +Arguments: coalesced + +(7) InputAdapter +Input [2]: [#3#1, #4#2] + +(8) NativeSort +Input [2]: [#3#1, #4#2] +Arguments: [c_custkey#1 ASC NULLS FIRST], false + +(98) Scan parquet +Output [3]: [o_orderkey#5, o_custkey#6, o_orderdate#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(10) InputAdapter +Input [3]: [o_orderkey#5, o_custkey#6, o_orderdate#7] +Arguments: [#8, #9, #7] + +(11) NativeFilter +Input [3]: [#8#5, #9#6, #7#7] +Condition : ((((isnotnull(o_orderdate#7) AND (o_orderdate#7 >= 1994-01-01)) AND (o_orderdate#7 < 1995-01-01)) AND isnotnull(o_custkey#6)) AND isnotnull(o_orderkey#5)) + +(12) NativeProject +Output [2]: [o_orderkey#5, o_custkey#6] +Input [3]: [#8#5, #9#6, #7#7] + +(13) NativeShuffleExchange +Input [2]: [o_orderkey#5, o_custkey#6] +Arguments: hashpartitioning(o_custkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [2]: [o_orderkey#5, o_custkey#6] +Arguments: X + +(15) AQEShuffleRead +Input [2]: [o_orderkey#5, o_custkey#6] +Arguments: coalesced + +(16) InputAdapter +Input [2]: [o_orderkey#5, o_custkey#6] + +(17) NativeSort +Input [2]: [o_orderkey#5, o_custkey#6] +Arguments: [o_custkey#6 ASC NULLS FIRST], false + +(18) NativeSortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#6] +Join type: Inner +Join condition: None + +(19) NativeProject +Output [2]: [c_nationkey#2, o_orderkey#5] +Input [4]: [#3#1, #4#2, o_orderkey#5, o_custkey#6] + +(20) NativeShuffleExchange +Input [2]: [c_nationkey#2, o_orderkey#5] +Arguments: hashpartitioning(o_orderkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(21) ShuffleQueryStage +Output [2]: [c_nationkey#2, o_orderkey#5] +Arguments: X + +(22) AQEShuffleRead +Input [2]: [c_nationkey#2, o_orderkey#5] +Arguments: coalesced + +(23) InputAdapter +Input [2]: [c_nationkey#2, o_orderkey#5] + +(24) NativeSort +Input [2]: [c_nationkey#2, o_orderkey#5] +Arguments: [o_orderkey#5 ASC NULLS FIRST], false + +(107) Scan parquet +Output [4]: [l_orderkey#10, l_suppkey#11, l_extendedprice#12, l_discount#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(26) InputAdapter +Input [4]: [l_orderkey#10, l_suppkey#11, l_extendedprice#12, l_discount#13] +Arguments: [#14, #15, #12, #13] + +(27) NativeFilter +Input [4]: [#14#10, #15#11, #12#12, #13#13] +Condition : (isnotnull(l_orderkey#10) AND isnotnull(l_suppkey#11)) + +(28) NativeShuffleExchange +Input [4]: [#14#10, #15#11, #12#12, #13#13] +Arguments: hashpartitioning(l_orderkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(29) ShuffleQueryStage +Output [4]: [#14#10, #15#11, #12#12, #13#13] +Arguments: X + +(30) AQEShuffleRead +Input [4]: [#14#10, #15#11, #12#12, #13#13] +Arguments: coalesced + +(31) InputAdapter +Input [4]: [#14#10, #15#11, #12#12, #13#13] + +(32) NativeSort +Input [4]: [#14#10, #15#11, #12#12, #13#13] +Arguments: [l_orderkey#10 ASC NULLS FIRST], false + +(33) NativeSortMergeJoin +Left keys [1]: [o_orderkey#5] +Right keys [1]: [l_orderkey#10] +Join type: Inner +Join condition: None + +(34) NativeProject +Output [4]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13] +Input [6]: [c_nationkey#2, o_orderkey#5, #14#10, #15#11, #12#12, #13#13] + +(35) NativeShuffleExchange +Input [4]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13] +Arguments: hashpartitioning(l_suppkey#11, c_nationkey#2, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(36) ShuffleQueryStage +Output [4]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13] +Arguments: X + +(37) AQEShuffleRead +Input [4]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13] +Arguments: coalesced + +(38) InputAdapter +Input [4]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13] + +(39) NativeSort +Input [4]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13] +Arguments: [l_suppkey#11 ASC NULLS FIRST, c_nationkey#2 ASC NULLS FIRST], false + +(115) Scan parquet +Output [2]: [s_suppkey#16, s_nationkey#17] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(41) InputAdapter +Input [2]: [s_suppkey#16, s_nationkey#17] +Arguments: [#18, #19] + +(42) NativeFilter +Input [2]: [#18#16, #19#17] +Condition : (isnotnull(s_suppkey#16) AND isnotnull(s_nationkey#17)) + +(43) NativeShuffleExchange +Input [2]: [#18#16, #19#17] +Arguments: hashpartitioning(s_suppkey#16, s_nationkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(44) ShuffleQueryStage +Output [2]: [#18#16, #19#17] +Arguments: X + +(45) AQEShuffleRead +Input [2]: [#18#16, #19#17] +Arguments: coalesced + +(46) InputAdapter +Input [2]: [#18#16, #19#17] + +(47) NativeSort +Input [2]: [#18#16, #19#17] +Arguments: [s_suppkey#16 ASC NULLS FIRST, s_nationkey#17 ASC NULLS FIRST], false + +(48) NativeSortMergeJoin +Left keys [2]: [l_suppkey#11, c_nationkey#2] +Right keys [2]: [s_suppkey#16, s_nationkey#17] +Join type: Inner +Join condition: None + +(49) NativeProject +Output [3]: [l_extendedprice#12, l_discount#13, s_nationkey#17] +Input [6]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13, #18#16, #19#17] + +(50) NativeShuffleExchange +Input [3]: [l_extendedprice#12, l_discount#13, s_nationkey#17] +Arguments: hashpartitioning(s_nationkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(51) ShuffleQueryStage +Output [3]: [l_extendedprice#12, l_discount#13, s_nationkey#17] +Arguments: X + +(52) AQEShuffleRead +Input [3]: [l_extendedprice#12, l_discount#13, s_nationkey#17] +Arguments: coalesced + +(53) InputAdapter +Input [3]: [l_extendedprice#12, l_discount#13, s_nationkey#17] + +(54) NativeSort +Input [3]: [l_extendedprice#12, l_discount#13, s_nationkey#17] +Arguments: [s_nationkey#17 ASC NULLS FIRST], false + +(123) Scan parquet +Output [3]: [n_nationkey#20, n_name#21, n_regionkey#22] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(56) InputAdapter +Input [3]: [n_nationkey#20, n_name#21, n_regionkey#22] +Arguments: [#23, #21, #24] + +(57) NativeFilter +Input [3]: [#23#20, #21#21, #24#22] +Condition : (isnotnull(n_nationkey#20) AND isnotnull(n_regionkey#22)) + +(58) NativeShuffleExchange +Input [3]: [#23#20, #21#21, #24#22] +Arguments: hashpartitioning(n_nationkey#20, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(59) ShuffleQueryStage +Output [3]: [#23#20, #21#21, #24#22] +Arguments: X + +(60) AQEShuffleRead +Input [3]: [#23#20, #21#21, #24#22] +Arguments: coalesced + +(61) InputAdapter +Input [3]: [#23#20, #21#21, #24#22] + +(62) NativeSort +Input [3]: [#23#20, #21#21, #24#22] +Arguments: [n_nationkey#20 ASC NULLS FIRST], false + +(63) NativeSortMergeJoin +Left keys [1]: [s_nationkey#17] +Right keys [1]: [n_nationkey#20] +Join type: Inner +Join condition: None + +(64) NativeProject +Output [4]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22] +Input [6]: [l_extendedprice#12, l_discount#13, s_nationkey#17, #23#20, #21#21, #24#22] + +(65) NativeShuffleExchange +Input [4]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22] +Arguments: hashpartitioning(n_regionkey#22, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(66) ShuffleQueryStage +Output [4]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22] +Arguments: X + +(67) AQEShuffleRead +Input [4]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22] +Arguments: coalesced + +(68) InputAdapter +Input [4]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22] + +(69) NativeSort +Input [4]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22] +Arguments: [n_regionkey#22 ASC NULLS FIRST], false + +(131) Scan parquet +Output [2]: [r_regionkey#25, r_name#26] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(71) InputAdapter +Input [2]: [r_regionkey#25, r_name#26] +Arguments: [#27, #26] + +(72) NativeFilter +Input [2]: [#27#25, #26#26] +Condition : ((isnotnull(r_name#26) AND (r_name#26 = ASIA)) AND isnotnull(r_regionkey#25)) + +(73) NativeProject +Output [1]: [r_regionkey#25] +Input [2]: [#27#25, #26#26] + +(74) NativeShuffleExchange +Input [1]: [r_regionkey#25] +Arguments: hashpartitioning(r_regionkey#25, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(75) ShuffleQueryStage +Output [1]: [r_regionkey#25] +Arguments: X + +(76) AQEShuffleRead +Input [1]: [r_regionkey#25] +Arguments: coalesced + +(77) InputAdapter +Input [1]: [r_regionkey#25] + +(78) NativeSort +Input [1]: [r_regionkey#25] +Arguments: [r_regionkey#25 ASC NULLS FIRST], false + +(79) NativeSortMergeJoin +Left keys [1]: [n_regionkey#22] +Right keys [1]: [r_regionkey#25] +Join type: Inner +Join condition: None + +(80) NativeProject +Output [3]: [l_extendedprice#12, l_discount#13, n_name#21] +Input [5]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22, r_regionkey#25] + +(81) NativeProject +Output [2]: [n_name#21 AS n_name#21, (l_extendedprice#12 * (1 - l_discount#13)) AS _c1#28] +Input [3]: [l_extendedprice#12, l_discount#13, n_name#21] + +(82) NativeHashAggregate +Input [2]: [n_name#21, _c1#28] +Keys [1]: [n_name#21] +Functions [1]: [partial_sum(_c1#28)] +Aggregate Attributes [2]: [sum#29, isEmpty#30] +Results [2]: [n_name#21, #31#31] + +(83) NativeShuffleExchange +Input [2]: [n_name#21, #31#31] +Arguments: hashpartitioning(n_name#21, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(84) ShuffleQueryStage +Output [2]: [n_name#21, #31#31] +Arguments: X + +(85) AQEShuffleRead +Input [2]: [n_name#21, #31#31] +Arguments: coalesced + +(86) InputAdapter +Input [2]: [n_name#21, #31#31] + +(87) NativeHashAggregate +Input [2]: [n_name#21, #31#31] +Keys [1]: [n_name#21] +Functions [1]: [sum((l_extendedprice#12 * (1 - l_discount#13)))] +Aggregate Attributes [1]: [sum((l_extendedprice#12 * (1 - l_discount#13)))#32] +Results [2]: [n_name#21, sum((l_extendedprice#12 * (1 - l_discount#13)))#32] + +(88) NativeProject +Output [2]: [n_name#21, sum((l_extendedprice#12 * (1 - l_discount#13)))#32 AS revenue#33] +Input [2]: [n_name#21, sum((l_extendedprice#12 * (1 - l_discount#13)))#32] + +(89) Exchange +Input [2]: [n_name#21, revenue#33] +Arguments: rangepartitioning(revenue#33 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=12] + +(90) ShuffleQueryStage +Output [2]: [n_name#21, revenue#33] +Arguments: X + +(91) AQEShuffleRead +Input [2]: [n_name#21, revenue#33] +Arguments: coalesced + +(92) ConvertToNative +Input [2]: [n_name#21, revenue#33] + +(93) NativeSort +Input [2]: [n_name#21, revenue#33] +Arguments: [revenue#33 DESC NULLS LAST], true + +(94) Scan parquet +Output [2]: [c_custkey#1, c_nationkey#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(95) Filter +Input [2]: [c_custkey#1, c_nationkey#2] +Condition : (isnotnull(c_custkey#1) AND isnotnull(c_nationkey#2)) + +(96) Exchange +Input [2]: [c_custkey#1, c_nationkey#2] +Arguments: hashpartitioning(c_custkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=13] + +(97) Sort +Input [2]: [c_custkey#1, c_nationkey#2] +Arguments: [c_custkey#1 ASC NULLS FIRST], false, 0 + +(98) Scan parquet +Output [3]: [o_orderkey#5, o_custkey#6, o_orderdate#7] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] +ReadSchema: struct + +(99) Filter +Input [3]: [o_orderkey#5, o_custkey#6, o_orderdate#7] +Condition : ((((isnotnull(o_orderdate#7) AND (o_orderdate#7 >= 1994-01-01)) AND (o_orderdate#7 < 1995-01-01)) AND isnotnull(o_custkey#6)) AND isnotnull(o_orderkey#5)) + +(100) Project +Output [2]: [o_orderkey#5, o_custkey#6] +Input [3]: [o_orderkey#5, o_custkey#6, o_orderdate#7] + +(101) Exchange +Input [2]: [o_orderkey#5, o_custkey#6] +Arguments: hashpartitioning(o_custkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=14] + +(102) Sort +Input [2]: [o_orderkey#5, o_custkey#6] +Arguments: [o_custkey#6 ASC NULLS FIRST], false, 0 + +(103) SortMergeJoin +Left keys [1]: [c_custkey#1] +Right keys [1]: [o_custkey#6] +Join type: Inner +Join condition: None + +(104) Project +Output [2]: [c_nationkey#2, o_orderkey#5] +Input [4]: [c_custkey#1, c_nationkey#2, o_orderkey#5, o_custkey#6] + +(105) Exchange +Input [2]: [c_nationkey#2, o_orderkey#5] +Arguments: hashpartitioning(o_orderkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=15] + +(106) Sort +Input [2]: [c_nationkey#2, o_orderkey#5] +Arguments: [o_orderkey#5 ASC NULLS FIRST], false, 0 + +(107) Scan parquet +Output [4]: [l_orderkey#10, l_suppkey#11, l_extendedprice#12, l_discount#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] +ReadSchema: struct + +(108) Filter +Input [4]: [l_orderkey#10, l_suppkey#11, l_extendedprice#12, l_discount#13] +Condition : (isnotnull(l_orderkey#10) AND isnotnull(l_suppkey#11)) + +(109) Exchange +Input [4]: [l_orderkey#10, l_suppkey#11, l_extendedprice#12, l_discount#13] +Arguments: hashpartitioning(l_orderkey#10, 5), ENSURE_REQUIREMENTS, [plan_id=16] + +(110) Sort +Input [4]: [l_orderkey#10, l_suppkey#11, l_extendedprice#12, l_discount#13] +Arguments: [l_orderkey#10 ASC NULLS FIRST], false, 0 + +(111) SortMergeJoin +Left keys [1]: [o_orderkey#5] +Right keys [1]: [l_orderkey#10] +Join type: Inner +Join condition: None + +(112) Project +Output [4]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13] +Input [6]: [c_nationkey#2, o_orderkey#5, l_orderkey#10, l_suppkey#11, l_extendedprice#12, l_discount#13] + +(113) Exchange +Input [4]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13] +Arguments: hashpartitioning(l_suppkey#11, c_nationkey#2, 5), ENSURE_REQUIREMENTS, [plan_id=17] + +(114) Sort +Input [4]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13] +Arguments: [l_suppkey#11 ASC NULLS FIRST, c_nationkey#2 ASC NULLS FIRST], false, 0 + +(115) Scan parquet +Output [2]: [s_suppkey#16, s_nationkey#17] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(116) Filter +Input [2]: [s_suppkey#16, s_nationkey#17] +Condition : (isnotnull(s_suppkey#16) AND isnotnull(s_nationkey#17)) + +(117) Exchange +Input [2]: [s_suppkey#16, s_nationkey#17] +Arguments: hashpartitioning(s_suppkey#16, s_nationkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=18] + +(118) Sort +Input [2]: [s_suppkey#16, s_nationkey#17] +Arguments: [s_suppkey#16 ASC NULLS FIRST, s_nationkey#17 ASC NULLS FIRST], false, 0 + +(119) SortMergeJoin +Left keys [2]: [l_suppkey#11, c_nationkey#2] +Right keys [2]: [s_suppkey#16, s_nationkey#17] +Join type: Inner +Join condition: None + +(120) Project +Output [3]: [l_extendedprice#12, l_discount#13, s_nationkey#17] +Input [6]: [c_nationkey#2, l_suppkey#11, l_extendedprice#12, l_discount#13, s_suppkey#16, s_nationkey#17] + +(121) Exchange +Input [3]: [l_extendedprice#12, l_discount#13, s_nationkey#17] +Arguments: hashpartitioning(s_nationkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=19] + +(122) Sort +Input [3]: [l_extendedprice#12, l_discount#13, s_nationkey#17] +Arguments: [s_nationkey#17 ASC NULLS FIRST], false, 0 + +(123) Scan parquet +Output [3]: [n_nationkey#20, n_name#21, n_regionkey#22] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(124) Filter +Input [3]: [n_nationkey#20, n_name#21, n_regionkey#22] +Condition : (isnotnull(n_nationkey#20) AND isnotnull(n_regionkey#22)) + +(125) Exchange +Input [3]: [n_nationkey#20, n_name#21, n_regionkey#22] +Arguments: hashpartitioning(n_nationkey#20, 5), ENSURE_REQUIREMENTS, [plan_id=20] + +(126) Sort +Input [3]: [n_nationkey#20, n_name#21, n_regionkey#22] +Arguments: [n_nationkey#20 ASC NULLS FIRST], false, 0 + +(127) SortMergeJoin +Left keys [1]: [s_nationkey#17] +Right keys [1]: [n_nationkey#20] +Join type: Inner +Join condition: None + +(128) Project +Output [4]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22] +Input [6]: [l_extendedprice#12, l_discount#13, s_nationkey#17, n_nationkey#20, n_name#21, n_regionkey#22] + +(129) Exchange +Input [4]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22] +Arguments: hashpartitioning(n_regionkey#22, 5), ENSURE_REQUIREMENTS, [plan_id=21] + +(130) Sort +Input [4]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22] +Arguments: [n_regionkey#22 ASC NULLS FIRST], false, 0 + +(131) Scan parquet +Output [2]: [r_regionkey#25, r_name#26] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(132) Filter +Input [2]: [r_regionkey#25, r_name#26] +Condition : ((isnotnull(r_name#26) AND (r_name#26 = ASIA)) AND isnotnull(r_regionkey#25)) + +(133) Project +Output [1]: [r_regionkey#25] +Input [2]: [r_regionkey#25, r_name#26] + +(134) Exchange +Input [1]: [r_regionkey#25] +Arguments: hashpartitioning(r_regionkey#25, 5), ENSURE_REQUIREMENTS, [plan_id=22] + +(135) Sort +Input [1]: [r_regionkey#25] +Arguments: [r_regionkey#25 ASC NULLS FIRST], false, 0 + +(136) SortMergeJoin +Left keys [1]: [n_regionkey#22] +Right keys [1]: [r_regionkey#25] +Join type: Inner +Join condition: None + +(137) Project +Output [3]: [l_extendedprice#12, l_discount#13, n_name#21] +Input [5]: [l_extendedprice#12, l_discount#13, n_name#21, n_regionkey#22, r_regionkey#25] + +(138) HashAggregate +Input [3]: [l_extendedprice#12, l_discount#13, n_name#21] +Keys [1]: [n_name#21] +Functions [1]: [partial_sum((l_extendedprice#12 * (1 - l_discount#13)))] +Aggregate Attributes [2]: [sum#29, isEmpty#30] +Results [3]: [n_name#21, sum#34, isEmpty#35] + +(139) Exchange +Input [3]: [n_name#21, sum#34, isEmpty#35] +Arguments: hashpartitioning(n_name#21, 5), ENSURE_REQUIREMENTS, [plan_id=23] + +(140) HashAggregate +Input [3]: [n_name#21, sum#34, isEmpty#35] +Keys [1]: [n_name#21] +Functions [1]: [sum((l_extendedprice#12 * (1 - l_discount#13)))] +Aggregate Attributes [1]: [sum((l_extendedprice#12 * (1 - l_discount#13)))#32] +Results [2]: [n_name#21, sum((l_extendedprice#12 * (1 - l_discount#13)))#32 AS revenue#33] + +(141) Exchange +Input [2]: [n_name#21, revenue#33] +Arguments: rangepartitioning(revenue#33 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=24] + +(142) Sort +Input [2]: [n_name#21, revenue#33] +Arguments: [revenue#33 DESC NULLS LAST], true, 0 + +(143) AdaptiveSparkPlan +Output [2]: [n_name#21, revenue#33] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q6.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q6.txt new file mode 100644 index 000000000..6a0b00304 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q6.txt @@ -0,0 +1,108 @@ +== Physical Plan == +AdaptiveSparkPlan (17) ++- == Final Plan == + NativeProject (10) + +- NativeHashAggregate (9) + +- ShuffleQueryStage (8), Statistics(sizeInBytes=360.0 B, rowCount=1) + +- NativeShuffleExchange (7) + +- NativeHashAggregate (6) + +- NativeProject (5) + +- NativeProject (4) + +- NativeFilter (3) + +- InputAdapter (2) + +- NativeParquetScan (1) ++- == Initial Plan == + HashAggregate (16) + +- Exchange (15) + +- HashAggregate (14) + +- Project (13) + +- Filter (12) + +- Scan parquet (11) + + +(11) Scan parquet +Output [4]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_shipdate#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] +ReadSchema: struct + +(2) InputAdapter +Input [4]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_shipdate#4] +Arguments: [#1, #2, #3, #4] + +(3) NativeFilter +Input [4]: [#1#1, #2#2, #3#3, #4#4] +Condition : (((((((isnotnull(l_shipdate#4) AND isnotnull(l_discount#3)) AND isnotnull(l_quantity#1)) AND (l_shipdate#4 >= 1994-01-01)) AND (l_shipdate#4 < 1995-01-01)) AND (l_discount#3 >= 0.05)) AND (l_discount#3 <= 0.07)) AND (l_quantity#1 < 24.00)) + +(4) NativeProject +Output [2]: [l_extendedprice#2, l_discount#3] +Input [4]: [#1#1, #2#2, #3#3, #4#4] + +(5) NativeProject +Output [1]: [(l_extendedprice#2 * l_discount#3) AS _c0#5] +Input [2]: [l_extendedprice#2, l_discount#3] + +(6) NativeHashAggregate +Input [1]: [_c0#5] +Keys: [] +Functions [1]: [partial_sum(_c0#5)] +Aggregate Attributes [2]: [sum#6, isEmpty#7] +Results [1]: [#8#8] + +(7) NativeShuffleExchange +Input [1]: [#8#8] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] + +(8) ShuffleQueryStage +Output [1]: [#8#8] +Arguments: X + +(9) NativeHashAggregate +Input [1]: [#8#8] +Keys: [] +Functions [1]: [sum((l_extendedprice#2 * l_discount#3))] +Aggregate Attributes [1]: [sum((l_extendedprice#2 * l_discount#3))#9] +Results [1]: [sum((l_extendedprice#2 * l_discount#3))#9] + +(10) NativeProject +Output [1]: [sum((l_extendedprice#2 * l_discount#3))#9 AS revenue#10] +Input [1]: [sum((l_extendedprice#2 * l_discount#3))#9] + +(11) Scan parquet +Output [4]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_shipdate#4] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] +ReadSchema: struct + +(12) Filter +Input [4]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_shipdate#4] +Condition : (((((((isnotnull(l_shipdate#4) AND isnotnull(l_discount#3)) AND isnotnull(l_quantity#1)) AND (l_shipdate#4 >= 1994-01-01)) AND (l_shipdate#4 < 1995-01-01)) AND (l_discount#3 >= 0.05)) AND (l_discount#3 <= 0.07)) AND (l_quantity#1 < 24.00)) + +(13) Project +Output [2]: [l_extendedprice#2, l_discount#3] +Input [4]: [l_quantity#1, l_extendedprice#2, l_discount#3, l_shipdate#4] + +(14) HashAggregate +Input [2]: [l_extendedprice#2, l_discount#3] +Keys: [] +Functions [1]: [partial_sum((l_extendedprice#2 * l_discount#3))] +Aggregate Attributes [2]: [sum#6, isEmpty#7] +Results [2]: [sum#11, isEmpty#12] + +(15) Exchange +Input [2]: [sum#11, isEmpty#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] + +(16) HashAggregate +Input [2]: [sum#11, isEmpty#12] +Keys: [] +Functions [1]: [sum((l_extendedprice#2 * l_discount#3))] +Aggregate Attributes [1]: [sum((l_extendedprice#2 * l_discount#3))#9] +Results [1]: [sum((l_extendedprice#2 * l_discount#3))#9 AS revenue#10] + +(17) AdaptiveSparkPlan +Output [1]: [revenue#10] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q7.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q7.txt new file mode 100644 index 000000000..54e127e8a --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q7.txt @@ -0,0 +1,728 @@ +== Physical Plan == +AdaptiveSparkPlan (134) ++- == Final Plan == + NativeSort (86) + +- ConvertToNative (85) + +- AQEShuffleRead (84) + +- ShuffleQueryStage (83), Statistics(sizeInBytes=288.0 B, rowCount=4) + +- Exchange (82) + +- * HashAggregate (81) + +- AQEShuffleRead (80) + +- ShuffleQueryStage (79), Statistics(sizeInBytes=320.0 B, rowCount=4) + +- Exchange (78) + +- * HashAggregate (77) + +- * Project (76) + +- * SortMergeJoin Inner (75) + :- NativeSort (68) + : +- InputAdapter (67) + : +- AQEShuffleRead (66) + : +- ShuffleQueryStage (65), Statistics(sizeInBytes=69.2 KiB, rowCount=1.28E+3) + : +- NativeShuffleExchange (64) + : +- NativeProject (63) + : +- NativeSortMergeJoin Inner (62) + : :- NativeSort (53) + : : +- InputAdapter (52) + : : +- AQEShuffleRead (51) + : : +- ShuffleQueryStage (50), Statistics(sizeInBytes=914.5 KiB, rowCount=1.80E+4) + : : +- NativeShuffleExchange (49) + : : +- NativeProject (48) + : : +- NativeSortMergeJoin Inner (47) + : : :- NativeSort (38) + : : : +- InputAdapter (37) + : : : +- AQEShuffleRead (36) + : : : +- ShuffleQueryStage (35), Statistics(sizeInBytes=914.7 KiB, rowCount=1.80E+4) + : : : +- NativeShuffleExchange (34) + : : : +- NativeProject (33) + : : : +- NativeSortMergeJoin Inner (32) + : : : :- NativeSort (23) + : : : : +- InputAdapter (22) + : : : : +- AQEShuffleRead (21) + : : : : +- ShuffleQueryStage (20), Statistics(sizeInBytes=914.5 KiB, rowCount=1.80E+4) + : : : : +- NativeShuffleExchange (19) + : : : : +- NativeProject (18) + : : : : +- NativeSortMergeJoin Inner (17) + : : : : :- NativeSort (8) + : : : : : +- InputAdapter (7) + : : : : : +- AQEShuffleRead (6) + : : : : : +- ShuffleQueryStage (5), Statistics(sizeInBytes=1992.0 B, rowCount=100) + : : : : : +- NativeShuffleExchange (4) + : : : : : +- NativeFilter (3) + : : : : : +- InputAdapter (2) + : : : : : +- NativeParquetScan (1) + : : : : +- NativeSort (16) + : : : : +- InputAdapter (15) + : : : : +- AQEShuffleRead (14) + : : : : +- ShuffleQueryStage (13), Statistics(sizeInBytes=919.1 KiB, rowCount=1.80E+4) + : : : : +- NativeShuffleExchange (12) + : : : : +- NativeFilter (11) + : : : : +- InputAdapter (10) + : : : : +- NativeParquetScan (9) + : : : +- NativeSort (31) + : : : +- InputAdapter (30) + : : : +- AQEShuffleRead (29) + : : : +- ShuffleQueryStage (28), Statistics(sizeInBytes=235.1 KiB, rowCount=1.50E+4) + : : : +- NativeShuffleExchange (27) + : : : +- NativeFilter (26) + : : : +- InputAdapter (25) + : : : +- NativeParquetScan (24) + : : +- NativeSort (46) + : : +- InputAdapter (45) + : : +- AQEShuffleRead (44) + : : +- ShuffleQueryStage (43), Statistics(sizeInBytes=23.8 KiB, rowCount=1.50E+3) + : : +- NativeShuffleExchange (42) + : : +- NativeFilter (41) + : : +- InputAdapter (40) + : : +- NativeParquetScan (39) + : +- NativeSort (61) + : +- InputAdapter (60) + : +- AQEShuffleRead (59) + : +- ShuffleQueryStage (58), Statistics(sizeInBytes=457.0 B, rowCount=2) + : +- NativeShuffleExchange (57) + : +- NativeFilter (56) + : +- InputAdapter (55) + : +- NativeParquetScan (54) + +- NativeSort (74) + +- InputAdapter (73) + +- InputAdapter (72) + +- AQEShuffleRead (71) + +- ShuffleQueryStage (70), Statistics(sizeInBytes=457.0 B, rowCount=2) + +- ReusedExchange (69) ++- == Initial Plan == + Sort (133) + +- Exchange (132) + +- HashAggregate (131) + +- Exchange (130) + +- HashAggregate (129) + +- Project (128) + +- SortMergeJoin Inner (127) + :- Sort (122) + : +- Exchange (121) + : +- Project (120) + : +- SortMergeJoin Inner (119) + : :- Sort (114) + : : +- Exchange (113) + : : +- Project (112) + : : +- SortMergeJoin Inner (111) + : : :- Sort (106) + : : : +- Exchange (105) + : : : +- Project (104) + : : : +- SortMergeJoin Inner (103) + : : : :- Sort (98) + : : : : +- Exchange (97) + : : : : +- Project (96) + : : : : +- SortMergeJoin Inner (95) + : : : : :- Sort (90) + : : : : : +- Exchange (89) + : : : : : +- Filter (88) + : : : : : +- Scan parquet (87) + : : : : +- Sort (94) + : : : : +- Exchange (93) + : : : : +- Filter (92) + : : : : +- Scan parquet (91) + : : : +- Sort (102) + : : : +- Exchange (101) + : : : +- Filter (100) + : : : +- Scan parquet (99) + : : +- Sort (110) + : : +- Exchange (109) + : : +- Filter (108) + : : +- Scan parquet (107) + : +- Sort (118) + : +- Exchange (117) + : +- Filter (116) + : +- Scan parquet (115) + +- Sort (126) + +- Exchange (125) + +- Filter (124) + +- Scan parquet (123) + + +(87) Scan parquet +Output [2]: [s_suppkey#1, s_nationkey#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(2) InputAdapter +Input [2]: [s_suppkey#1, s_nationkey#2] +Arguments: [#3, #4] + +(3) NativeFilter +Input [2]: [#3#1, #4#2] +Condition : (isnotnull(s_suppkey#1) AND isnotnull(s_nationkey#2)) + +(4) NativeShuffleExchange +Input [2]: [#3#1, #4#2] +Arguments: hashpartitioning(s_suppkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(5) ShuffleQueryStage +Output [2]: [#3#1, #4#2] +Arguments: X + +(6) AQEShuffleRead +Input [2]: [#3#1, #4#2] +Arguments: coalesced + +(7) InputAdapter +Input [2]: [#3#1, #4#2] + +(8) NativeSort +Input [2]: [#3#1, #4#2] +Arguments: [s_suppkey#1 ASC NULLS FIRST], false + +(91) Scan parquet +Output [5]: [l_orderkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(10) InputAdapter +Input [5]: [l_orderkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] +Arguments: [#10, #11, #7, #8, #9] + +(11) NativeFilter +Input [5]: [#10#5, #11#6, #7#7, #8#8, #9#9] +Condition : ((((isnotnull(l_shipdate#9) AND (l_shipdate#9 >= 1995-01-01)) AND (l_shipdate#9 <= 1996-12-31)) AND isnotnull(l_suppkey#6)) AND isnotnull(l_orderkey#5)) + +(12) NativeShuffleExchange +Input [5]: [#10#5, #11#6, #7#7, #8#8, #9#9] +Arguments: hashpartitioning(l_suppkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(13) ShuffleQueryStage +Output [5]: [#10#5, #11#6, #7#7, #8#8, #9#9] +Arguments: X + +(14) AQEShuffleRead +Input [5]: [#10#5, #11#6, #7#7, #8#8, #9#9] +Arguments: coalesced + +(15) InputAdapter +Input [5]: [#10#5, #11#6, #7#7, #8#8, #9#9] + +(16) NativeSort +Input [5]: [#10#5, #11#6, #7#7, #8#8, #9#9] +Arguments: [l_suppkey#6 ASC NULLS FIRST], false + +(17) NativeSortMergeJoin +Left keys [1]: [s_suppkey#1] +Right keys [1]: [l_suppkey#6] +Join type: Inner +Join condition: None + +(18) NativeProject +Output [5]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9] +Input [7]: [#3#1, #4#2, #10#5, #11#6, #7#7, #8#8, #9#9] + +(19) NativeShuffleExchange +Input [5]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9] +Arguments: hashpartitioning(l_orderkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(20) ShuffleQueryStage +Output [5]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9] +Arguments: X + +(21) AQEShuffleRead +Input [5]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9] +Arguments: coalesced + +(22) InputAdapter +Input [5]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9] + +(23) NativeSort +Input [5]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9] +Arguments: [l_orderkey#5 ASC NULLS FIRST], false + +(99) Scan parquet +Output [2]: [o_orderkey#12, o_custkey#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(25) InputAdapter +Input [2]: [o_orderkey#12, o_custkey#13] +Arguments: [#14, #15] + +(26) NativeFilter +Input [2]: [#14#12, #15#13] +Condition : (isnotnull(o_orderkey#12) AND isnotnull(o_custkey#13)) + +(27) NativeShuffleExchange +Input [2]: [#14#12, #15#13] +Arguments: hashpartitioning(o_orderkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(28) ShuffleQueryStage +Output [2]: [#14#12, #15#13] +Arguments: X + +(29) AQEShuffleRead +Input [2]: [#14#12, #15#13] +Arguments: coalesced + +(30) InputAdapter +Input [2]: [#14#12, #15#13] + +(31) NativeSort +Input [2]: [#14#12, #15#13] +Arguments: [o_orderkey#12 ASC NULLS FIRST], false + +(32) NativeSortMergeJoin +Left keys [1]: [l_orderkey#5] +Right keys [1]: [o_orderkey#12] +Join type: Inner +Join condition: None + +(33) NativeProject +Output [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13] +Input [7]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9, #14#12, #15#13] + +(34) NativeShuffleExchange +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13] +Arguments: hashpartitioning(o_custkey#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(35) ShuffleQueryStage +Output [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13] +Arguments: X + +(36) AQEShuffleRead +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13] +Arguments: coalesced + +(37) InputAdapter +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13] + +(38) NativeSort +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13] +Arguments: [o_custkey#13 ASC NULLS FIRST], false + +(107) Scan parquet +Output [2]: [c_custkey#16, c_nationkey#17] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(40) InputAdapter +Input [2]: [c_custkey#16, c_nationkey#17] +Arguments: [#18, #19] + +(41) NativeFilter +Input [2]: [#18#16, #19#17] +Condition : (isnotnull(c_custkey#16) AND isnotnull(c_nationkey#17)) + +(42) NativeShuffleExchange +Input [2]: [#18#16, #19#17] +Arguments: hashpartitioning(c_custkey#16, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(43) ShuffleQueryStage +Output [2]: [#18#16, #19#17] +Arguments: X + +(44) AQEShuffleRead +Input [2]: [#18#16, #19#17] +Arguments: coalesced + +(45) InputAdapter +Input [2]: [#18#16, #19#17] + +(46) NativeSort +Input [2]: [#18#16, #19#17] +Arguments: [c_custkey#16 ASC NULLS FIRST], false + +(47) NativeSortMergeJoin +Left keys [1]: [o_custkey#13] +Right keys [1]: [c_custkey#16] +Join type: Inner +Join condition: None + +(48) NativeProject +Output [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17] +Input [7]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13, #18#16, #19#17] + +(49) NativeShuffleExchange +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17] +Arguments: hashpartitioning(s_nationkey#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(50) ShuffleQueryStage +Output [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17] +Arguments: X + +(51) AQEShuffleRead +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17] +Arguments: coalesced + +(52) InputAdapter +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17] + +(53) NativeSort +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17] +Arguments: [s_nationkey#2 ASC NULLS FIRST], false + +(115) Scan parquet +Output [2]: [n_nationkey#20, n_name#21] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] +ReadSchema: struct + +(55) InputAdapter +Input [2]: [n_nationkey#20, n_name#21] +Arguments: [#22, #21] + +(56) NativeFilter +Input [2]: [#22#20, #21#21] +Condition : (isnotnull(n_nationkey#20) AND ((n_name#21 = FRANCE) OR (n_name#21 = GERMANY))) + +(57) NativeShuffleExchange +Input [2]: [#22#20, #21#21] +Arguments: hashpartitioning(n_nationkey#20, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(58) ShuffleQueryStage +Output [2]: [#22#20, #21#21] +Arguments: X + +(59) AQEShuffleRead +Input [2]: [#22#20, #21#21] +Arguments: coalesced + +(60) InputAdapter +Input [2]: [#22#20, #21#21] + +(61) NativeSort +Input [2]: [#22#20, #21#21] +Arguments: [n_nationkey#20 ASC NULLS FIRST], false + +(62) NativeSortMergeJoin +Left keys [1]: [s_nationkey#2] +Right keys [1]: [n_nationkey#20] +Join type: Inner +Join condition: None + +(63) NativeProject +Output [5]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21] +Input [7]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, #22#20, #21#21] + +(64) NativeShuffleExchange +Input [5]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21] +Arguments: hashpartitioning(c_nationkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(65) ShuffleQueryStage +Output [5]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21] +Arguments: X + +(66) AQEShuffleRead +Input [5]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21] +Arguments: coalesced + +(67) InputAdapter +Input [5]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21] + +(68) NativeSort +Input [5]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21] +Arguments: [c_nationkey#17 ASC NULLS FIRST], false + +(69) ReusedExchange [Reuses operator id: 57] +Output [2]: [n_nationkey#23, n_name#24] + +(70) ShuffleQueryStage +Output [2]: [n_nationkey#23, n_name#24] +Arguments: X + +(71) AQEShuffleRead +Input [2]: [n_nationkey#23, n_name#24] +Arguments: coalesced + +(72) InputAdapter +Input [2]: [n_nationkey#23, n_name#24] +Arguments: [#25, #24] + +(73) InputAdapter +Input [2]: [#25#23, #24#24] + +(74) NativeSort +Input [2]: [#25#23, #24#24] +Arguments: [n_nationkey#23 ASC NULLS FIRST], false + +(75) SortMergeJoin [codegen id : 1] +Left keys [1]: [c_nationkey#17] +Right keys [1]: [n_nationkey#23] +Join type: Inner +Join condition: (((n_name#21 = FRANCE) AND (n_name#24 = GERMANY)) OR ((n_name#21 = GERMANY) AND (n_name#24 = FRANCE))) + +(76) Project [codegen id : 1] +Output [4]: [n_name#21 AS supp_nation#26, n_name#24 AS cust_nation#27, year(l_shipdate#9) AS l_year#28, (l_extendedprice#7 * (1 - l_discount#8)) AS volume#29] +Input [7]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21, #25#23, #24#24] + +(77) HashAggregate [codegen id : 1] +Input [4]: [supp_nation#26, cust_nation#27, l_year#28, volume#29] +Keys [3]: [supp_nation#26, cust_nation#27, l_year#28] +Functions [1]: [partial_sum(volume#29)] +Aggregate Attributes [2]: [sum#30, isEmpty#31] +Results [5]: [supp_nation#26, cust_nation#27, l_year#28, sum#32, isEmpty#33] + +(78) Exchange +Input [5]: [supp_nation#26, cust_nation#27, l_year#28, sum#32, isEmpty#33] +Arguments: hashpartitioning(supp_nation#26, cust_nation#27, l_year#28, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(79) ShuffleQueryStage +Output [5]: [supp_nation#26, cust_nation#27, l_year#28, sum#32, isEmpty#33] +Arguments: X + +(80) AQEShuffleRead +Input [5]: [supp_nation#26, cust_nation#27, l_year#28, sum#32, isEmpty#33] +Arguments: coalesced + +(81) HashAggregate [codegen id : 2] +Input [5]: [supp_nation#26, cust_nation#27, l_year#28, sum#32, isEmpty#33] +Keys [3]: [supp_nation#26, cust_nation#27, l_year#28] +Functions [1]: [sum(volume#29)] +Aggregate Attributes [1]: [sum(volume#29)#34] +Results [4]: [supp_nation#26, cust_nation#27, l_year#28, sum(volume#29)#34 AS revenue#35] + +(82) Exchange +Input [4]: [supp_nation#26, cust_nation#27, l_year#28, revenue#35] +Arguments: rangepartitioning(supp_nation#26 ASC NULLS FIRST, cust_nation#27 ASC NULLS FIRST, l_year#28 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(83) ShuffleQueryStage +Output [4]: [supp_nation#26, cust_nation#27, l_year#28, revenue#35] +Arguments: X + +(84) AQEShuffleRead +Input [4]: [supp_nation#26, cust_nation#27, l_year#28, revenue#35] +Arguments: coalesced + +(85) ConvertToNative +Input [4]: [supp_nation#26, cust_nation#27, l_year#28, revenue#35] + +(86) NativeSort +Input [4]: [supp_nation#26, cust_nation#27, l_year#28, revenue#35] +Arguments: [supp_nation#26 ASC NULLS FIRST, cust_nation#27 ASC NULLS FIRST, l_year#28 ASC NULLS FIRST], true + +(87) Scan parquet +Output [2]: [s_suppkey#1, s_nationkey#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(88) Filter +Input [2]: [s_suppkey#1, s_nationkey#2] +Condition : (isnotnull(s_suppkey#1) AND isnotnull(s_nationkey#2)) + +(89) Exchange +Input [2]: [s_suppkey#1, s_nationkey#2] +Arguments: hashpartitioning(s_suppkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=12] + +(90) Sort +Input [2]: [s_suppkey#1, s_nationkey#2] +Arguments: [s_suppkey#1 ASC NULLS FIRST], false, 0 + +(91) Scan parquet +Output [5]: [l_orderkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(92) Filter +Input [5]: [l_orderkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] +Condition : ((((isnotnull(l_shipdate#9) AND (l_shipdate#9 >= 1995-01-01)) AND (l_shipdate#9 <= 1996-12-31)) AND isnotnull(l_suppkey#6)) AND isnotnull(l_orderkey#5)) + +(93) Exchange +Input [5]: [l_orderkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] +Arguments: hashpartitioning(l_suppkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=13] + +(94) Sort +Input [5]: [l_orderkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] +Arguments: [l_suppkey#6 ASC NULLS FIRST], false, 0 + +(95) SortMergeJoin +Left keys [1]: [s_suppkey#1] +Right keys [1]: [l_suppkey#6] +Join type: Inner +Join condition: None + +(96) Project +Output [5]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9] +Input [7]: [s_suppkey#1, s_nationkey#2, l_orderkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8, l_shipdate#9] + +(97) Exchange +Input [5]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9] +Arguments: hashpartitioning(l_orderkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=14] + +(98) Sort +Input [5]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9] +Arguments: [l_orderkey#5 ASC NULLS FIRST], false, 0 + +(99) Scan parquet +Output [2]: [o_orderkey#12, o_custkey#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(100) Filter +Input [2]: [o_orderkey#12, o_custkey#13] +Condition : (isnotnull(o_orderkey#12) AND isnotnull(o_custkey#13)) + +(101) Exchange +Input [2]: [o_orderkey#12, o_custkey#13] +Arguments: hashpartitioning(o_orderkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=15] + +(102) Sort +Input [2]: [o_orderkey#12, o_custkey#13] +Arguments: [o_orderkey#12 ASC NULLS FIRST], false, 0 + +(103) SortMergeJoin +Left keys [1]: [l_orderkey#5] +Right keys [1]: [o_orderkey#12] +Join type: Inner +Join condition: None + +(104) Project +Output [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13] +Input [7]: [s_nationkey#2, l_orderkey#5, l_extendedprice#7, l_discount#8, l_shipdate#9, o_orderkey#12, o_custkey#13] + +(105) Exchange +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13] +Arguments: hashpartitioning(o_custkey#13, 5), ENSURE_REQUIREMENTS, [plan_id=16] + +(106) Sort +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13] +Arguments: [o_custkey#13 ASC NULLS FIRST], false, 0 + +(107) Scan parquet +Output [2]: [c_custkey#16, c_nationkey#17] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(108) Filter +Input [2]: [c_custkey#16, c_nationkey#17] +Condition : (isnotnull(c_custkey#16) AND isnotnull(c_nationkey#17)) + +(109) Exchange +Input [2]: [c_custkey#16, c_nationkey#17] +Arguments: hashpartitioning(c_custkey#16, 5), ENSURE_REQUIREMENTS, [plan_id=17] + +(110) Sort +Input [2]: [c_custkey#16, c_nationkey#17] +Arguments: [c_custkey#16 ASC NULLS FIRST], false, 0 + +(111) SortMergeJoin +Left keys [1]: [o_custkey#13] +Right keys [1]: [c_custkey#16] +Join type: Inner +Join condition: None + +(112) Project +Output [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17] +Input [7]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, o_custkey#13, c_custkey#16, c_nationkey#17] + +(113) Exchange +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17] +Arguments: hashpartitioning(s_nationkey#2, 5), ENSURE_REQUIREMENTS, [plan_id=18] + +(114) Sort +Input [5]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17] +Arguments: [s_nationkey#2 ASC NULLS FIRST], false, 0 + +(115) Scan parquet +Output [2]: [n_nationkey#20, n_name#21] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] +ReadSchema: struct + +(116) Filter +Input [2]: [n_nationkey#20, n_name#21] +Condition : (isnotnull(n_nationkey#20) AND ((n_name#21 = FRANCE) OR (n_name#21 = GERMANY))) + +(117) Exchange +Input [2]: [n_nationkey#20, n_name#21] +Arguments: hashpartitioning(n_nationkey#20, 5), ENSURE_REQUIREMENTS, [plan_id=19] + +(118) Sort +Input [2]: [n_nationkey#20, n_name#21] +Arguments: [n_nationkey#20 ASC NULLS FIRST], false, 0 + +(119) SortMergeJoin +Left keys [1]: [s_nationkey#2] +Right keys [1]: [n_nationkey#20] +Join type: Inner +Join condition: None + +(120) Project +Output [5]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21] +Input [7]: [s_nationkey#2, l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_nationkey#20, n_name#21] + +(121) Exchange +Input [5]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21] +Arguments: hashpartitioning(c_nationkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=20] + +(122) Sort +Input [5]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21] +Arguments: [c_nationkey#17 ASC NULLS FIRST], false, 0 + +(123) Scan parquet +Output [2]: [n_nationkey#23, n_name#24] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] +ReadSchema: struct + +(124) Filter +Input [2]: [n_nationkey#23, n_name#24] +Condition : (isnotnull(n_nationkey#23) AND ((n_name#24 = GERMANY) OR (n_name#24 = FRANCE))) + +(125) Exchange +Input [2]: [n_nationkey#23, n_name#24] +Arguments: hashpartitioning(n_nationkey#23, 5), ENSURE_REQUIREMENTS, [plan_id=21] + +(126) Sort +Input [2]: [n_nationkey#23, n_name#24] +Arguments: [n_nationkey#23 ASC NULLS FIRST], false, 0 + +(127) SortMergeJoin +Left keys [1]: [c_nationkey#17] +Right keys [1]: [n_nationkey#23] +Join type: Inner +Join condition: (((n_name#21 = FRANCE) AND (n_name#24 = GERMANY)) OR ((n_name#21 = GERMANY) AND (n_name#24 = FRANCE))) + +(128) Project +Output [4]: [n_name#21 AS supp_nation#26, n_name#24 AS cust_nation#27, year(l_shipdate#9) AS l_year#28, (l_extendedprice#7 * (1 - l_discount#8)) AS volume#29] +Input [7]: [l_extendedprice#7, l_discount#8, l_shipdate#9, c_nationkey#17, n_name#21, n_nationkey#23, n_name#24] + +(129) HashAggregate +Input [4]: [supp_nation#26, cust_nation#27, l_year#28, volume#29] +Keys [3]: [supp_nation#26, cust_nation#27, l_year#28] +Functions [1]: [partial_sum(volume#29)] +Aggregate Attributes [2]: [sum#30, isEmpty#31] +Results [5]: [supp_nation#26, cust_nation#27, l_year#28, sum#32, isEmpty#33] + +(130) Exchange +Input [5]: [supp_nation#26, cust_nation#27, l_year#28, sum#32, isEmpty#33] +Arguments: hashpartitioning(supp_nation#26, cust_nation#27, l_year#28, 5), ENSURE_REQUIREMENTS, [plan_id=22] + +(131) HashAggregate +Input [5]: [supp_nation#26, cust_nation#27, l_year#28, sum#32, isEmpty#33] +Keys [3]: [supp_nation#26, cust_nation#27, l_year#28] +Functions [1]: [sum(volume#29)] +Aggregate Attributes [1]: [sum(volume#29)#34] +Results [4]: [supp_nation#26, cust_nation#27, l_year#28, sum(volume#29)#34 AS revenue#35] + +(132) Exchange +Input [4]: [supp_nation#26, cust_nation#27, l_year#28, revenue#35] +Arguments: rangepartitioning(supp_nation#26 ASC NULLS FIRST, cust_nation#27 ASC NULLS FIRST, l_year#28 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=23] + +(133) Sort +Input [4]: [supp_nation#26, cust_nation#27, l_year#28, revenue#35] +Arguments: [supp_nation#26 ASC NULLS FIRST, cust_nation#27 ASC NULLS FIRST, l_year#28 ASC NULLS FIRST], true, 0 + +(134) AdaptiveSparkPlan +Output [4]: [supp_nation#26, cust_nation#27, l_year#28, revenue#35] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q8.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q8.txt new file mode 100644 index 000000000..769824f0d --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q8.txt @@ -0,0 +1,1022 @@ +== Physical Plan == +AdaptiveSparkPlan (189) ++- == Final Plan == + NativeSort (123) + +- InputAdapter (122) + +- AQEShuffleRead (121) + +- ShuffleQueryStage (120), Statistics(sizeInBytes=488.0 B, rowCount=2) + +- NativeShuffleExchange (119) + +- NativeProject (118) + +- NativeHashAggregate (117) + +- InputAdapter (116) + +- AQEShuffleRead (115) + +- ShuffleQueryStage (114), Statistics(sizeInBytes=672.0 B, rowCount=2) + +- NativeShuffleExchange (113) + +- NativeHashAggregate (112) + +- NativeProject (111) + +- NativeProject (110) + +- NativeSortMergeJoin Inner (109) + :- NativeSort (99) + : +- InputAdapter (98) + : +- AQEShuffleRead (97) + : +- ShuffleQueryStage (96), Statistics(sizeInBytes=7.3 KiB, rowCount=116) + : +- NativeShuffleExchange (95) + : +- NativeProject (94) + : +- NativeSortMergeJoin Inner (93) + : :- NativeSort (84) + : : +- InputAdapter (83) + : : +- AQEShuffleRead (82) + : : +- ShuffleQueryStage (81), Statistics(sizeInBytes=6.8 KiB, rowCount=116) + : : +- NativeShuffleExchange (80) + : : +- NativeProject (79) + : : +- NativeSortMergeJoin Inner (78) + : : :- NativeSort (69) + : : : +- InputAdapter (68) + : : : +- AQEShuffleRead (67) + : : : +- ShuffleQueryStage (66), Statistics(sizeInBytes=6.8 KiB, rowCount=116) + : : : +- NativeShuffleExchange (65) + : : : +- NativeProject (64) + : : : +- NativeSortMergeJoin Inner (63) + : : : :- NativeSort (54) + : : : : +- InputAdapter (53) + : : : : +- AQEShuffleRead (52) + : : : : +- ShuffleQueryStage (51), Statistics(sizeInBytes=6.8 KiB, rowCount=116) + : : : : +- NativeShuffleExchange (50) + : : : : +- NativeProject (49) + : : : : +- NativeSortMergeJoin Inner (48) + : : : : :- NativeSort (39) + : : : : : +- InputAdapter (38) + : : : : : +- AQEShuffleRead (37) + : : : : : +- ShuffleQueryStage (36), Statistics(sizeInBytes=17.9 KiB, rowCount=366) + : : : : : +- NativeShuffleExchange (35) + : : : : : +- NativeProject (34) + : : : : : +- NativeSortMergeJoin Inner (33) + : : : : : :- NativeSort (24) + : : : : : : +- InputAdapter (23) + : : : : : : +- AQEShuffleRead (22) + : : : : : : +- ShuffleQueryStage (21), Statistics(sizeInBytes=18.0 KiB, rowCount=366) + : : : : : : +- NativeShuffleExchange (20) + : : : : : : +- NativeProject (19) + : : : : : : +- NativeSortMergeJoin Inner (18) + : : : : : : :- NativeSort (9) + : : : : : : : +- InputAdapter (8) + : : : : : : : +- AQEShuffleRead (7) + : : : : : : : +- ShuffleQueryStage (6), Statistics(sizeInBytes=304.0 B, rowCount=12) + : : : : : : : +- NativeShuffleExchange (5) + : : : : : : : +- NativeProject (4) + : : : : : : : +- NativeFilter (3) + : : : : : : : +- InputAdapter (2) + : : : : : : : +- NativeParquetScan (1) + : : : : : : +- NativeSort (17) + : : : : : : +- InputAdapter (16) + : : : : : : +- AQEShuffleRead (15) + : : : : : : +- ShuffleQueryStage (14), Statistics(sizeInBytes=3.2 MiB, rowCount=6.02E+4) + : : : : : : +- NativeShuffleExchange (13) + : : : : : : +- NativeFilter (12) + : : : : : : +- InputAdapter (11) + : : : : : : +- NativeParquetScan (10) + : : : : : +- NativeSort (32) + : : : : : +- InputAdapter (31) + : : : : : +- AQEShuffleRead (30) + : : : : : +- ShuffleQueryStage (29), Statistics(sizeInBytes=1992.0 B, rowCount=100) + : : : : : +- NativeShuffleExchange (28) + : : : : : +- NativeFilter (27) + : : : : : +- InputAdapter (26) + : : : : : +- NativeParquetScan (25) + : : : : +- NativeSort (47) + : : : : +- InputAdapter (46) + : : : : +- AQEShuffleRead (45) + : : : : +- ShuffleQueryStage (44), Statistics(sizeInBytes=89.0 KiB, rowCount=4.50E+3) + : : : : +- NativeShuffleExchange (43) + : : : : +- NativeFilter (42) + : : : : +- InputAdapter (41) + : : : : +- NativeParquetScan (40) + : : : +- NativeSort (62) + : : : +- InputAdapter (61) + : : : +- AQEShuffleRead (60) + : : : +- ShuffleQueryStage (59), Statistics(sizeInBytes=23.8 KiB, rowCount=1.50E+3) + : : : +- NativeShuffleExchange (58) + : : : +- NativeFilter (57) + : : : +- InputAdapter (56) + : : : +- NativeParquetScan (55) + : : +- NativeSort (77) + : : +- InputAdapter (76) + : : +- AQEShuffleRead (75) + : : +- ShuffleQueryStage (74), Statistics(sizeInBytes=792.0 B, rowCount=25) + : : +- NativeShuffleExchange (73) + : : +- NativeFilter (72) + : : +- InputAdapter (71) + : : +- NativeParquetScan (70) + : +- NativeSort (92) + : +- InputAdapter (91) + : +- AQEShuffleRead (90) + : +- ShuffleQueryStage (89), Statistics(sizeInBytes=997.0 B, rowCount=25) + : +- NativeShuffleExchange (88) + : +- NativeFilter (87) + : +- InputAdapter (86) + : +- NativeParquetScan (85) + +- NativeSort (108) + +- InputAdapter (107) + +- AQEShuffleRead (106) + +- ShuffleQueryStage (105), Statistics(sizeInBytes=216.0 B, rowCount=1) + +- NativeShuffleExchange (104) + +- NativeProject (103) + +- NativeFilter (102) + +- InputAdapter (101) + +- NativeParquetScan (100) ++- == Initial Plan == + Sort (188) + +- Exchange (187) + +- HashAggregate (186) + +- Exchange (185) + +- HashAggregate (184) + +- Project (183) + +- SortMergeJoin Inner (182) + :- Sort (176) + : +- Exchange (175) + : +- Project (174) + : +- SortMergeJoin Inner (173) + : :- Sort (168) + : : +- Exchange (167) + : : +- Project (166) + : : +- SortMergeJoin Inner (165) + : : :- Sort (160) + : : : +- Exchange (159) + : : : +- Project (158) + : : : +- SortMergeJoin Inner (157) + : : : :- Sort (152) + : : : : +- Exchange (151) + : : : : +- Project (150) + : : : : +- SortMergeJoin Inner (149) + : : : : :- Sort (144) + : : : : : +- Exchange (143) + : : : : : +- Project (142) + : : : : : +- SortMergeJoin Inner (141) + : : : : : :- Sort (136) + : : : : : : +- Exchange (135) + : : : : : : +- Project (134) + : : : : : : +- SortMergeJoin Inner (133) + : : : : : : :- Sort (128) + : : : : : : : +- Exchange (127) + : : : : : : : +- Project (126) + : : : : : : : +- Filter (125) + : : : : : : : +- Scan parquet (124) + : : : : : : +- Sort (132) + : : : : : : +- Exchange (131) + : : : : : : +- Filter (130) + : : : : : : +- Scan parquet (129) + : : : : : +- Sort (140) + : : : : : +- Exchange (139) + : : : : : +- Filter (138) + : : : : : +- Scan parquet (137) + : : : : +- Sort (148) + : : : : +- Exchange (147) + : : : : +- Filter (146) + : : : : +- Scan parquet (145) + : : : +- Sort (156) + : : : +- Exchange (155) + : : : +- Filter (154) + : : : +- Scan parquet (153) + : : +- Sort (164) + : : +- Exchange (163) + : : +- Filter (162) + : : +- Scan parquet (161) + : +- Sort (172) + : +- Exchange (171) + : +- Filter (170) + : +- Scan parquet (169) + +- Sort (181) + +- Exchange (180) + +- Project (179) + +- Filter (178) + +- Scan parquet (177) + + +(124) Scan parquet +Output [2]: [p_partkey#1, p_type#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) InputAdapter +Input [2]: [p_partkey#1, p_type#2] +Arguments: [#3, #2] + +(3) NativeFilter +Input [2]: [#3#1, #2#2] +Condition : ((isnotnull(p_type#2) AND (p_type#2 = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#1)) + +(4) NativeProject +Output [1]: [p_partkey#1] +Input [2]: [#3#1, #2#2] + +(5) NativeShuffleExchange +Input [1]: [p_partkey#1] +Arguments: hashpartitioning(p_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(6) ShuffleQueryStage +Output [1]: [p_partkey#1] +Arguments: X + +(7) AQEShuffleRead +Input [1]: [p_partkey#1] +Arguments: coalesced + +(8) InputAdapter +Input [1]: [p_partkey#1] + +(9) NativeSort +Input [1]: [p_partkey#1] +Arguments: [p_partkey#1 ASC NULLS FIRST], false + +(129) Scan parquet +Output [5]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) InputAdapter +Input [5]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8] +Arguments: [#9, #10, #11, #7, #8] + +(12) NativeFilter +Input [5]: [#9#4, #10#5, #11#6, #7#7, #8#8] +Condition : ((isnotnull(l_partkey#5) AND isnotnull(l_suppkey#6)) AND isnotnull(l_orderkey#4)) + +(13) NativeShuffleExchange +Input [5]: [#9#4, #10#5, #11#6, #7#7, #8#8] +Arguments: hashpartitioning(l_partkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [5]: [#9#4, #10#5, #11#6, #7#7, #8#8] +Arguments: X + +(15) AQEShuffleRead +Input [5]: [#9#4, #10#5, #11#6, #7#7, #8#8] +Arguments: coalesced + +(16) InputAdapter +Input [5]: [#9#4, #10#5, #11#6, #7#7, #8#8] + +(17) NativeSort +Input [5]: [#9#4, #10#5, #11#6, #7#7, #8#8] +Arguments: [l_partkey#5 ASC NULLS FIRST], false + +(18) NativeSortMergeJoin +Left keys [1]: [p_partkey#1] +Right keys [1]: [l_partkey#5] +Join type: Inner +Join condition: None + +(19) NativeProject +Output [4]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8] +Input [6]: [p_partkey#1, #9#4, #10#5, #11#6, #7#7, #8#8] + +(20) NativeShuffleExchange +Input [4]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8] +Arguments: hashpartitioning(l_suppkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(21) ShuffleQueryStage +Output [4]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8] +Arguments: X + +(22) AQEShuffleRead +Input [4]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8] +Arguments: coalesced + +(23) InputAdapter +Input [4]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8] + +(24) NativeSort +Input [4]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8] +Arguments: [l_suppkey#6 ASC NULLS FIRST], false + +(137) Scan parquet +Output [2]: [s_suppkey#12, s_nationkey#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(26) InputAdapter +Input [2]: [s_suppkey#12, s_nationkey#13] +Arguments: [#14, #15] + +(27) NativeFilter +Input [2]: [#14#12, #15#13] +Condition : (isnotnull(s_suppkey#12) AND isnotnull(s_nationkey#13)) + +(28) NativeShuffleExchange +Input [2]: [#14#12, #15#13] +Arguments: hashpartitioning(s_suppkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(29) ShuffleQueryStage +Output [2]: [#14#12, #15#13] +Arguments: X + +(30) AQEShuffleRead +Input [2]: [#14#12, #15#13] +Arguments: coalesced + +(31) InputAdapter +Input [2]: [#14#12, #15#13] + +(32) NativeSort +Input [2]: [#14#12, #15#13] +Arguments: [s_suppkey#12 ASC NULLS FIRST], false + +(33) NativeSortMergeJoin +Left keys [1]: [l_suppkey#6] +Right keys [1]: [s_suppkey#12] +Join type: Inner +Join condition: None + +(34) NativeProject +Output [4]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13] +Input [6]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8, #14#12, #15#13] + +(35) NativeShuffleExchange +Input [4]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13] +Arguments: hashpartitioning(l_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(36) ShuffleQueryStage +Output [4]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13] +Arguments: X + +(37) AQEShuffleRead +Input [4]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13] +Arguments: coalesced + +(38) InputAdapter +Input [4]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13] + +(39) NativeSort +Input [4]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13] +Arguments: [l_orderkey#4 ASC NULLS FIRST], false + +(145) Scan parquet +Output [3]: [o_orderkey#16, o_custkey#17, o_orderdate#18] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(41) InputAdapter +Input [3]: [o_orderkey#16, o_custkey#17, o_orderdate#18] +Arguments: [#19, #20, #18] + +(42) NativeFilter +Input [3]: [#19#16, #20#17, #18#18] +Condition : ((((isnotnull(o_orderdate#18) AND (o_orderdate#18 >= 1995-01-01)) AND (o_orderdate#18 <= 1996-12-31)) AND isnotnull(o_orderkey#16)) AND isnotnull(o_custkey#17)) + +(43) NativeShuffleExchange +Input [3]: [#19#16, #20#17, #18#18] +Arguments: hashpartitioning(o_orderkey#16, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(44) ShuffleQueryStage +Output [3]: [#19#16, #20#17, #18#18] +Arguments: X + +(45) AQEShuffleRead +Input [3]: [#19#16, #20#17, #18#18] +Arguments: coalesced + +(46) InputAdapter +Input [3]: [#19#16, #20#17, #18#18] + +(47) NativeSort +Input [3]: [#19#16, #20#17, #18#18] +Arguments: [o_orderkey#16 ASC NULLS FIRST], false + +(48) NativeSortMergeJoin +Left keys [1]: [l_orderkey#4] +Right keys [1]: [o_orderkey#16] +Join type: Inner +Join condition: None + +(49) NativeProject +Output [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18] +Input [7]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13, #19#16, #20#17, #18#18] + +(50) NativeShuffleExchange +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18] +Arguments: hashpartitioning(o_custkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(51) ShuffleQueryStage +Output [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18] +Arguments: X + +(52) AQEShuffleRead +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18] +Arguments: coalesced + +(53) InputAdapter +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18] + +(54) NativeSort +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18] +Arguments: [o_custkey#17 ASC NULLS FIRST], false + +(153) Scan parquet +Output [2]: [c_custkey#21, c_nationkey#22] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(56) InputAdapter +Input [2]: [c_custkey#21, c_nationkey#22] +Arguments: [#23, #24] + +(57) NativeFilter +Input [2]: [#23#21, #24#22] +Condition : (isnotnull(c_custkey#21) AND isnotnull(c_nationkey#22)) + +(58) NativeShuffleExchange +Input [2]: [#23#21, #24#22] +Arguments: hashpartitioning(c_custkey#21, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(59) ShuffleQueryStage +Output [2]: [#23#21, #24#22] +Arguments: X + +(60) AQEShuffleRead +Input [2]: [#23#21, #24#22] +Arguments: coalesced + +(61) InputAdapter +Input [2]: [#23#21, #24#22] + +(62) NativeSort +Input [2]: [#23#21, #24#22] +Arguments: [c_custkey#21 ASC NULLS FIRST], false + +(63) NativeSortMergeJoin +Left keys [1]: [o_custkey#17] +Right keys [1]: [c_custkey#21] +Join type: Inner +Join condition: None + +(64) NativeProject +Output [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22] +Input [7]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18, #23#21, #24#22] + +(65) NativeShuffleExchange +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22] +Arguments: hashpartitioning(c_nationkey#22, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(66) ShuffleQueryStage +Output [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22] +Arguments: X + +(67) AQEShuffleRead +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22] +Arguments: coalesced + +(68) InputAdapter +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22] + +(69) NativeSort +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22] +Arguments: [c_nationkey#22 ASC NULLS FIRST], false + +(161) Scan parquet +Output [2]: [n_nationkey#25, n_regionkey#26] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(71) InputAdapter +Input [2]: [n_nationkey#25, n_regionkey#26] +Arguments: [#27, #28] + +(72) NativeFilter +Input [2]: [#27#25, #28#26] +Condition : (isnotnull(n_nationkey#25) AND isnotnull(n_regionkey#26)) + +(73) NativeShuffleExchange +Input [2]: [#27#25, #28#26] +Arguments: hashpartitioning(n_nationkey#25, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(74) ShuffleQueryStage +Output [2]: [#27#25, #28#26] +Arguments: X + +(75) AQEShuffleRead +Input [2]: [#27#25, #28#26] +Arguments: coalesced + +(76) InputAdapter +Input [2]: [#27#25, #28#26] + +(77) NativeSort +Input [2]: [#27#25, #28#26] +Arguments: [n_nationkey#25 ASC NULLS FIRST], false + +(78) NativeSortMergeJoin +Left keys [1]: [c_nationkey#22] +Right keys [1]: [n_nationkey#25] +Join type: Inner +Join condition: None + +(79) NativeProject +Output [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26] +Input [7]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22, #27#25, #28#26] + +(80) NativeShuffleExchange +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26] +Arguments: hashpartitioning(s_nationkey#13, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(81) ShuffleQueryStage +Output [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26] +Arguments: X + +(82) AQEShuffleRead +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26] +Arguments: coalesced + +(83) InputAdapter +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26] + +(84) NativeSort +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26] +Arguments: [s_nationkey#13 ASC NULLS FIRST], false + +(169) Scan parquet +Output [2]: [n_nationkey#29, n_name#30] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(86) InputAdapter +Input [2]: [n_nationkey#29, n_name#30] +Arguments: [#31, #30] + +(87) NativeFilter +Input [2]: [#31#29, #30#30] +Condition : isnotnull(n_nationkey#29) + +(88) NativeShuffleExchange +Input [2]: [#31#29, #30#30] +Arguments: hashpartitioning(n_nationkey#29, 5), ENSURE_REQUIREMENTS, [plan_id=12] + +(89) ShuffleQueryStage +Output [2]: [#31#29, #30#30] +Arguments: X + +(90) AQEShuffleRead +Input [2]: [#31#29, #30#30] +Arguments: coalesced + +(91) InputAdapter +Input [2]: [#31#29, #30#30] + +(92) NativeSort +Input [2]: [#31#29, #30#30] +Arguments: [n_nationkey#29 ASC NULLS FIRST], false + +(93) NativeSortMergeJoin +Left keys [1]: [s_nationkey#13] +Right keys [1]: [n_nationkey#29] +Join type: Inner +Join condition: None + +(94) NativeProject +Output [5]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30] +Input [7]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26, #31#29, #30#30] + +(95) NativeShuffleExchange +Input [5]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30] +Arguments: hashpartitioning(n_regionkey#26, 5), ENSURE_REQUIREMENTS, [plan_id=13] + +(96) ShuffleQueryStage +Output [5]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30] +Arguments: X + +(97) AQEShuffleRead +Input [5]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30] +Arguments: coalesced + +(98) InputAdapter +Input [5]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30] + +(99) NativeSort +Input [5]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30] +Arguments: [n_regionkey#26 ASC NULLS FIRST], false + +(177) Scan parquet +Output [2]: [r_regionkey#32, r_name#33] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(101) InputAdapter +Input [2]: [r_regionkey#32, r_name#33] +Arguments: [#34, #33] + +(102) NativeFilter +Input [2]: [#34#32, #33#33] +Condition : ((isnotnull(r_name#33) AND (r_name#33 = AMERICA)) AND isnotnull(r_regionkey#32)) + +(103) NativeProject +Output [1]: [r_regionkey#32] +Input [2]: [#34#32, #33#33] + +(104) NativeShuffleExchange +Input [1]: [r_regionkey#32] +Arguments: hashpartitioning(r_regionkey#32, 5), ENSURE_REQUIREMENTS, [plan_id=14] + +(105) ShuffleQueryStage +Output [1]: [r_regionkey#32] +Arguments: X + +(106) AQEShuffleRead +Input [1]: [r_regionkey#32] +Arguments: coalesced + +(107) InputAdapter +Input [1]: [r_regionkey#32] + +(108) NativeSort +Input [1]: [r_regionkey#32] +Arguments: [r_regionkey#32 ASC NULLS FIRST], false + +(109) NativeSortMergeJoin +Left keys [1]: [n_regionkey#26] +Right keys [1]: [r_regionkey#32] +Join type: Inner +Join condition: None + +(110) NativeProject +Output [3]: [year(o_orderdate#18) AS o_year#35, (l_extendedprice#7 * (1 - l_discount#8)) AS volume#36, n_name#30 AS nation#37] +Input [6]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30, r_regionkey#32] + +(111) NativeProject +Output [3]: [o_year#35 AS o_year#35, CASE WHEN (nation#37 = BRAZIL) THEN volume#36 ELSE 0.0000 END AS _c1#38, volume#36 AS _c2#39] +Input [3]: [o_year#35, volume#36, nation#37] + +(112) NativeHashAggregate +Input [3]: [o_year#35, _c1#38, _c2#39] +Keys [1]: [o_year#35] +Functions [2]: [partial_sum(_c1#38), partial_sum(_c2#39)] +Aggregate Attributes [4]: [sum#40, isEmpty#41, sum#42, isEmpty#43] +Results [2]: [o_year#35, #44#44] + +(113) NativeShuffleExchange +Input [2]: [o_year#35, #44#44] +Arguments: hashpartitioning(o_year#35, 5), ENSURE_REQUIREMENTS, [plan_id=15] + +(114) ShuffleQueryStage +Output [2]: [o_year#35, #44#44] +Arguments: X + +(115) AQEShuffleRead +Input [2]: [o_year#35, #44#44] +Arguments: coalesced + +(116) InputAdapter +Input [2]: [o_year#35, #44#44] + +(117) NativeHashAggregate +Input [2]: [o_year#35, #44#44] +Keys [1]: [o_year#35] +Functions [2]: [sum(CASE WHEN (nation#37 = BRAZIL) THEN volume#36 ELSE 0.0000 END), sum(volume#36)] +Aggregate Attributes [2]: [sum(CASE WHEN (nation#37 = BRAZIL) THEN volume#36 ELSE 0.0000 END)#45, sum(volume#36)#46] +Results [3]: [o_year#35, sum(CASE WHEN (nation#37 = BRAZIL) THEN volume#36 ELSE 0.0000 END)#45, sum(volume#36)#46] + +(118) NativeProject +Output [2]: [o_year#35, (sum(CASE WHEN (nation#37 = BRAZIL) THEN volume#36 ELSE 0.0000 END)#45 / sum(volume#36)#46) AS mkt_share#47] +Input [3]: [o_year#35, sum(CASE WHEN (nation#37 = BRAZIL) THEN volume#36 ELSE 0.0000 END)#45, sum(volume#36)#46] + +(119) NativeShuffleExchange +Input [2]: [o_year#35, mkt_share#47] +Arguments: rangepartitioning(o_year#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=16] + +(120) ShuffleQueryStage +Output [2]: [o_year#35, mkt_share#47] +Arguments: X + +(121) AQEShuffleRead +Input [2]: [o_year#35, mkt_share#47] +Arguments: coalesced + +(122) InputAdapter +Input [2]: [o_year#35, mkt_share#47] + +(123) NativeSort +Input [2]: [o_year#35, mkt_share#47] +Arguments: [o_year#35 ASC NULLS FIRST], true + +(124) Scan parquet +Output [2]: [p_partkey#1, p_type#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] +ReadSchema: struct + +(125) Filter +Input [2]: [p_partkey#1, p_type#2] +Condition : ((isnotnull(p_type#2) AND (p_type#2 = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#1)) + +(126) Project +Output [1]: [p_partkey#1] +Input [2]: [p_partkey#1, p_type#2] + +(127) Exchange +Input [1]: [p_partkey#1] +Arguments: hashpartitioning(p_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=17] + +(128) Sort +Input [1]: [p_partkey#1] +Arguments: [p_partkey#1 ASC NULLS FIRST], false, 0 + +(129) Scan parquet +Output [5]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(130) Filter +Input [5]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8] +Condition : ((isnotnull(l_partkey#5) AND isnotnull(l_suppkey#6)) AND isnotnull(l_orderkey#4)) + +(131) Exchange +Input [5]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8] +Arguments: hashpartitioning(l_partkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=18] + +(132) Sort +Input [5]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8] +Arguments: [l_partkey#5 ASC NULLS FIRST], false, 0 + +(133) SortMergeJoin +Left keys [1]: [p_partkey#1] +Right keys [1]: [l_partkey#5] +Join type: Inner +Join condition: None + +(134) Project +Output [4]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8] +Input [6]: [p_partkey#1, l_orderkey#4, l_partkey#5, l_suppkey#6, l_extendedprice#7, l_discount#8] + +(135) Exchange +Input [4]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8] +Arguments: hashpartitioning(l_suppkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=19] + +(136) Sort +Input [4]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8] +Arguments: [l_suppkey#6 ASC NULLS FIRST], false, 0 + +(137) Scan parquet +Output [2]: [s_suppkey#12, s_nationkey#13] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(138) Filter +Input [2]: [s_suppkey#12, s_nationkey#13] +Condition : (isnotnull(s_suppkey#12) AND isnotnull(s_nationkey#13)) + +(139) Exchange +Input [2]: [s_suppkey#12, s_nationkey#13] +Arguments: hashpartitioning(s_suppkey#12, 5), ENSURE_REQUIREMENTS, [plan_id=20] + +(140) Sort +Input [2]: [s_suppkey#12, s_nationkey#13] +Arguments: [s_suppkey#12 ASC NULLS FIRST], false, 0 + +(141) SortMergeJoin +Left keys [1]: [l_suppkey#6] +Right keys [1]: [s_suppkey#12] +Join type: Inner +Join condition: None + +(142) Project +Output [4]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13] +Input [6]: [l_orderkey#4, l_suppkey#6, l_extendedprice#7, l_discount#8, s_suppkey#12, s_nationkey#13] + +(143) Exchange +Input [4]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13] +Arguments: hashpartitioning(l_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=21] + +(144) Sort +Input [4]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13] +Arguments: [l_orderkey#4 ASC NULLS FIRST], false, 0 + +(145) Scan parquet +Output [3]: [o_orderkey#16, o_custkey#17, o_orderdate#18] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] +ReadSchema: struct + +(146) Filter +Input [3]: [o_orderkey#16, o_custkey#17, o_orderdate#18] +Condition : ((((isnotnull(o_orderdate#18) AND (o_orderdate#18 >= 1995-01-01)) AND (o_orderdate#18 <= 1996-12-31)) AND isnotnull(o_orderkey#16)) AND isnotnull(o_custkey#17)) + +(147) Exchange +Input [3]: [o_orderkey#16, o_custkey#17, o_orderdate#18] +Arguments: hashpartitioning(o_orderkey#16, 5), ENSURE_REQUIREMENTS, [plan_id=22] + +(148) Sort +Input [3]: [o_orderkey#16, o_custkey#17, o_orderdate#18] +Arguments: [o_orderkey#16 ASC NULLS FIRST], false, 0 + +(149) SortMergeJoin +Left keys [1]: [l_orderkey#4] +Right keys [1]: [o_orderkey#16] +Join type: Inner +Join condition: None + +(150) Project +Output [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18] +Input [7]: [l_orderkey#4, l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderkey#16, o_custkey#17, o_orderdate#18] + +(151) Exchange +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18] +Arguments: hashpartitioning(o_custkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=23] + +(152) Sort +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18] +Arguments: [o_custkey#17 ASC NULLS FIRST], false, 0 + +(153) Scan parquet +Output [2]: [c_custkey#21, c_nationkey#22] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] +ReadSchema: struct + +(154) Filter +Input [2]: [c_custkey#21, c_nationkey#22] +Condition : (isnotnull(c_custkey#21) AND isnotnull(c_nationkey#22)) + +(155) Exchange +Input [2]: [c_custkey#21, c_nationkey#22] +Arguments: hashpartitioning(c_custkey#21, 5), ENSURE_REQUIREMENTS, [plan_id=24] + +(156) Sort +Input [2]: [c_custkey#21, c_nationkey#22] +Arguments: [c_custkey#21 ASC NULLS FIRST], false, 0 + +(157) SortMergeJoin +Left keys [1]: [o_custkey#17] +Right keys [1]: [c_custkey#21] +Join type: Inner +Join condition: None + +(158) Project +Output [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22] +Input [7]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_custkey#17, o_orderdate#18, c_custkey#21, c_nationkey#22] + +(159) Exchange +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22] +Arguments: hashpartitioning(c_nationkey#22, 5), ENSURE_REQUIREMENTS, [plan_id=25] + +(160) Sort +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22] +Arguments: [c_nationkey#22 ASC NULLS FIRST], false, 0 + +(161) Scan parquet +Output [2]: [n_nationkey#25, n_regionkey#26] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] +ReadSchema: struct + +(162) Filter +Input [2]: [n_nationkey#25, n_regionkey#26] +Condition : (isnotnull(n_nationkey#25) AND isnotnull(n_regionkey#26)) + +(163) Exchange +Input [2]: [n_nationkey#25, n_regionkey#26] +Arguments: hashpartitioning(n_nationkey#25, 5), ENSURE_REQUIREMENTS, [plan_id=26] + +(164) Sort +Input [2]: [n_nationkey#25, n_regionkey#26] +Arguments: [n_nationkey#25 ASC NULLS FIRST], false, 0 + +(165) SortMergeJoin +Left keys [1]: [c_nationkey#22] +Right keys [1]: [n_nationkey#25] +Join type: Inner +Join condition: None + +(166) Project +Output [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26] +Input [7]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, c_nationkey#22, n_nationkey#25, n_regionkey#26] + +(167) Exchange +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26] +Arguments: hashpartitioning(s_nationkey#13, 5), ENSURE_REQUIREMENTS, [plan_id=27] + +(168) Sort +Input [5]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26] +Arguments: [s_nationkey#13 ASC NULLS FIRST], false, 0 + +(169) Scan parquet +Output [2]: [n_nationkey#29, n_name#30] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(170) Filter +Input [2]: [n_nationkey#29, n_name#30] +Condition : isnotnull(n_nationkey#29) + +(171) Exchange +Input [2]: [n_nationkey#29, n_name#30] +Arguments: hashpartitioning(n_nationkey#29, 5), ENSURE_REQUIREMENTS, [plan_id=28] + +(172) Sort +Input [2]: [n_nationkey#29, n_name#30] +Arguments: [n_nationkey#29 ASC NULLS FIRST], false, 0 + +(173) SortMergeJoin +Left keys [1]: [s_nationkey#13] +Right keys [1]: [n_nationkey#29] +Join type: Inner +Join condition: None + +(174) Project +Output [5]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30] +Input [7]: [l_extendedprice#7, l_discount#8, s_nationkey#13, o_orderdate#18, n_regionkey#26, n_nationkey#29, n_name#30] + +(175) Exchange +Input [5]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30] +Arguments: hashpartitioning(n_regionkey#26, 5), ENSURE_REQUIREMENTS, [plan_id=29] + +(176) Sort +Input [5]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30] +Arguments: [n_regionkey#26 ASC NULLS FIRST], false, 0 + +(177) Scan parquet +Output [2]: [r_regionkey#32, r_name#33] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] +ReadSchema: struct + +(178) Filter +Input [2]: [r_regionkey#32, r_name#33] +Condition : ((isnotnull(r_name#33) AND (r_name#33 = AMERICA)) AND isnotnull(r_regionkey#32)) + +(179) Project +Output [1]: [r_regionkey#32] +Input [2]: [r_regionkey#32, r_name#33] + +(180) Exchange +Input [1]: [r_regionkey#32] +Arguments: hashpartitioning(r_regionkey#32, 5), ENSURE_REQUIREMENTS, [plan_id=30] + +(181) Sort +Input [1]: [r_regionkey#32] +Arguments: [r_regionkey#32 ASC NULLS FIRST], false, 0 + +(182) SortMergeJoin +Left keys [1]: [n_regionkey#26] +Right keys [1]: [r_regionkey#32] +Join type: Inner +Join condition: None + +(183) Project +Output [3]: [year(o_orderdate#18) AS o_year#35, (l_extendedprice#7 * (1 - l_discount#8)) AS volume#36, n_name#30 AS nation#37] +Input [6]: [l_extendedprice#7, l_discount#8, o_orderdate#18, n_regionkey#26, n_name#30, r_regionkey#32] + +(184) HashAggregate +Input [3]: [o_year#35, volume#36, nation#37] +Keys [1]: [o_year#35] +Functions [2]: [partial_sum(CASE WHEN (nation#37 = BRAZIL) THEN volume#36 ELSE 0.0000 END), partial_sum(volume#36)] +Aggregate Attributes [4]: [sum#40, isEmpty#41, sum#42, isEmpty#43] +Results [5]: [o_year#35, sum#48, isEmpty#49, sum#50, isEmpty#51] + +(185) Exchange +Input [5]: [o_year#35, sum#48, isEmpty#49, sum#50, isEmpty#51] +Arguments: hashpartitioning(o_year#35, 5), ENSURE_REQUIREMENTS, [plan_id=31] + +(186) HashAggregate +Input [5]: [o_year#35, sum#48, isEmpty#49, sum#50, isEmpty#51] +Keys [1]: [o_year#35] +Functions [2]: [sum(CASE WHEN (nation#37 = BRAZIL) THEN volume#36 ELSE 0.0000 END), sum(volume#36)] +Aggregate Attributes [2]: [sum(CASE WHEN (nation#37 = BRAZIL) THEN volume#36 ELSE 0.0000 END)#45, sum(volume#36)#46] +Results [2]: [o_year#35, (sum(CASE WHEN (nation#37 = BRAZIL) THEN volume#36 ELSE 0.0000 END)#45 / sum(volume#36)#46) AS mkt_share#47] + +(187) Exchange +Input [2]: [o_year#35, mkt_share#47] +Arguments: rangepartitioning(o_year#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=32] + +(188) Sort +Input [2]: [o_year#35, mkt_share#47] +Arguments: [o_year#35 ASC NULLS FIRST], true, 0 + +(189) AdaptiveSparkPlan +Output [2]: [o_year#35, mkt_share#47] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q9.txt b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q9.txt new file mode 100644 index 000000000..831ed4dba --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-plan-stability/q9.txt @@ -0,0 +1,761 @@ +== Physical Plan == +AdaptiveSparkPlan (140) ++- == Final Plan == + NativeSort (91) + +- InputAdapter (90) + +- AQEShuffleRead (89) + +- ShuffleQueryStage (88), Statistics(sizeInBytes=5.9 KiB, rowCount=173) + +- NativeShuffleExchange (87) + +- NativeProject (86) + +- NativeHashAggregate (85) + +- InputAdapter (84) + +- AQEShuffleRead (83) + +- ShuffleQueryStage (82), Statistics(sizeInBytes=7.9 KiB, rowCount=173) + +- NativeShuffleExchange (81) + +- NativeHashAggregate (80) + +- NativeProject (79) + +- NativeSortMergeJoin Inner (78) + :- NativeSort (69) + : +- InputAdapter (68) + : +- AQEShuffleRead (67) + : +- ShuffleQueryStage (66), Statistics(sizeInBytes=240.4 KiB, rowCount=3.22E+3) + : +- NativeShuffleExchange (65) + : +- NativeProject (64) + : +- NativeSortMergeJoin Inner (63) + : :- NativeSort (54) + : : +- InputAdapter (53) + : : +- AQEShuffleRead (52) + : : +- ShuffleQueryStage (51), Statistics(sizeInBytes=252.9 KiB, rowCount=3.22E+3) + : : +- NativeShuffleExchange (50) + : : +- NativeProject (49) + : : +- NativeSortMergeJoin Inner (48) + : : :- NativeSort (39) + : : : +- InputAdapter (38) + : : : +- AQEShuffleRead (37) + : : : +- ShuffleQueryStage (36), Statistics(sizeInBytes=253.1 KiB, rowCount=3.22E+3) + : : : +- NativeShuffleExchange (35) + : : : +- NativeProject (34) + : : : +- NativeSortMergeJoin Inner (33) + : : : :- NativeSort (24) + : : : : +- InputAdapter (23) + : : : : +- AQEShuffleRead (22) + : : : : +- ShuffleQueryStage (21), Statistics(sizeInBytes=227.8 KiB, rowCount=3.22E+3) + : : : : +- NativeShuffleExchange (20) + : : : : +- NativeProject (19) + : : : : +- NativeSortMergeJoin Inner (18) + : : : : :- NativeSort (9) + : : : : : +- InputAdapter (8) + : : : : : +- AQEShuffleRead (7) + : : : : : +- ShuffleQueryStage (6), Statistics(sizeInBytes=1064.0 B, rowCount=107) + : : : : : +- NativeShuffleExchange (5) + : : : : : +- NativeProject (4) + : : : : : +- NativeFilter (3) + : : : : : +- InputAdapter (2) + : : : : : +- NativeParquetScan (1) + : : : : +- NativeSort (17) + : : : : +- InputAdapter (16) + : : : : +- AQEShuffleRead (15) + : : : : +- ShuffleQueryStage (14), Statistics(sizeInBytes=4.1 MiB, rowCount=6.02E+4) + : : : : +- NativeShuffleExchange (13) + : : : : +- NativeFilter (12) + : : : : +- InputAdapter (11) + : : : : +- NativeParquetScan (10) + : : : +- NativeSort (32) + : : : +- InputAdapter (31) + : : : +- AQEShuffleRead (30) + : : : +- ShuffleQueryStage (29), Statistics(sizeInBytes=1992.0 B, rowCount=100) + : : : +- NativeShuffleExchange (28) + : : : +- NativeFilter (27) + : : : +- InputAdapter (26) + : : : +- NativeParquetScan (25) + : : +- NativeSort (47) + : : +- InputAdapter (46) + : : +- AQEShuffleRead (45) + : : +- ShuffleQueryStage (44), Statistics(sizeInBytes=250.6 KiB, rowCount=8.00E+3) + : : +- NativeShuffleExchange (43) + : : +- NativeFilter (42) + : : +- InputAdapter (41) + : : +- NativeParquetScan (40) + : +- NativeSort (62) + : +- InputAdapter (61) + : +- AQEShuffleRead (60) + : +- ShuffleQueryStage (59), Statistics(sizeInBytes=176.5 KiB, rowCount=1.50E+4) + : +- NativeShuffleExchange (58) + : +- NativeFilter (57) + : +- InputAdapter (56) + : +- NativeParquetScan (55) + +- NativeSort (77) + +- InputAdapter (76) + +- AQEShuffleRead (75) + +- ShuffleQueryStage (74), Statistics(sizeInBytes=997.0 B, rowCount=25) + +- NativeShuffleExchange (73) + +- NativeFilter (72) + +- InputAdapter (71) + +- NativeParquetScan (70) ++- == Initial Plan == + Sort (139) + +- Exchange (138) + +- HashAggregate (137) + +- Exchange (136) + +- HashAggregate (135) + +- Project (134) + +- SortMergeJoin Inner (133) + :- Sort (128) + : +- Exchange (127) + : +- Project (126) + : +- SortMergeJoin Inner (125) + : :- Sort (120) + : : +- Exchange (119) + : : +- Project (118) + : : +- SortMergeJoin Inner (117) + : : :- Sort (112) + : : : +- Exchange (111) + : : : +- Project (110) + : : : +- SortMergeJoin Inner (109) + : : : :- Sort (104) + : : : : +- Exchange (103) + : : : : +- Project (102) + : : : : +- SortMergeJoin Inner (101) + : : : : :- Sort (96) + : : : : : +- Exchange (95) + : : : : : +- Project (94) + : : : : : +- Filter (93) + : : : : : +- Scan parquet (92) + : : : : +- Sort (100) + : : : : +- Exchange (99) + : : : : +- Filter (98) + : : : : +- Scan parquet (97) + : : : +- Sort (108) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Sort (116) + : : +- Exchange (115) + : : +- Filter (114) + : : +- Scan parquet (113) + : +- Sort (124) + : +- Exchange (123) + : +- Filter (122) + : +- Scan parquet (121) + +- Sort (132) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) + + +(92) Scan parquet +Output [2]: [p_partkey#1, p_name#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] +ReadSchema: struct + +(2) InputAdapter +Input [2]: [p_partkey#1, p_name#2] +Arguments: [#3, #2] + +(3) NativeFilter +Input [2]: [#3#1, #2#2] +Condition : ((isnotnull(p_name#2) AND Contains(p_name#2, green)) AND isnotnull(p_partkey#1)) + +(4) NativeProject +Output [1]: [p_partkey#1] +Input [2]: [#3#1, #2#2] + +(5) NativeShuffleExchange +Input [1]: [p_partkey#1] +Arguments: hashpartitioning(p_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] + +(6) ShuffleQueryStage +Output [1]: [p_partkey#1] +Arguments: X + +(7) AQEShuffleRead +Input [1]: [p_partkey#1] +Arguments: coalesced + +(8) InputAdapter +Input [1]: [p_partkey#1] + +(9) NativeSort +Input [1]: [p_partkey#1] +Arguments: [p_partkey#1 ASC NULLS FIRST], false + +(97) Scan parquet +Output [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(11) InputAdapter +Input [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Arguments: [#10, #11, #12, #7, #8, #9] + +(12) NativeFilter +Input [6]: [#10#4, #11#5, #12#6, #7#7, #8#8, #9#9] +Condition : ((isnotnull(l_partkey#5) AND isnotnull(l_suppkey#6)) AND isnotnull(l_orderkey#4)) + +(13) NativeShuffleExchange +Input [6]: [#10#4, #11#5, #12#6, #7#7, #8#8, #9#9] +Arguments: hashpartitioning(l_partkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] + +(14) ShuffleQueryStage +Output [6]: [#10#4, #11#5, #12#6, #7#7, #8#8, #9#9] +Arguments: X + +(15) AQEShuffleRead +Input [6]: [#10#4, #11#5, #12#6, #7#7, #8#8, #9#9] +Arguments: coalesced + +(16) InputAdapter +Input [6]: [#10#4, #11#5, #12#6, #7#7, #8#8, #9#9] + +(17) NativeSort +Input [6]: [#10#4, #11#5, #12#6, #7#7, #8#8, #9#9] +Arguments: [l_partkey#5 ASC NULLS FIRST], false + +(18) NativeSortMergeJoin +Left keys [1]: [p_partkey#1] +Right keys [1]: [l_partkey#5] +Join type: Inner +Join condition: None + +(19) NativeProject +Output [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Input [7]: [p_partkey#1, #10#4, #11#5, #12#6, #7#7, #8#8, #9#9] + +(20) NativeShuffleExchange +Input [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Arguments: hashpartitioning(l_suppkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] + +(21) ShuffleQueryStage +Output [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Arguments: X + +(22) AQEShuffleRead +Input [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Arguments: coalesced + +(23) InputAdapter +Input [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] + +(24) NativeSort +Input [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Arguments: [l_suppkey#6 ASC NULLS FIRST], false + +(105) Scan parquet +Output [2]: [s_suppkey#13, s_nationkey#14] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(26) InputAdapter +Input [2]: [s_suppkey#13, s_nationkey#14] +Arguments: [#15, #16] + +(27) NativeFilter +Input [2]: [#15#13, #16#14] +Condition : (isnotnull(s_suppkey#13) AND isnotnull(s_nationkey#14)) + +(28) NativeShuffleExchange +Input [2]: [#15#13, #16#14] +Arguments: hashpartitioning(s_suppkey#13, 5), ENSURE_REQUIREMENTS, [plan_id=4] + +(29) ShuffleQueryStage +Output [2]: [#15#13, #16#14] +Arguments: X + +(30) AQEShuffleRead +Input [2]: [#15#13, #16#14] +Arguments: coalesced + +(31) InputAdapter +Input [2]: [#15#13, #16#14] + +(32) NativeSort +Input [2]: [#15#13, #16#14] +Arguments: [s_suppkey#13 ASC NULLS FIRST], false + +(33) NativeSortMergeJoin +Left keys [1]: [l_suppkey#6] +Right keys [1]: [s_suppkey#13] +Join type: Inner +Join condition: None + +(34) NativeProject +Output [7]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14] +Input [8]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, #15#13, #16#14] + +(35) NativeShuffleExchange +Input [7]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14] +Arguments: hashpartitioning(l_suppkey#6, l_partkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] + +(36) ShuffleQueryStage +Output [7]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14] +Arguments: X + +(37) AQEShuffleRead +Input [7]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14] +Arguments: coalesced + +(38) InputAdapter +Input [7]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14] + +(39) NativeSort +Input [7]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14] +Arguments: [l_suppkey#6 ASC NULLS FIRST, l_partkey#5 ASC NULLS FIRST], false + +(113) Scan parquet +Output [3]: [ps_partkey#17, ps_suppkey#18, ps_supplycost#19] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(41) InputAdapter +Input [3]: [ps_partkey#17, ps_suppkey#18, ps_supplycost#19] +Arguments: [#20, #21, #19] + +(42) NativeFilter +Input [3]: [#20#17, #21#18, #19#19] +Condition : (isnotnull(ps_suppkey#18) AND isnotnull(ps_partkey#17)) + +(43) NativeShuffleExchange +Input [3]: [#20#17, #21#18, #19#19] +Arguments: hashpartitioning(ps_suppkey#18, ps_partkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=6] + +(44) ShuffleQueryStage +Output [3]: [#20#17, #21#18, #19#19] +Arguments: X + +(45) AQEShuffleRead +Input [3]: [#20#17, #21#18, #19#19] +Arguments: coalesced + +(46) InputAdapter +Input [3]: [#20#17, #21#18, #19#19] + +(47) NativeSort +Input [3]: [#20#17, #21#18, #19#19] +Arguments: [ps_suppkey#18 ASC NULLS FIRST, ps_partkey#17 ASC NULLS FIRST], false + +(48) NativeSortMergeJoin +Left keys [2]: [l_suppkey#6, l_partkey#5] +Right keys [2]: [ps_suppkey#18, ps_partkey#17] +Join type: Inner +Join condition: None + +(49) NativeProject +Output [6]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19] +Input [10]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, #20#17, #21#18, #19#19] + +(50) NativeShuffleExchange +Input [6]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19] +Arguments: hashpartitioning(l_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=7] + +(51) ShuffleQueryStage +Output [6]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19] +Arguments: X + +(52) AQEShuffleRead +Input [6]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19] +Arguments: coalesced + +(53) InputAdapter +Input [6]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19] + +(54) NativeSort +Input [6]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19] +Arguments: [l_orderkey#4 ASC NULLS FIRST], false + +(121) Scan parquet +Output [2]: [o_orderkey#22, o_orderdate#23] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(56) InputAdapter +Input [2]: [o_orderkey#22, o_orderdate#23] +Arguments: [#24, #23] + +(57) NativeFilter +Input [2]: [#24#22, #23#23] +Condition : isnotnull(o_orderkey#22) + +(58) NativeShuffleExchange +Input [2]: [#24#22, #23#23] +Arguments: hashpartitioning(o_orderkey#22, 5), ENSURE_REQUIREMENTS, [plan_id=8] + +(59) ShuffleQueryStage +Output [2]: [#24#22, #23#23] +Arguments: X + +(60) AQEShuffleRead +Input [2]: [#24#22, #23#23] +Arguments: coalesced + +(61) InputAdapter +Input [2]: [#24#22, #23#23] + +(62) NativeSort +Input [2]: [#24#22, #23#23] +Arguments: [o_orderkey#22 ASC NULLS FIRST], false + +(63) NativeSortMergeJoin +Left keys [1]: [l_orderkey#4] +Right keys [1]: [o_orderkey#22] +Join type: Inner +Join condition: None + +(64) NativeProject +Output [6]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23] +Input [8]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, #24#22, #23#23] + +(65) NativeShuffleExchange +Input [6]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23] +Arguments: hashpartitioning(s_nationkey#14, 5), ENSURE_REQUIREMENTS, [plan_id=9] + +(66) ShuffleQueryStage +Output [6]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23] +Arguments: X + +(67) AQEShuffleRead +Input [6]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23] +Arguments: coalesced + +(68) InputAdapter +Input [6]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23] + +(69) NativeSort +Input [6]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23] +Arguments: [s_nationkey#14 ASC NULLS FIRST], false + +(129) Scan parquet +Output [2]: [n_nationkey#25, n_name#26] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(71) InputAdapter +Input [2]: [n_nationkey#25, n_name#26] +Arguments: [#27, #26] + +(72) NativeFilter +Input [2]: [#27#25, #26#26] +Condition : isnotnull(n_nationkey#25) + +(73) NativeShuffleExchange +Input [2]: [#27#25, #26#26] +Arguments: hashpartitioning(n_nationkey#25, 5), ENSURE_REQUIREMENTS, [plan_id=10] + +(74) ShuffleQueryStage +Output [2]: [#27#25, #26#26] +Arguments: X + +(75) AQEShuffleRead +Input [2]: [#27#25, #26#26] +Arguments: coalesced + +(76) InputAdapter +Input [2]: [#27#25, #26#26] + +(77) NativeSort +Input [2]: [#27#25, #26#26] +Arguments: [n_nationkey#25 ASC NULLS FIRST], false + +(78) NativeSortMergeJoin +Left keys [1]: [s_nationkey#14] +Right keys [1]: [n_nationkey#25] +Join type: Inner +Join condition: None + +(79) NativeProject +Output [3]: [n_name#26 AS nation#28, year(o_orderdate#23) AS o_year#29, ((l_extendedprice#8 * (1 - l_discount#9)) - (ps_supplycost#19 * l_quantity#7)) AS amount#30] +Input [8]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23, #27#25, #26#26] + +(80) NativeHashAggregate +Input [3]: [nation#28, o_year#29, amount#30] +Keys [2]: [nation#28, o_year#29] +Functions [1]: [partial_sum(amount#30)] +Aggregate Attributes [2]: [sum#31, isEmpty#32] +Results [3]: [nation#28, o_year#29, #33#33] + +(81) NativeShuffleExchange +Input [3]: [nation#28, o_year#29, #33#33] +Arguments: hashpartitioning(nation#28, o_year#29, 5), ENSURE_REQUIREMENTS, [plan_id=11] + +(82) ShuffleQueryStage +Output [3]: [nation#28, o_year#29, #33#33] +Arguments: X + +(83) AQEShuffleRead +Input [3]: [nation#28, o_year#29, #33#33] +Arguments: coalesced + +(84) InputAdapter +Input [3]: [nation#28, o_year#29, #33#33] + +(85) NativeHashAggregate +Input [3]: [nation#28, o_year#29, #33#33] +Keys [2]: [nation#28, o_year#29] +Functions [1]: [sum(amount#30)] +Aggregate Attributes [1]: [sum(amount#30)#34] +Results [3]: [nation#28, o_year#29, sum(amount#30)#34] + +(86) NativeProject +Output [3]: [nation#28, o_year#29, sum(amount#30)#34 AS sum_profit#35] +Input [3]: [nation#28, o_year#29, sum(amount#30)#34] + +(87) NativeShuffleExchange +Input [3]: [nation#28, o_year#29, sum_profit#35] +Arguments: rangepartitioning(nation#28 ASC NULLS FIRST, o_year#29 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=12] + +(88) ShuffleQueryStage +Output [3]: [nation#28, o_year#29, sum_profit#35] +Arguments: X + +(89) AQEShuffleRead +Input [3]: [nation#28, o_year#29, sum_profit#35] +Arguments: coalesced + +(90) InputAdapter +Input [3]: [nation#28, o_year#29, sum_profit#35] + +(91) NativeSort +Input [3]: [nation#28, o_year#29, sum_profit#35] +Arguments: [nation#28 ASC NULLS FIRST, o_year#29 DESC NULLS LAST], true + +(92) Scan parquet +Output [2]: [p_partkey#1, p_name#2] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] +ReadSchema: struct + +(93) Filter +Input [2]: [p_partkey#1, p_name#2] +Condition : ((isnotnull(p_name#2) AND Contains(p_name#2, green)) AND isnotnull(p_partkey#1)) + +(94) Project +Output [1]: [p_partkey#1] +Input [2]: [p_partkey#1, p_name#2] + +(95) Exchange +Input [1]: [p_partkey#1] +Arguments: hashpartitioning(p_partkey#1, 5), ENSURE_REQUIREMENTS, [plan_id=13] + +(96) Sort +Input [1]: [p_partkey#1] +Arguments: [p_partkey#1 ASC NULLS FIRST], false, 0 + +(97) Scan parquet +Output [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] +ReadSchema: struct + +(98) Filter +Input [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Condition : ((isnotnull(l_partkey#5) AND isnotnull(l_suppkey#6)) AND isnotnull(l_orderkey#4)) + +(99) Exchange +Input [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Arguments: hashpartitioning(l_partkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=14] + +(100) Sort +Input [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Arguments: [l_partkey#5 ASC NULLS FIRST], false, 0 + +(101) SortMergeJoin +Left keys [1]: [p_partkey#1] +Right keys [1]: [l_partkey#5] +Join type: Inner +Join condition: None + +(102) Project +Output [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Input [7]: [p_partkey#1, l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] + +(103) Exchange +Input [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Arguments: hashpartitioning(l_suppkey#6, 5), ENSURE_REQUIREMENTS, [plan_id=15] + +(104) Sort +Input [6]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9] +Arguments: [l_suppkey#6 ASC NULLS FIRST], false, 0 + +(105) Scan parquet +Output [2]: [s_suppkey#13, s_nationkey#14] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] +ReadSchema: struct + +(106) Filter +Input [2]: [s_suppkey#13, s_nationkey#14] +Condition : (isnotnull(s_suppkey#13) AND isnotnull(s_nationkey#14)) + +(107) Exchange +Input [2]: [s_suppkey#13, s_nationkey#14] +Arguments: hashpartitioning(s_suppkey#13, 5), ENSURE_REQUIREMENTS, [plan_id=16] + +(108) Sort +Input [2]: [s_suppkey#13, s_nationkey#14] +Arguments: [s_suppkey#13 ASC NULLS FIRST], false, 0 + +(109) SortMergeJoin +Left keys [1]: [l_suppkey#6] +Right keys [1]: [s_suppkey#13] +Join type: Inner +Join condition: None + +(110) Project +Output [7]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14] +Input [8]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_suppkey#13, s_nationkey#14] + +(111) Exchange +Input [7]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14] +Arguments: hashpartitioning(l_suppkey#6, l_partkey#5, 5), ENSURE_REQUIREMENTS, [plan_id=17] + +(112) Sort +Input [7]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14] +Arguments: [l_suppkey#6 ASC NULLS FIRST, l_partkey#5 ASC NULLS FIRST], false, 0 + +(113) Scan parquet +Output [3]: [ps_partkey#17, ps_suppkey#18, ps_supplycost#19] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] +ReadSchema: struct + +(114) Filter +Input [3]: [ps_partkey#17, ps_suppkey#18, ps_supplycost#19] +Condition : (isnotnull(ps_suppkey#18) AND isnotnull(ps_partkey#17)) + +(115) Exchange +Input [3]: [ps_partkey#17, ps_suppkey#18, ps_supplycost#19] +Arguments: hashpartitioning(ps_suppkey#18, ps_partkey#17, 5), ENSURE_REQUIREMENTS, [plan_id=18] + +(116) Sort +Input [3]: [ps_partkey#17, ps_suppkey#18, ps_supplycost#19] +Arguments: [ps_suppkey#18 ASC NULLS FIRST, ps_partkey#17 ASC NULLS FIRST], false, 0 + +(117) SortMergeJoin +Left keys [2]: [l_suppkey#6, l_partkey#5] +Right keys [2]: [ps_suppkey#18, ps_partkey#17] +Join type: Inner +Join condition: None + +(118) Project +Output [6]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19] +Input [10]: [l_orderkey#4, l_partkey#5, l_suppkey#6, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_partkey#17, ps_suppkey#18, ps_supplycost#19] + +(119) Exchange +Input [6]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19] +Arguments: hashpartitioning(l_orderkey#4, 5), ENSURE_REQUIREMENTS, [plan_id=19] + +(120) Sort +Input [6]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19] +Arguments: [l_orderkey#4 ASC NULLS FIRST], false, 0 + +(121) Scan parquet +Output [2]: [o_orderkey#22, o_orderdate#23] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(o_orderkey)] +ReadSchema: struct + +(122) Filter +Input [2]: [o_orderkey#22, o_orderdate#23] +Condition : isnotnull(o_orderkey#22) + +(123) Exchange +Input [2]: [o_orderkey#22, o_orderdate#23] +Arguments: hashpartitioning(o_orderkey#22, 5), ENSURE_REQUIREMENTS, [plan_id=20] + +(124) Sort +Input [2]: [o_orderkey#22, o_orderdate#23] +Arguments: [o_orderkey#22 ASC NULLS FIRST], false, 0 + +(125) SortMergeJoin +Left keys [1]: [l_orderkey#4] +Right keys [1]: [o_orderkey#22] +Join type: Inner +Join condition: None + +(126) Project +Output [6]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23] +Input [8]: [l_orderkey#4, l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderkey#22, o_orderdate#23] + +(127) Exchange +Input [6]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23] +Arguments: hashpartitioning(s_nationkey#14, 5), ENSURE_REQUIREMENTS, [plan_id=21] + +(128) Sort +Input [6]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23] +Arguments: [s_nationkey#14 ASC NULLS FIRST], false, 0 + +(129) Scan parquet +Output [2]: [n_nationkey#25, n_name#26] +Batched: true +Location: InMemoryFileIndex [file:/] +PushedFilters: [IsNotNull(n_nationkey)] +ReadSchema: struct + +(130) Filter +Input [2]: [n_nationkey#25, n_name#26] +Condition : isnotnull(n_nationkey#25) + +(131) Exchange +Input [2]: [n_nationkey#25, n_name#26] +Arguments: hashpartitioning(n_nationkey#25, 5), ENSURE_REQUIREMENTS, [plan_id=22] + +(132) Sort +Input [2]: [n_nationkey#25, n_name#26] +Arguments: [n_nationkey#25 ASC NULLS FIRST], false, 0 + +(133) SortMergeJoin +Left keys [1]: [s_nationkey#14] +Right keys [1]: [n_nationkey#25] +Join type: Inner +Join condition: None + +(134) Project +Output [3]: [n_name#26 AS nation#28, year(o_orderdate#23) AS o_year#29, ((l_extendedprice#8 * (1 - l_discount#9)) - (ps_supplycost#19 * l_quantity#7)) AS amount#30] +Input [8]: [l_quantity#7, l_extendedprice#8, l_discount#9, s_nationkey#14, ps_supplycost#19, o_orderdate#23, n_nationkey#25, n_name#26] + +(135) HashAggregate +Input [3]: [nation#28, o_year#29, amount#30] +Keys [2]: [nation#28, o_year#29] +Functions [1]: [partial_sum(amount#30)] +Aggregate Attributes [2]: [sum#31, isEmpty#32] +Results [4]: [nation#28, o_year#29, sum#36, isEmpty#37] + +(136) Exchange +Input [4]: [nation#28, o_year#29, sum#36, isEmpty#37] +Arguments: hashpartitioning(nation#28, o_year#29, 5), ENSURE_REQUIREMENTS, [plan_id=23] + +(137) HashAggregate +Input [4]: [nation#28, o_year#29, sum#36, isEmpty#37] +Keys [2]: [nation#28, o_year#29] +Functions [1]: [sum(amount#30)] +Aggregate Attributes [1]: [sum(amount#30)#34] +Results [3]: [nation#28, o_year#29, sum(amount#30)#34 AS sum_profit#35] + +(138) Exchange +Input [3]: [nation#28, o_year#29, sum_profit#35] +Arguments: rangepartitioning(nation#28 ASC NULLS FIRST, o_year#29 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=24] + +(139) Sort +Input [3]: [nation#28, o_year#29, sum_profit#35] +Arguments: [nation#28 ASC NULLS FIRST, o_year#29 DESC NULLS LAST], true, 0 + +(140) AdaptiveSparkPlan +Output [3]: [nation#28, o_year#29, sum_profit#35] +Arguments: isFinalPlan=true + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q1.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q1.sql new file mode 100644 index 000000000..97c62bb18 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q1.sql @@ -0,0 +1,22 @@ + +select + l_returnflag, + l_linestatus, + sum(l_quantity) as sum_qty, + sum(l_extendedprice) as sum_base_price, + sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, + avg(l_quantity) as avg_qty, + avg(l_extendedprice) as avg_price, + avg(l_discount) as avg_disc, + count(*) as count_order +from + lineitem +where + l_shipdate <= date '1998-12-01' - interval '90' day +group by + l_returnflag, + l_linestatus +order by + l_returnflag, + l_linestatus diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q10.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q10.sql new file mode 100644 index 000000000..7a7cb0577 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q10.sql @@ -0,0 +1,33 @@ + +select + c_custkey, + c_name, + sum(l_extendedprice * (1 - l_discount)) as revenue, + c_acctbal, + n_name, + c_address, + c_phone, + c_comment +from + customer, + orders, + lineitem, + nation +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate >= date '1993-10-01' + and o_orderdate < date '1993-10-01' + interval '3' month + and l_returnflag = 'R' + and c_nationkey = n_nationkey +group by + c_custkey, + c_name, + c_acctbal, + c_phone, + n_name, + c_address, + c_comment +order by + revenue desc +limit 20 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q11.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q11.sql new file mode 100644 index 000000000..17ed8fa2c --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q11.sql @@ -0,0 +1,28 @@ + +select + ps_partkey, + sum(ps_supplycost * ps_availqty) as value +from + partsupp, + supplier, + nation +where + ps_suppkey = s_suppkey + and s_nationkey = n_nationkey + and n_name = 'GERMANY' +group by + ps_partkey having + sum(ps_supplycost * ps_availqty) > ( + select + sum(ps_supplycost * ps_availqty) * 0.0001000000 + from + partsupp, + supplier, + nation + where + ps_suppkey = s_suppkey + and s_nationkey = n_nationkey + and n_name = 'GERMANY' + ) +order by + value desc diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q12.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q12.sql new file mode 100644 index 000000000..89bb838aa --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q12.sql @@ -0,0 +1,29 @@ + +select + l_shipmode, + sum(case + when o_orderpriority = '1-URGENT' + or o_orderpriority = '2-HIGH' + then 1 + else 0 + end) as high_line_count, + sum(case + when o_orderpriority <> '1-URGENT' + and o_orderpriority <> '2-HIGH' + then 1 + else 0 + end) as low_line_count +from + orders, + lineitem +where + o_orderkey = l_orderkey + and l_shipmode in ('MAIL', 'SHIP') + and l_commitdate < l_receiptdate + and l_shipdate < l_commitdate + and l_receiptdate >= date '1994-01-01' + and l_receiptdate < date '1994-01-01' + interval '1' year +group by + l_shipmode +order by + l_shipmode diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q13.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q13.sql new file mode 100644 index 000000000..195ad958c --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q13.sql @@ -0,0 +1,21 @@ + +select + c_count, + count(*) as custdist +from + ( + select + c_custkey, + count(o_orderkey) as c_count + from + customer left outer join orders on + c_custkey = o_custkey + and o_comment not like '%special%requests%' + group by + c_custkey + ) as c_orders +group by + c_count +order by + custdist desc, + c_count desc diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q14.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q14.sql new file mode 100644 index 000000000..6a4858b69 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q14.sql @@ -0,0 +1,14 @@ + +select + 100.00 * sum(case + when p_type like 'PROMO%' + then l_extendedprice * (1 - l_discount) + else 0 + end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue +from + lineitem, + part +where + l_partkey = p_partkey + and l_shipdate >= date '1995-09-01' + and l_shipdate < date '1995-09-01' + interval '1' month diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q15.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q15.sql new file mode 100644 index 000000000..89417f240 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q15.sql @@ -0,0 +1,34 @@ + +with revenue0 as + (select + l_suppkey as supplier_no, + sum(l_extendedprice * (1 - l_discount)) as total_revenue + from + lineitem + where + l_shipdate >= date '1996-01-01' + and l_shipdate < date '1996-01-01' + interval '3' month + group by + l_suppkey) + + +select + s_suppkey, + s_name, + s_address, + s_phone, + total_revenue +from + supplier, + revenue0 +where + s_suppkey = supplier_no + and total_revenue = ( + select + max(total_revenue) + from + revenue0 + ) +order by + s_suppkey + diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q16.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q16.sql new file mode 100644 index 000000000..ab5a9a559 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q16.sql @@ -0,0 +1,31 @@ + +select + p_brand, + p_type, + p_size, + count(distinct ps_suppkey) as supplier_cnt +from + partsupp, + part +where + p_partkey = ps_partkey + and p_brand <> 'Brand#45' + and p_type not like 'MEDIUM POLISHED%' + and p_size in (49, 14, 23, 45, 19, 3, 36, 9) + and ps_suppkey not in ( + select + s_suppkey + from + supplier + where + s_comment like '%Customer%Complaints%' + ) +group by + p_brand, + p_type, + p_size +order by + supplier_cnt desc, + p_brand, + p_type, + p_size diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q17.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q17.sql new file mode 100644 index 000000000..921dac59a --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q17.sql @@ -0,0 +1,18 @@ + +select + sum(l_extendedprice) / 7.0 as avg_yearly +from + lineitem, + part +where + p_partkey = l_partkey + and p_brand = 'Brand#23' + and p_container = 'WRAP BAG' + and l_quantity < ( + select + 0.2 * avg(l_quantity) + from + lineitem + where + l_partkey = p_partkey + ) diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q18.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q18.sql new file mode 100644 index 000000000..ee427984e --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q18.sql @@ -0,0 +1,34 @@ + +select + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) +from + customer, + orders, + lineitem +where + o_orderkey in ( + select + l_orderkey + from + lineitem + group by + l_orderkey having + sum(l_quantity) > 300 + ) + and c_custkey = o_custkey + and o_orderkey = l_orderkey +group by + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice +order by + o_totalprice desc, + o_orderdate +limit 100 \ No newline at end of file diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q19.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q19.sql new file mode 100644 index 000000000..b3a78269f --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q19.sql @@ -0,0 +1,36 @@ + +select + sum(l_extendedprice* (1 - l_discount)) as revenue +from + lineitem, + part +where + ( + p_partkey = l_partkey + and p_brand = 'Brand#12' + and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') + and l_quantity >= 1 and l_quantity <= 1 + 10 + and p_size between 1 and 5 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#23' + and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') + and l_quantity >= 10 and l_quantity <= 10 + 10 + and p_size between 1 and 10 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#34' + and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') + and l_quantity >= 20 and l_quantity <= 20 + 10 + and p_size between 1 and 15 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q2.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q2.sql new file mode 100644 index 000000000..38e17b595 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q2.sql @@ -0,0 +1,45 @@ + +select + s_acctbal, + s_name, + n_name, + p_partkey, + p_mfgr, + s_address, + s_phone, + s_comment +from + part, + supplier, + partsupp, + nation, + region +where + p_partkey = ps_partkey + and s_suppkey = ps_suppkey + and p_size = 15 + and p_type like '%BRASS' + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'EUROPE' + and ps_supplycost = ( + select + min(ps_supplycost) + from + partsupp, + supplier, + nation, + region + where + p_partkey = ps_partkey + and s_suppkey = ps_suppkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'EUROPE' + ) +order by + s_acctbal desc, + n_name, + s_name, + p_partkey +limit 100 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q20.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q20.sql new file mode 100644 index 000000000..d3c4ad567 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q20.sql @@ -0,0 +1,38 @@ + +select + s_name, + s_address +from + supplier, + nation +where + s_suppkey in ( + select + ps_suppkey + from + partsupp + where + ps_partkey in ( + select + p_partkey + from + part + where + p_name like 'forest%' + ) + and ps_availqty > ( + select + 0.5 * sum(l_quantity) + from + lineitem + where + l_partkey = ps_partkey + and l_suppkey = ps_suppkey + and l_shipdate >= date '1994-01-01' + and l_shipdate < date '1994-01-01' + interval '1' year + ) + ) + and s_nationkey = n_nationkey + and n_name = 'CANADA' +order by + s_name diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q21.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q21.sql new file mode 100644 index 000000000..303722b23 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q21.sql @@ -0,0 +1,41 @@ + +select + s_name, + count(*) as numwait +from + supplier, + lineitem l1, + orders, + nation +where + s_suppkey = l1.l_suppkey + and o_orderkey = l1.l_orderkey + and o_orderstatus = 'F' + and l1.l_receiptdate > l1.l_commitdate + and exists ( + select + * + from + lineitem l2 + where + l2.l_orderkey = l1.l_orderkey + and l2.l_suppkey <> l1.l_suppkey + ) + and not exists ( + select + * + from + lineitem l3 + where + l3.l_orderkey = l1.l_orderkey + and l3.l_suppkey <> l1.l_suppkey + and l3.l_receiptdate > l3.l_commitdate + ) + and s_nationkey = n_nationkey + and n_name = 'SAUDI ARABIA' +group by + s_name +order by + numwait desc, + s_name +limit 100 \ No newline at end of file diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q22.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q22.sql new file mode 100644 index 000000000..46a8e539c --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q22.sql @@ -0,0 +1,38 @@ + +select + cntrycode, + count(*) as numcust, + sum(c_acctbal) as totacctbal +from + ( + select + substring(c_phone, 1, 2) as cntrycode, + c_acctbal + from + customer + where + substring(c_phone, 1, 2) in + ('13', '31', '23', '29', '30', '18', '17') + and c_acctbal > ( + select + avg(c_acctbal) + from + customer + where + c_acctbal > 0.00 + and substring(c_phone, 1, 2) in + ('13', '31', '23', '29', '30', '18', '17') + ) + and not exists ( + select + * + from + orders + where + o_custkey = c_custkey + ) + ) as custsale +group by + cntrycode +order by + cntrycode diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q3.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q3.sql new file mode 100644 index 000000000..c7aa60433 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q3.sql @@ -0,0 +1,24 @@ + +select + l_orderkey, + sum(l_extendedprice * (1 - l_discount)) as revenue, + o_orderdate, + o_shippriority +from + customer, + orders, + lineitem +where + c_mktsegment = 'BUILDING' + and c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate < date '1995-03-15' + and l_shipdate > date '1995-03-15' +group by + l_orderkey, + o_orderdate, + o_shippriority +order by + revenue desc, + o_orderdate +limit 10 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q4.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q4.sql new file mode 100644 index 000000000..b0df589b6 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q4.sql @@ -0,0 +1,22 @@ + +select + o_orderpriority, + count(*) as order_count +from + orders +where + o_orderdate >= date '1993-07-01' + and o_orderdate < date '1993-07-01' + interval '3' month + and exists ( + select + * + from + lineitem + where + l_orderkey = o_orderkey + and l_commitdate < l_receiptdate + ) +group by + o_orderpriority +order by + o_orderpriority diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q5.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q5.sql new file mode 100644 index 000000000..d6fe0200c --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q5.sql @@ -0,0 +1,25 @@ + +select + n_name, + sum(l_extendedprice * (1 - l_discount)) as revenue +from + customer, + orders, + lineitem, + supplier, + nation, + region +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and l_suppkey = s_suppkey + and c_nationkey = s_nationkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'ASIA' + and o_orderdate >= date '1994-01-01' + and o_orderdate < date '1994-01-01' + interval '1' year +group by + n_name +order by + revenue desc diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q6.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q6.sql new file mode 100644 index 000000000..526d8a389 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q6.sql @@ -0,0 +1,10 @@ + +select + sum(l_extendedprice * l_discount) as revenue +from + lineitem +where + l_shipdate >= date '1994-01-01' + and l_shipdate < date '1994-01-01' + interval '1' year + and l_discount between .06 - 0.01 and .06 + 0.01 + and l_quantity < 24 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q7.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q7.sql new file mode 100644 index 000000000..0bd6fefdb --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q7.sql @@ -0,0 +1,40 @@ + +select + supp_nation, + cust_nation, + l_year, + sum(volume) as revenue +from + ( + select + n1.n_name as supp_nation, + n2.n_name as cust_nation, + year(l_shipdate) as l_year, + l_extendedprice * (1 - l_discount) as volume + from + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2 + where + s_suppkey = l_suppkey + and o_orderkey = l_orderkey + and c_custkey = o_custkey + and s_nationkey = n1.n_nationkey + and c_nationkey = n2.n_nationkey + and ( + (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY') + or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE') + ) + and l_shipdate between date '1995-01-01' and date '1996-12-31' + ) as shipping +group by + supp_nation, + cust_nation, + l_year +order by + supp_nation, + cust_nation, + l_year diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q8.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q8.sql new file mode 100644 index 000000000..fac1aa288 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q8.sql @@ -0,0 +1,38 @@ + +select + o_year, + sum(case + when nation = 'BRAZIL' then volume + else 0 + end) / sum(volume) as mkt_share +from + ( + select + year(o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) as volume, + n2.n_name as nation + from + part, + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2, + region + where + p_partkey = l_partkey + and s_suppkey = l_suppkey + and l_orderkey = o_orderkey + and o_custkey = c_custkey + and c_nationkey = n1.n_nationkey + and n1.n_regionkey = r_regionkey + and r_name = 'AMERICA' + and s_nationkey = n2.n_nationkey + and o_orderdate between date '1995-01-01' and date '1996-12-31' + and p_type = 'ECONOMY ANODIZED STEEL' + ) as all_nations +group by + o_year +order by + o_year diff --git a/spark-extension-shims-spark/src/test/resources/tpch-queries/q9.sql b/spark-extension-shims-spark/src/test/resources/tpch-queries/q9.sql new file mode 100644 index 000000000..aab8fde2c --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-queries/q9.sql @@ -0,0 +1,33 @@ + +select + nation, + o_year, + sum(amount) as sum_profit +from + ( + select + n_name as nation, + year(o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount + from + part, + supplier, + lineitem, + partsupp, + orders, + nation + where + s_suppkey = l_suppkey + and ps_suppkey = l_suppkey + and ps_partkey = l_partkey + and p_partkey = l_partkey + and o_orderkey = l_orderkey + and s_nationkey = n_nationkey + and p_name like '%green%' + ) as profit +group by + nation, + o_year +order by + nation, + o_year desc diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q1.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q1.out new file mode 100644 index 000000000..1a25a0c5c --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q1.out @@ -0,0 +1,5 @@ +4 +A<|COL|>F<|COL|>380456.00<|COL|>532348211.65<|COL|>505822441.4861<|COL|>526165934.000839<|COL|>25.575154<|COL|>35785.709306<|COL|>0.050081<|COL|>14876 +N<|COL|>F<|COL|>8971.00<|COL|>12384801.37<|COL|>11798257.2080<|COL|>12282485.056933<|COL|>25.778735<|COL|>35588.509683<|COL|>0.047758<|COL|>348 +N<|COL|>O<|COL|>742802.00<|COL|>1041502841.45<|COL|>989737518.6346<|COL|>1029418531.523350<|COL|>25.454987<|COL|>35691.129209<|COL|>0.049931<|COL|>29181 +R<|COL|>F<|COL|>381449.00<|COL|>534594445.35<|COL|>507996454.4067<|COL|>528524219.358903<|COL|>25.597168<|COL|>35874.006532<|COL|>0.049827<|COL|>14902 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q10.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q10.out new file mode 100644 index 000000000..58f509a38 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q10.out @@ -0,0 +1,21 @@ +20 +679<|COL|>Customer#000000679<|COL|>378211.3252<|COL|>1394.44<|COL|>IRAN<|COL|>IJf1FlZL9I9m,rvofcoKy5pRUOjUQV<|COL|>20-146-696-9508<|COL|>ely pending frays boost carefully +1201<|COL|>Customer#000001201<|COL|>374331.5340<|COL|>5165.39<|COL|>IRAN<|COL|>LfCSVKWozyWOGDW02g9UX,XgH5YU2o5ql1zBrN<|COL|>20-825-400-1187<|COL|>lyly pending packages. special requests sleep-- platelets use blithely after the instructions. sometimes even id +422<|COL|>Customer#000000422<|COL|>366451.0126<|COL|>-272.14<|COL|>INDONESIA<|COL|>AyNzZBvmIDo42JtjP9xzaK3pnvkh Qc0o08ssnvq<|COL|>19-299-247-2444<|COL|>eposits; furiously ironic packages accordi +334<|COL|>Customer#000000334<|COL|>360370.7550<|COL|>-405.91<|COL|>EGYPT<|COL|>OPN1N7t4aQ23TnCpc<|COL|>14-947-291-5002<|COL|>fully busily special ideas. carefully final excuses lose slyly carefully express accounts. even, ironic platelets ar +805<|COL|>Customer#000000805<|COL|>359448.9036<|COL|>511.69<|COL|>IRAN<|COL|>wCKx5zcHvwpSffyc9qfi9dvqcm9LT,cLAG<|COL|>20-732-989-5653<|COL|>busy sentiments. pending packages haggle among the express requests-- slyly regular excuses above the slyl +932<|COL|>Customer#000000932<|COL|>341608.2753<|COL|>6553.37<|COL|>JORDAN<|COL|>HN9Ap0NsJG7Mb8O<|COL|>23-300-708-7927<|COL|>packages boost slyly along the furiously express foxes. ev +853<|COL|>Customer#000000853<|COL|>341236.6246<|COL|>-444.73<|COL|>BRAZIL<|COL|>U0 9PrwAgWK8AE0GHmnCGtH9BTexWWv87k<|COL|>12-869-161-3468<|COL|>yly special deposits wake alongside of +872<|COL|>Customer#000000872<|COL|>338328.7808<|COL|>-858.61<|COL|>PERU<|COL|>vLP7iNZBK4B,HANFTKabVI3AO Y9O8H<|COL|>27-357-139-7164<|COL|> detect. packages wake slyly express foxes. even deposits ru +737<|COL|>Customer#000000737<|COL|>338185.3365<|COL|>2501.74<|COL|>CHINA<|COL|>NdjG1k243iCLSoy1lYqMIrpvuH1Uf75<|COL|>28-658-938-1102<|COL|>ding to the final platelets. regular packages against the carefully final ideas hag +1118<|COL|>Customer#000001118<|COL|>319875.7280<|COL|>4130.18<|COL|>IRAQ<|COL|>QHg,DNvEVXaYoCdrywazjAJ<|COL|>21-583-715-8627<|COL|>y regular requests above the blithely ironic accounts use slyly bold packages: regular pinto beans eat carefully spe +223<|COL|>Customer#000000223<|COL|>319564.2750<|COL|>7476.20<|COL|>SAUDI ARABIA<|COL|>ftau6Pk,brboMyEl,,kFm<|COL|>30-193-643-1517<|COL|>al, regular requests run furiously blithely silent packages. blithely ironic accounts across the furious +808<|COL|>Customer#000000808<|COL|>314774.6167<|COL|>5561.93<|COL|>ROMANIA<|COL|>S2WkSKCGtnbhcFOp6MWcuB3rzFlFemVNrg <|COL|>29-531-319-7726<|COL|> unusual deposits. furiously even packages against the furiously even ac +478<|COL|>Customer#000000478<|COL|>299651.8026<|COL|>-210.40<|COL|>ARGENTINA<|COL|>clyq458DIkXXt4qLyHlbe,n JueoniF<|COL|>11-655-291-2694<|COL|>o the foxes. ironic requests sleep. c +1441<|COL|>Customer#000001441<|COL|>294705.3935<|COL|>9465.15<|COL|>UNITED KINGDOM<|COL|>u0YYZb46w,pwKo5H9vz d6B9zK4BOHhG jx<|COL|>33-681-334-4499<|COL|>nts haggle quietly quickly final accounts. slyly regular accounts among the sl +1478<|COL|>Customer#000001478<|COL|>294431.9178<|COL|>9701.54<|COL|>GERMANY<|COL|>x7HDvJDDpR3MqZ5vg2CanfQ1hF0j4<|COL|>17-420-484-5959<|COL|>ng the furiously bold foxes. even notornis above the unusual +211<|COL|>Customer#000000211<|COL|>287905.6368<|COL|>4198.72<|COL|>JORDAN<|COL|>URhlVPzz4FqXem<|COL|>23-965-335-9471<|COL|>furiously regular foxes boost fluffily special ideas. carefully regular dependencies are. slyly ironic +197<|COL|>Customer#000000197<|COL|>283190.4807<|COL|>9860.22<|COL|>ARGENTINA<|COL|>UeVqssepNuXmtZ38D<|COL|>11-107-312-6585<|COL|>ickly final accounts cajole. furiously re +1030<|COL|>Customer#000001030<|COL|>282557.3566<|COL|>6359.27<|COL|>INDIA<|COL|>Xpt1BiB5h9o<|COL|>18-759-877-1870<|COL|>ding to the slyly unusual accounts. even requests among the evenly +1049<|COL|>Customer#000001049<|COL|>281134.1117<|COL|>8747.99<|COL|>INDONESIA<|COL|>bZ1OcFhHaIZ5gMiH<|COL|>19-499-258-2851<|COL|>uriously according to the furiously silent packages +1094<|COL|>Customer#000001094<|COL|>274877.4440<|COL|>2544.49<|COL|>BRAZIL<|COL|>OFz0eedTmPmXk2 3XM9v9Mcp13NVC0PK<|COL|>12-234-721-9871<|COL|>tes serve blithely quickly pending foxes. express, quick accounts diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q11.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q11.out new file mode 100644 index 000000000..f9aefe29b --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q11.out @@ -0,0 +1,360 @@ +359 +1376<|COL|>13271249.89 +788<|COL|>9498648.06 +1071<|COL|>9388264.40 +1768<|COL|>9207199.75 +1168<|COL|>8881908.96 +1084<|COL|>8709494.16 +1415<|COL|>8471489.56 +1338<|COL|>8293841.12 +124<|COL|>8203209.30 +1232<|COL|>8111663.34 +1643<|COL|>7975862.75 +1952<|COL|>7936947.61 +1944<|COL|>7880018.60 +1884<|COL|>7513422.84 +942<|COL|>7511018.76 +670<|COL|>7299956.80 +1532<|COL|>7222347.20 +1052<|COL|>7158586.00 +455<|COL|>7064285.84 +1176<|COL|>7060670.89 +143<|COL|>7037648.64 +1653<|COL|>6949533.70 +1140<|COL|>6929464.08 +1076<|COL|>6877472.96 +2000<|COL|>6720009.38 +348<|COL|>6681307.34 +810<|COL|>6576640.95 +943<|COL|>6458641.70 +720<|COL|>6391330.27 +1748<|COL|>6341530.40 +1241<|COL|>6304944.66 +1384<|COL|>6279261.12 +1784<|COL|>6247863.25 +984<|COL|>6136927.00 +445<|COL|>6127784.28 +1976<|COL|>6079237.08 +1609<|COL|>6022720.80 +1563<|COL|>5978195.08 +452<|COL|>5838052.00 +222<|COL|>5737162.24 +1629<|COL|>5703117.12 +1454<|COL|>5694804.18 +1082<|COL|>5681981.25 +691<|COL|>5633589.72 +1474<|COL|>5614673.64 +1900<|COL|>5591905.36 +262<|COL|>5553285.32 +1876<|COL|>5517997.59 +1027<|COL|>5490916.00 +1833<|COL|>5451495.00 +513<|COL|>5374426.22 +752<|COL|>5358919.70 +1367<|COL|>5352773.25 +543<|COL|>5189101.68 +1144<|COL|>5174388.56 +403<|COL|>5126118.15 +1406<|COL|>5121886.44 +320<|COL|>5072099.76 +1940<|COL|>5069178.40 +1503<|COL|>5050895.50 +1437<|COL|>5039590.60 +743<|COL|>5039271.42 +82<|COL|>4995939.00 +916<|COL|>4994730.10 +732<|COL|>4932809.82 +356<|COL|>4879860.09 +1592<|COL|>4831242.60 +1043<|COL|>4825921.31 +132<|COL|>4781984.14 +1006<|COL|>4733954.64 +497<|COL|>4711173.60 +1008<|COL|>4565588.85 +1370<|COL|>4563830.10 +216<|COL|>4561143.80 +34<|COL|>4501982.71 +1908<|COL|>4417931.80 +982<|COL|>4391495.46 +1652<|COL|>4358793.14 +614<|COL|>4356657.45 +1552<|COL|>4355541.70 +359<|COL|>4353566.87 +1104<|COL|>4347515.90 +198<|COL|>4315049.00 +998<|COL|>4167784.88 +1543<|COL|>4159568.16 +1308<|COL|>4153124.95 +474<|COL|>4123819.20 +1394<|COL|>4122729.33 +271<|COL|>4095180.96 +908<|COL|>4088856.20 +1135<|COL|>4045014.13 +1632<|COL|>4010794.90 +1362<|COL|>3982060.16 +158<|COL|>3941881.65 +1852<|COL|>3923035.02 +1556<|COL|>3896709.54 +584<|COL|>3843848.30 +885<|COL|>3826021.16 +376<|COL|>3781201.96 +712<|COL|>3749696.80 +2<|COL|>3743241.43 +676<|COL|>3735715.20 +1832<|COL|>3709008.60 +1955<|COL|>3702794.70 +68<|COL|>3690702.41 +1435<|COL|>3659114.10 +1443<|COL|>3656762.84 +1278<|COL|>3653100.66 +1920<|COL|>3647892.54 +423<|COL|>3602031.80 +818<|COL|>3589047.60 +779<|COL|>3559597.53 +485<|COL|>3558511.44 +552<|COL|>3555470.10 +1269<|COL|>3510427.65 +1602<|COL|>3492117.70 +426<|COL|>3486888.02 +1452<|COL|>3480825.60 +756<|COL|>3469373.70 +832<|COL|>3447746.46 +1493<|COL|>3446867.40 +1650<|COL|>3417752.58 +205<|COL|>3403046.25 +93<|COL|>3361425.89 +76<|COL|>3342081.82 +1759<|COL|>3303050.40 +886<|COL|>3302180.70 +1544<|COL|>3288573.16 +1932<|COL|>3270900.40 +489<|COL|>3253368.30 +594<|COL|>3177408.57 +184<|COL|>3177162.05 +950<|COL|>3165213.01 +1124<|COL|>3143279.36 +106<|COL|>3099021.98 +1964<|COL|>3016553.10 +384<|COL|>2964262.77 +974<|COL|>2959497.10 +964<|COL|>2951329.45 +1984<|COL|>2907345.36 +200<|COL|>2895688.32 +683<|COL|>2829476.95 +1564<|COL|>2816506.56 +546<|COL|>2788059.64 +502<|COL|>2780828.64 +396<|COL|>2778421.39 +203<|COL|>2761439.88 +866<|COL|>2753031.20 +1743<|COL|>2743889.49 +1041<|COL|>2738083.92 +1432<|COL|>2713412.16 +43<|COL|>2587359.58 +941<|COL|>2587091.52 +1890<|COL|>2558739.69 +1866<|COL|>2545838.40 +747<|COL|>2511745.32 +776<|COL|>2506489.89 +554<|COL|>2505417.25 +1210<|COL|>2490820.92 +1239<|COL|>2405206.30 +443<|COL|>2382150.05 +1661<|COL|>2370574.16 +1079<|COL|>2363505.11 +1329<|COL|>2305870.42 +1691<|COL|>2261159.92 +1247<|COL|>2239553.28 +1752<|COL|>2230055.76 +150<|COL|>2217043.59 +1814<|COL|>2213635.20 +289<|COL|>2187160.45 +1400<|COL|>2139845.10 +1898<|COL|>2130114.96 +1809<|COL|>2122758.72 +884<|COL|>2107479.56 +1038<|COL|>2096868.97 +1318<|COL|>2051302.44 +524<|COL|>2035262.22 +414<|COL|>2029692.45 +298<|COL|>2026981.74 +1996<|COL|>2020953.54 +1742<|COL|>2019190.80 +1620<|COL|>2010112.00 +877<|COL|>1956429.18 +1332<|COL|>1919029.56 +1536<|COL|>1859318.15 +1116<|COL|>1852588.28 +447<|COL|>1817951.32 +1676<|COL|>1802306.08 +1911<|COL|>1779646.44 +1459<|COL|>1767602.30 +576<|COL|>1761838.75 +1273<|COL|>1754235.01 +583<|COL|>1725649.92 +532<|COL|>1682311.48 +1732<|COL|>1652831.20 +1572<|COL|>1650953.52 +1889<|COL|>1638443.72 +476<|COL|>1631154.06 +1221<|COL|>1629883.46 +1792<|COL|>1606346.10 +243<|COL|>1603235.16 +328<|COL|>1569826.72 +1999<|COL|>1553706.00 +1611<|COL|>1529857.01 +643<|COL|>1512838.80 +1276<|COL|>1467567.28 +1823<|COL|>1462293.00 +1<|COL|>1456050.96 +27<|COL|>1425832.40 +632<|COL|>1408087.26 +1184<|COL|>1406101.78 +252<|COL|>1379186.35 +392<|COL|>1354813.18 +1215<|COL|>1344383.20 +26<|COL|>1337002.89 +84<|COL|>1334146.71 +784<|COL|>1327297.01 +1803<|COL|>1327045.06 +352<|COL|>1326102.34 +165<|COL|>1289075.76 +176<|COL|>1285866.20 +1314<|COL|>1244173.26 +1701<|COL|>1239095.44 +844<|COL|>1225696.05 +1988<|COL|>1216798.33 +1847<|COL|>1202012.13 +1706<|COL|>1184125.10 +744<|COL|>1182820.80 +230<|COL|>1165932.30 +418<|COL|>1078321.44 +174<|COL|>1060584.80 +1073<|COL|>1028449.89 +1726<|COL|>1018673.04 +1206<|COL|>1002319.49 +1343<|COL|>998105.76 +952<|COL|>997684.24 +484<|COL|>991530.93 +932<|COL|>980620.68 +843<|COL|>978862.92 +1841<|COL|>962131.86 +494<|COL|>957575.34 +659<|COL|>954291.05 +251<|COL|>939764.70 +1413<|COL|>936951.94 +572<|COL|>906111.99 +32<|COL|>894484.09 +9<|COL|>893905.92 +1498<|COL|>890887.85 +1790<|COL|>878923.64 +1670<|COL|>854046.43 +876<|COL|>842245.67 +1758<|COL|>841275.42 +930<|COL|>832963.68 +284<|COL|>826642.60 +1710<|COL|>811504.38 +1047<|COL|>791214.45 +653<|COL|>788974.21 +315<|COL|>770526.05 +1734<|COL|>763569.40 +1017<|COL|>715302.72 +1305<|COL|>713351.43 +77<|COL|>688865.82 +1512<|COL|>682434.15 +276<|COL|>680239.04 +1284<|COL|>671225.94 +1356<|COL|>665716.83 +800<|COL|>663414.65 +117<|COL|>639650.88 +652<|COL|>635629.28 +57<|COL|>630987.44 +1426<|COL|>628241.25 +1196<|COL|>622427.16 +51<|COL|>622249.54 +1846<|COL|>621068.80 +601<|COL|>615942.60 +645<|COL|>607985.84 +684<|COL|>571490.70 +465<|COL|>570337.40 +562<|COL|>567651.24 +387<|COL|>556634.76 +1152<|COL|>555989.28 +1202<|COL|>553818.18 +1112<|COL|>552658.68 +304<|COL|>535868.16 +368<|COL|>526995.84 +1800<|COL|>526711.11 +1148<|COL|>515702.16 +225<|COL|>513587.57 +324<|COL|>500954.58 +586<|COL|>499475.58 +1576<|COL|>494401.05 +1484<|COL|>462396.27 +126<|COL|>461263.74 +1132<|COL|>455492.24 +622<|COL|>449685.60 +1160<|COL|>448183.06 +1352<|COL|>439967.04 +18<|COL|>426442.08 +7<|COL|>414558.20 +833<|COL|>398540.87 +1694<|COL|>376443.98 +650<|COL|>370900.99 +1504<|COL|>370815.90 +432<|COL|>370528.52 +612<|COL|>367894.50 +542<|COL|>367653.66 +456<|COL|>360911.32 +52<|COL|>358792.36 +1346<|COL|>350637.43 +59<|COL|>342221.48 +1107<|COL|>341805.20 +1171<|COL|>334938.04 +1062<|COL|>326445.90 +592<|COL|>313081.75 +1750<|COL|>312229.33 +1843<|COL|>309456.95 +180<|COL|>308539.84 +899<|COL|>301989.50 +1180<|COL|>293452.50 +522<|COL|>291601.75 +249<|COL|>282520.32 +1584<|COL|>278559.38 +1404<|COL|>276057.90 +1265<|COL|>271079.76 +154<|COL|>269641.42 +1295<|COL|>265566.56 +1523<|COL|>263158.90 +1635<|COL|>254834.56 +1776<|COL|>234181.20 +1097<|COL|>234113.55 +1258<|COL|>233500.61 +621<|COL|>233431.30 +152<|COL|>229781.60 +278<|COL|>216372.84 +232<|COL|>211879.92 +1684<|COL|>201386.22 +1243<|COL|>199587.54 +976<|COL|>197432.10 +819<|COL|>191475.90 +1943<|COL|>191247.76 +853<|COL|>189232.64 +400<|COL|>188941.20 +639<|COL|>186533.28 +851<|COL|>184103.16 +909<|COL|>175099.00 +257<|COL|>169033.44 +1445<|COL|>164888.68 +1855<|COL|>164614.81 +1252<|COL|>158680.90 +1014<|COL|>156465.82 +1717<|COL|>148325.75 +1032<|COL|>146408.40 +780<|COL|>136296.26 +918<|COL|>135268.32 +690<|COL|>133826.88 +711<|COL|>113268.84 +332<|COL|>112181.30 +1596<|COL|>110565.00 +295<|COL|>97604.25 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q12.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q12.out new file mode 100644 index 000000000..83e3c8676 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q12.out @@ -0,0 +1,3 @@ +2 +MAIL<|COL|>64<|COL|>86 +SHIP<|COL|>61<|COL|>96 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q13.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q13.out new file mode 100644 index 000000000..f2ff7cf8c --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q13.out @@ -0,0 +1,34 @@ +33 +0<|COL|>500 +11<|COL|>68 +10<|COL|>64 +12<|COL|>62 +9<|COL|>62 +8<|COL|>61 +14<|COL|>54 +13<|COL|>52 +7<|COL|>49 +20<|COL|>48 +21<|COL|>47 +16<|COL|>46 +15<|COL|>45 +19<|COL|>44 +17<|COL|>41 +18<|COL|>38 +22<|COL|>33 +6<|COL|>33 +24<|COL|>30 +23<|COL|>27 +25<|COL|>21 +27<|COL|>17 +26<|COL|>15 +5<|COL|>14 +28<|COL|>6 +4<|COL|>6 +32<|COL|>5 +29<|COL|>5 +30<|COL|>2 +3<|COL|>2 +31<|COL|>1 +2<|COL|>1 +1<|COL|>1 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q14.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q14.out new file mode 100644 index 000000000..f43ad2b65 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q14.out @@ -0,0 +1,2 @@ +1 +15.486546 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q15.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q15.out new file mode 100644 index 000000000..8abf7e9b8 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q15.out @@ -0,0 +1,2 @@ +1 +21<|COL|>Supplier#000000021<|COL|>81CavellcrJ0PQ3CPBID0Z0JwyJm0ka5igEs<|COL|>12-253-590-5816<|COL|>1161099.4636 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q16.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q16.out new file mode 100644 index 000000000..1c5392f57 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q16.out @@ -0,0 +1,297 @@ +296 +Brand#14<|COL|>PROMO BRUSHED STEEL<|COL|>9<|COL|>8 +Brand#35<|COL|>SMALL POLISHED COPPER<|COL|>14<|COL|>8 +Brand#22<|COL|>LARGE BURNISHED TIN<|COL|>36<|COL|>6 +Brand#11<|COL|>ECONOMY BURNISHED NICKEL<|COL|>49<|COL|>4 +Brand#11<|COL|>LARGE PLATED TIN<|COL|>23<|COL|>4 +Brand#11<|COL|>MEDIUM ANODIZED BRASS<|COL|>45<|COL|>4 +Brand#11<|COL|>MEDIUM BRUSHED BRASS<|COL|>45<|COL|>4 +Brand#11<|COL|>PROMO ANODIZED BRASS<|COL|>3<|COL|>4 +Brand#11<|COL|>PROMO ANODIZED BRASS<|COL|>49<|COL|>4 +Brand#11<|COL|>PROMO ANODIZED TIN<|COL|>45<|COL|>4 +Brand#11<|COL|>PROMO BURNISHED BRASS<|COL|>36<|COL|>4 +Brand#11<|COL|>SMALL ANODIZED TIN<|COL|>45<|COL|>4 +Brand#11<|COL|>SMALL PLATED COPPER<|COL|>45<|COL|>4 +Brand#11<|COL|>STANDARD POLISHED NICKEL<|COL|>45<|COL|>4 +Brand#11<|COL|>STANDARD POLISHED TIN<|COL|>45<|COL|>4 +Brand#12<|COL|>ECONOMY BURNISHED COPPER<|COL|>45<|COL|>4 +Brand#12<|COL|>LARGE ANODIZED TIN<|COL|>45<|COL|>4 +Brand#12<|COL|>LARGE BURNISHED BRASS<|COL|>19<|COL|>4 +Brand#12<|COL|>LARGE PLATED STEEL<|COL|>36<|COL|>4 +Brand#12<|COL|>MEDIUM PLATED BRASS<|COL|>23<|COL|>4 +Brand#12<|COL|>PROMO BRUSHED COPPER<|COL|>14<|COL|>4 +Brand#12<|COL|>PROMO BURNISHED BRASS<|COL|>49<|COL|>4 +Brand#12<|COL|>SMALL ANODIZED COPPER<|COL|>23<|COL|>4 +Brand#12<|COL|>STANDARD ANODIZED BRASS<|COL|>3<|COL|>4 +Brand#12<|COL|>STANDARD BURNISHED TIN<|COL|>23<|COL|>4 +Brand#12<|COL|>STANDARD PLATED STEEL<|COL|>36<|COL|>4 +Brand#13<|COL|>ECONOMY PLATED STEEL<|COL|>23<|COL|>4 +Brand#13<|COL|>ECONOMY POLISHED BRASS<|COL|>9<|COL|>4 +Brand#13<|COL|>ECONOMY POLISHED COPPER<|COL|>9<|COL|>4 +Brand#13<|COL|>LARGE ANODIZED TIN<|COL|>19<|COL|>4 +Brand#13<|COL|>LARGE BURNISHED TIN<|COL|>49<|COL|>4 +Brand#13<|COL|>LARGE POLISHED BRASS<|COL|>3<|COL|>4 +Brand#13<|COL|>MEDIUM ANODIZED STEEL<|COL|>36<|COL|>4 +Brand#13<|COL|>MEDIUM PLATED COPPER<|COL|>19<|COL|>4 +Brand#13<|COL|>PROMO BRUSHED COPPER<|COL|>49<|COL|>4 +Brand#13<|COL|>PROMO PLATED TIN<|COL|>19<|COL|>4 +Brand#13<|COL|>SMALL BRUSHED NICKEL<|COL|>19<|COL|>4 +Brand#13<|COL|>SMALL BURNISHED BRASS<|COL|>45<|COL|>4 +Brand#14<|COL|>ECONOMY ANODIZED STEEL<|COL|>19<|COL|>4 +Brand#14<|COL|>ECONOMY BURNISHED TIN<|COL|>23<|COL|>4 +Brand#14<|COL|>ECONOMY PLATED STEEL<|COL|>45<|COL|>4 +Brand#14<|COL|>ECONOMY PLATED TIN<|COL|>9<|COL|>4 +Brand#14<|COL|>LARGE ANODIZED NICKEL<|COL|>9<|COL|>4 +Brand#14<|COL|>LARGE BRUSHED NICKEL<|COL|>45<|COL|>4 +Brand#14<|COL|>SMALL ANODIZED NICKEL<|COL|>45<|COL|>4 +Brand#14<|COL|>SMALL BURNISHED COPPER<|COL|>14<|COL|>4 +Brand#14<|COL|>SMALL BURNISHED TIN<|COL|>23<|COL|>4 +Brand#15<|COL|>ECONOMY ANODIZED STEEL<|COL|>36<|COL|>4 +Brand#15<|COL|>ECONOMY BRUSHED BRASS<|COL|>36<|COL|>4 +Brand#15<|COL|>ECONOMY BURNISHED BRASS<|COL|>14<|COL|>4 +Brand#15<|COL|>ECONOMY PLATED STEEL<|COL|>45<|COL|>4 +Brand#15<|COL|>LARGE ANODIZED BRASS<|COL|>45<|COL|>4 +Brand#15<|COL|>LARGE ANODIZED COPPER<|COL|>3<|COL|>4 +Brand#15<|COL|>MEDIUM ANODIZED COPPER<|COL|>9<|COL|>4 +Brand#15<|COL|>MEDIUM PLATED TIN<|COL|>9<|COL|>4 +Brand#15<|COL|>PROMO POLISHED TIN<|COL|>49<|COL|>4 +Brand#15<|COL|>SMALL POLISHED STEEL<|COL|>19<|COL|>4 +Brand#15<|COL|>STANDARD BURNISHED STEEL<|COL|>45<|COL|>4 +Brand#15<|COL|>STANDARD PLATED NICKEL<|COL|>19<|COL|>4 +Brand#15<|COL|>STANDARD PLATED TIN<|COL|>3<|COL|>4 +Brand#21<|COL|>ECONOMY ANODIZED STEEL<|COL|>19<|COL|>4 +Brand#21<|COL|>ECONOMY BRUSHED TIN<|COL|>49<|COL|>4 +Brand#21<|COL|>LARGE BURNISHED COPPER<|COL|>19<|COL|>4 +Brand#21<|COL|>MEDIUM ANODIZED TIN<|COL|>9<|COL|>4 +Brand#21<|COL|>MEDIUM BURNISHED STEEL<|COL|>23<|COL|>4 +Brand#21<|COL|>PROMO BRUSHED STEEL<|COL|>23<|COL|>4 +Brand#21<|COL|>PROMO BURNISHED COPPER<|COL|>19<|COL|>4 +Brand#21<|COL|>STANDARD PLATED BRASS<|COL|>49<|COL|>4 +Brand#21<|COL|>STANDARD POLISHED TIN<|COL|>36<|COL|>4 +Brand#22<|COL|>ECONOMY BURNISHED NICKEL<|COL|>19<|COL|>4 +Brand#22<|COL|>LARGE ANODIZED STEEL<|COL|>3<|COL|>4 +Brand#22<|COL|>LARGE BURNISHED STEEL<|COL|>23<|COL|>4 +Brand#22<|COL|>LARGE BURNISHED STEEL<|COL|>45<|COL|>4 +Brand#22<|COL|>LARGE BURNISHED TIN<|COL|>45<|COL|>4 +Brand#22<|COL|>LARGE POLISHED NICKEL<|COL|>19<|COL|>4 +Brand#22<|COL|>MEDIUM ANODIZED TIN<|COL|>9<|COL|>4 +Brand#22<|COL|>MEDIUM BRUSHED BRASS<|COL|>14<|COL|>4 +Brand#22<|COL|>MEDIUM BRUSHED COPPER<|COL|>3<|COL|>4 +Brand#22<|COL|>MEDIUM BRUSHED COPPER<|COL|>45<|COL|>4 +Brand#22<|COL|>MEDIUM BURNISHED TIN<|COL|>19<|COL|>4 +Brand#22<|COL|>MEDIUM BURNISHED TIN<|COL|>23<|COL|>4 +Brand#22<|COL|>MEDIUM PLATED BRASS<|COL|>49<|COL|>4 +Brand#22<|COL|>PROMO BRUSHED BRASS<|COL|>9<|COL|>4 +Brand#22<|COL|>PROMO BRUSHED STEEL<|COL|>36<|COL|>4 +Brand#22<|COL|>SMALL BRUSHED NICKEL<|COL|>3<|COL|>4 +Brand#22<|COL|>SMALL BURNISHED STEEL<|COL|>23<|COL|>4 +Brand#22<|COL|>STANDARD PLATED NICKEL<|COL|>3<|COL|>4 +Brand#22<|COL|>STANDARD PLATED TIN<|COL|>19<|COL|>4 +Brand#23<|COL|>ECONOMY BRUSHED COPPER<|COL|>9<|COL|>4 +Brand#23<|COL|>LARGE ANODIZED COPPER<|COL|>14<|COL|>4 +Brand#23<|COL|>LARGE PLATED BRASS<|COL|>49<|COL|>4 +Brand#23<|COL|>MEDIUM BRUSHED NICKEL<|COL|>3<|COL|>4 +Brand#23<|COL|>PROMO ANODIZED COPPER<|COL|>19<|COL|>4 +Brand#23<|COL|>PROMO BURNISHED COPPER<|COL|>14<|COL|>4 +Brand#23<|COL|>PROMO POLISHED BRASS<|COL|>14<|COL|>4 +Brand#23<|COL|>SMALL BRUSHED BRASS<|COL|>49<|COL|>4 +Brand#23<|COL|>SMALL BRUSHED COPPER<|COL|>45<|COL|>4 +Brand#23<|COL|>SMALL BURNISHED COPPER<|COL|>49<|COL|>4 +Brand#23<|COL|>SMALL PLATED BRASS<|COL|>36<|COL|>4 +Brand#23<|COL|>SMALL POLISHED BRASS<|COL|>9<|COL|>4 +Brand#23<|COL|>STANDARD BRUSHED TIN<|COL|>3<|COL|>4 +Brand#23<|COL|>STANDARD PLATED BRASS<|COL|>9<|COL|>4 +Brand#23<|COL|>STANDARD PLATED STEEL<|COL|>36<|COL|>4 +Brand#23<|COL|>STANDARD PLATED TIN<|COL|>19<|COL|>4 +Brand#24<|COL|>ECONOMY BRUSHED BRASS<|COL|>36<|COL|>4 +Brand#24<|COL|>ECONOMY PLATED COPPER<|COL|>36<|COL|>4 +Brand#24<|COL|>LARGE PLATED NICKEL<|COL|>36<|COL|>4 +Brand#24<|COL|>MEDIUM PLATED STEEL<|COL|>19<|COL|>4 +Brand#24<|COL|>PROMO POLISHED BRASS<|COL|>14<|COL|>4 +Brand#24<|COL|>SMALL ANODIZED COPPER<|COL|>3<|COL|>4 +Brand#24<|COL|>STANDARD BRUSHED BRASS<|COL|>14<|COL|>4 +Brand#24<|COL|>STANDARD BRUSHED STEEL<|COL|>14<|COL|>4 +Brand#24<|COL|>STANDARD POLISHED NICKEL<|COL|>14<|COL|>4 +Brand#25<|COL|>ECONOMY BURNISHED TIN<|COL|>19<|COL|>4 +Brand#25<|COL|>ECONOMY PLATED NICKEL<|COL|>23<|COL|>4 +Brand#25<|COL|>LARGE ANODIZED NICKEL<|COL|>23<|COL|>4 +Brand#25<|COL|>LARGE BRUSHED NICKEL<|COL|>19<|COL|>4 +Brand#25<|COL|>LARGE BURNISHED TIN<|COL|>49<|COL|>4 +Brand#25<|COL|>MEDIUM BURNISHED NICKEL<|COL|>49<|COL|>4 +Brand#25<|COL|>MEDIUM PLATED BRASS<|COL|>45<|COL|>4 +Brand#25<|COL|>PROMO ANODIZED TIN<|COL|>3<|COL|>4 +Brand#25<|COL|>PROMO BURNISHED COPPER<|COL|>45<|COL|>4 +Brand#25<|COL|>PROMO PLATED NICKEL<|COL|>3<|COL|>4 +Brand#25<|COL|>SMALL BURNISHED COPPER<|COL|>3<|COL|>4 +Brand#25<|COL|>SMALL PLATED TIN<|COL|>36<|COL|>4 +Brand#25<|COL|>STANDARD ANODIZED TIN<|COL|>9<|COL|>4 +Brand#25<|COL|>STANDARD PLATED NICKEL<|COL|>36<|COL|>4 +Brand#31<|COL|>ECONOMY BURNISHED COPPER<|COL|>36<|COL|>4 +Brand#31<|COL|>ECONOMY PLATED STEEL<|COL|>23<|COL|>4 +Brand#31<|COL|>LARGE PLATED NICKEL<|COL|>14<|COL|>4 +Brand#31<|COL|>MEDIUM BURNISHED COPPER<|COL|>3<|COL|>4 +Brand#31<|COL|>MEDIUM PLATED TIN<|COL|>36<|COL|>4 +Brand#31<|COL|>PROMO ANODIZED NICKEL<|COL|>9<|COL|>4 +Brand#31<|COL|>PROMO POLISHED TIN<|COL|>23<|COL|>4 +Brand#31<|COL|>SMALL ANODIZED COPPER<|COL|>3<|COL|>4 +Brand#31<|COL|>SMALL ANODIZED COPPER<|COL|>45<|COL|>4 +Brand#31<|COL|>SMALL BRUSHED NICKEL<|COL|>23<|COL|>4 +Brand#31<|COL|>SMALL PLATED COPPER<|COL|>36<|COL|>4 +Brand#32<|COL|>ECONOMY ANODIZED COPPER<|COL|>36<|COL|>4 +Brand#32<|COL|>ECONOMY PLATED COPPER<|COL|>9<|COL|>4 +Brand#32<|COL|>LARGE ANODIZED STEEL<|COL|>14<|COL|>4 +Brand#32<|COL|>MEDIUM ANODIZED STEEL<|COL|>49<|COL|>4 +Brand#32<|COL|>MEDIUM BURNISHED BRASS<|COL|>9<|COL|>4 +Brand#32<|COL|>MEDIUM BURNISHED BRASS<|COL|>49<|COL|>4 +Brand#32<|COL|>PROMO BRUSHED STEEL<|COL|>23<|COL|>4 +Brand#32<|COL|>PROMO BURNISHED TIN<|COL|>45<|COL|>4 +Brand#32<|COL|>SMALL ANODIZED TIN<|COL|>9<|COL|>4 +Brand#32<|COL|>SMALL BRUSHED COPPER<|COL|>3<|COL|>4 +Brand#32<|COL|>SMALL PLATED COPPER<|COL|>45<|COL|>4 +Brand#32<|COL|>SMALL POLISHED STEEL<|COL|>36<|COL|>4 +Brand#32<|COL|>SMALL POLISHED TIN<|COL|>45<|COL|>4 +Brand#32<|COL|>STANDARD PLATED STEEL<|COL|>36<|COL|>4 +Brand#33<|COL|>ECONOMY BURNISHED COPPER<|COL|>14<|COL|>4 +Brand#33<|COL|>ECONOMY POLISHED BRASS<|COL|>14<|COL|>4 +Brand#33<|COL|>LARGE BRUSHED TIN<|COL|>36<|COL|>4 +Brand#33<|COL|>MEDIUM ANODIZED BRASS<|COL|>3<|COL|>4 +Brand#33<|COL|>MEDIUM BURNISHED COPPER<|COL|>14<|COL|>4 +Brand#33<|COL|>MEDIUM PLATED STEEL<|COL|>49<|COL|>4 +Brand#33<|COL|>PROMO PLATED STEEL<|COL|>49<|COL|>4 +Brand#33<|COL|>PROMO PLATED TIN<|COL|>49<|COL|>4 +Brand#33<|COL|>PROMO POLISHED STEEL<|COL|>9<|COL|>4 +Brand#33<|COL|>SMALL ANODIZED COPPER<|COL|>23<|COL|>4 +Brand#33<|COL|>SMALL BRUSHED STEEL<|COL|>3<|COL|>4 +Brand#33<|COL|>SMALL BURNISHED NICKEL<|COL|>3<|COL|>4 +Brand#33<|COL|>STANDARD PLATED NICKEL<|COL|>36<|COL|>4 +Brand#34<|COL|>ECONOMY ANODIZED TIN<|COL|>49<|COL|>4 +Brand#34<|COL|>LARGE ANODIZED BRASS<|COL|>23<|COL|>4 +Brand#34<|COL|>LARGE BRUSHED COPPER<|COL|>23<|COL|>4 +Brand#34<|COL|>LARGE BURNISHED TIN<|COL|>49<|COL|>4 +Brand#34<|COL|>LARGE PLATED BRASS<|COL|>45<|COL|>4 +Brand#34<|COL|>MEDIUM BRUSHED COPPER<|COL|>9<|COL|>4 +Brand#34<|COL|>MEDIUM BRUSHED TIN<|COL|>14<|COL|>4 +Brand#34<|COL|>MEDIUM BURNISHED NICKEL<|COL|>3<|COL|>4 +Brand#34<|COL|>SMALL ANODIZED STEEL<|COL|>23<|COL|>4 +Brand#34<|COL|>SMALL BRUSHED TIN<|COL|>9<|COL|>4 +Brand#34<|COL|>SMALL PLATED BRASS<|COL|>14<|COL|>4 +Brand#34<|COL|>STANDARD ANODIZED NICKEL<|COL|>36<|COL|>4 +Brand#34<|COL|>STANDARD BRUSHED TIN<|COL|>19<|COL|>4 +Brand#34<|COL|>STANDARD BURNISHED TIN<|COL|>23<|COL|>4 +Brand#34<|COL|>STANDARD PLATED NICKEL<|COL|>36<|COL|>4 +Brand#35<|COL|>PROMO BURNISHED BRASS<|COL|>3<|COL|>4 +Brand#35<|COL|>PROMO BURNISHED STEEL<|COL|>14<|COL|>4 +Brand#35<|COL|>PROMO PLATED BRASS<|COL|>19<|COL|>4 +Brand#35<|COL|>STANDARD ANODIZED NICKEL<|COL|>14<|COL|>4 +Brand#35<|COL|>STANDARD ANODIZED STEEL<|COL|>23<|COL|>4 +Brand#35<|COL|>STANDARD BRUSHED BRASS<|COL|>3<|COL|>4 +Brand#35<|COL|>STANDARD BRUSHED NICKEL<|COL|>49<|COL|>4 +Brand#35<|COL|>STANDARD PLATED STEEL<|COL|>14<|COL|>4 +Brand#41<|COL|>MEDIUM ANODIZED NICKEL<|COL|>9<|COL|>4 +Brand#41<|COL|>MEDIUM BRUSHED TIN<|COL|>9<|COL|>4 +Brand#41<|COL|>MEDIUM PLATED STEEL<|COL|>19<|COL|>4 +Brand#41<|COL|>PROMO ANODIZED NICKEL<|COL|>9<|COL|>4 +Brand#41<|COL|>SMALL ANODIZED STEEL<|COL|>45<|COL|>4 +Brand#41<|COL|>SMALL POLISHED COPPER<|COL|>14<|COL|>4 +Brand#41<|COL|>STANDARD ANODIZED NICKEL<|COL|>9<|COL|>4 +Brand#41<|COL|>STANDARD ANODIZED TIN<|COL|>36<|COL|>4 +Brand#41<|COL|>STANDARD ANODIZED TIN<|COL|>49<|COL|>4 +Brand#41<|COL|>STANDARD BRUSHED TIN<|COL|>45<|COL|>4 +Brand#41<|COL|>STANDARD PLATED TIN<|COL|>49<|COL|>4 +Brand#42<|COL|>ECONOMY BRUSHED COPPER<|COL|>14<|COL|>4 +Brand#42<|COL|>LARGE ANODIZED NICKEL<|COL|>49<|COL|>4 +Brand#42<|COL|>MEDIUM PLATED TIN<|COL|>45<|COL|>4 +Brand#42<|COL|>PROMO BRUSHED STEEL<|COL|>19<|COL|>4 +Brand#42<|COL|>PROMO BURNISHED TIN<|COL|>49<|COL|>4 +Brand#42<|COL|>PROMO PLATED STEEL<|COL|>19<|COL|>4 +Brand#42<|COL|>PROMO PLATED STEEL<|COL|>45<|COL|>4 +Brand#42<|COL|>STANDARD BURNISHED NICKEL<|COL|>49<|COL|>4 +Brand#42<|COL|>STANDARD PLATED COPPER<|COL|>19<|COL|>4 +Brand#43<|COL|>ECONOMY ANODIZED COPPER<|COL|>19<|COL|>4 +Brand#43<|COL|>ECONOMY ANODIZED NICKEL<|COL|>49<|COL|>4 +Brand#43<|COL|>ECONOMY PLATED TIN<|COL|>19<|COL|>4 +Brand#43<|COL|>ECONOMY POLISHED TIN<|COL|>45<|COL|>4 +Brand#43<|COL|>LARGE BURNISHED COPPER<|COL|>3<|COL|>4 +Brand#43<|COL|>LARGE POLISHED TIN<|COL|>45<|COL|>4 +Brand#43<|COL|>MEDIUM ANODIZED BRASS<|COL|>14<|COL|>4 +Brand#43<|COL|>MEDIUM ANODIZED COPPER<|COL|>36<|COL|>4 +Brand#43<|COL|>MEDIUM ANODIZED COPPER<|COL|>49<|COL|>4 +Brand#43<|COL|>MEDIUM BURNISHED TIN<|COL|>23<|COL|>4 +Brand#43<|COL|>PROMO BRUSHED BRASS<|COL|>36<|COL|>4 +Brand#43<|COL|>PROMO BURNISHED STEEL<|COL|>3<|COL|>4 +Brand#43<|COL|>PROMO POLISHED BRASS<|COL|>19<|COL|>4 +Brand#43<|COL|>SMALL BRUSHED NICKEL<|COL|>9<|COL|>4 +Brand#43<|COL|>SMALL POLISHED STEEL<|COL|>19<|COL|>4 +Brand#43<|COL|>STANDARD ANODIZED BRASS<|COL|>3<|COL|>4 +Brand#43<|COL|>STANDARD PLATED TIN<|COL|>14<|COL|>4 +Brand#44<|COL|>ECONOMY ANODIZED NICKEL<|COL|>36<|COL|>4 +Brand#44<|COL|>ECONOMY POLISHED NICKEL<|COL|>23<|COL|>4 +Brand#44<|COL|>LARGE ANODIZED BRASS<|COL|>19<|COL|>4 +Brand#44<|COL|>LARGE BRUSHED TIN<|COL|>3<|COL|>4 +Brand#44<|COL|>MEDIUM BRUSHED STEEL<|COL|>19<|COL|>4 +Brand#44<|COL|>MEDIUM BURNISHED COPPER<|COL|>45<|COL|>4 +Brand#44<|COL|>MEDIUM BURNISHED NICKEL<|COL|>23<|COL|>4 +Brand#44<|COL|>MEDIUM PLATED COPPER<|COL|>14<|COL|>4 +Brand#44<|COL|>SMALL ANODIZED COPPER<|COL|>23<|COL|>4 +Brand#44<|COL|>SMALL ANODIZED TIN<|COL|>45<|COL|>4 +Brand#44<|COL|>SMALL PLATED COPPER<|COL|>19<|COL|>4 +Brand#44<|COL|>STANDARD ANODIZED COPPER<|COL|>3<|COL|>4 +Brand#44<|COL|>STANDARD ANODIZED NICKEL<|COL|>36<|COL|>4 +Brand#51<|COL|>ECONOMY ANODIZED STEEL<|COL|>9<|COL|>4 +Brand#51<|COL|>ECONOMY PLATED NICKEL<|COL|>49<|COL|>4 +Brand#51<|COL|>ECONOMY POLISHED COPPER<|COL|>9<|COL|>4 +Brand#51<|COL|>ECONOMY POLISHED STEEL<|COL|>49<|COL|>4 +Brand#51<|COL|>LARGE BURNISHED BRASS<|COL|>19<|COL|>4 +Brand#51<|COL|>LARGE POLISHED STEEL<|COL|>19<|COL|>4 +Brand#51<|COL|>MEDIUM ANODIZED TIN<|COL|>14<|COL|>4 +Brand#51<|COL|>PROMO BRUSHED BRASS<|COL|>23<|COL|>4 +Brand#51<|COL|>PROMO POLISHED STEEL<|COL|>49<|COL|>4 +Brand#51<|COL|>SMALL BRUSHED TIN<|COL|>36<|COL|>4 +Brand#51<|COL|>SMALL POLISHED STEEL<|COL|>49<|COL|>4 +Brand#51<|COL|>STANDARD BRUSHED COPPER<|COL|>3<|COL|>4 +Brand#51<|COL|>STANDARD BRUSHED NICKEL<|COL|>19<|COL|>4 +Brand#51<|COL|>STANDARD BURNISHED COPPER<|COL|>19<|COL|>4 +Brand#52<|COL|>ECONOMY ANODIZED BRASS<|COL|>14<|COL|>4 +Brand#52<|COL|>ECONOMY ANODIZED COPPER<|COL|>36<|COL|>4 +Brand#52<|COL|>ECONOMY BURNISHED NICKEL<|COL|>19<|COL|>4 +Brand#52<|COL|>ECONOMY BURNISHED STEEL<|COL|>36<|COL|>4 +Brand#52<|COL|>ECONOMY PLATED TIN<|COL|>23<|COL|>4 +Brand#52<|COL|>LARGE BRUSHED NICKEL<|COL|>19<|COL|>4 +Brand#52<|COL|>LARGE BURNISHED TIN<|COL|>45<|COL|>4 +Brand#52<|COL|>LARGE PLATED STEEL<|COL|>9<|COL|>4 +Brand#52<|COL|>LARGE PLATED TIN<|COL|>9<|COL|>4 +Brand#52<|COL|>LARGE POLISHED NICKEL<|COL|>36<|COL|>4 +Brand#52<|COL|>MEDIUM BURNISHED TIN<|COL|>45<|COL|>4 +Brand#52<|COL|>SMALL ANODIZED NICKEL<|COL|>36<|COL|>4 +Brand#52<|COL|>SMALL ANODIZED STEEL<|COL|>9<|COL|>4 +Brand#52<|COL|>SMALL BRUSHED STEEL<|COL|>23<|COL|>4 +Brand#52<|COL|>SMALL BURNISHED NICKEL<|COL|>14<|COL|>4 +Brand#52<|COL|>STANDARD POLISHED STEEL<|COL|>19<|COL|>4 +Brand#53<|COL|>LARGE BURNISHED NICKEL<|COL|>23<|COL|>4 +Brand#53<|COL|>LARGE PLATED BRASS<|COL|>9<|COL|>4 +Brand#53<|COL|>LARGE PLATED STEEL<|COL|>49<|COL|>4 +Brand#53<|COL|>MEDIUM BRUSHED COPPER<|COL|>3<|COL|>4 +Brand#53<|COL|>MEDIUM BRUSHED STEEL<|COL|>45<|COL|>4 +Brand#53<|COL|>SMALL BRUSHED BRASS<|COL|>36<|COL|>4 +Brand#53<|COL|>STANDARD PLATED STEEL<|COL|>45<|COL|>4 +Brand#54<|COL|>ECONOMY ANODIZED BRASS<|COL|>9<|COL|>4 +Brand#54<|COL|>ECONOMY BRUSHED TIN<|COL|>19<|COL|>4 +Brand#54<|COL|>ECONOMY POLISHED BRASS<|COL|>49<|COL|>4 +Brand#54<|COL|>LARGE ANODIZED BRASS<|COL|>49<|COL|>4 +Brand#54<|COL|>LARGE BURNISHED BRASS<|COL|>49<|COL|>4 +Brand#54<|COL|>LARGE BURNISHED TIN<|COL|>14<|COL|>4 +Brand#54<|COL|>LARGE POLISHED BRASS<|COL|>19<|COL|>4 +Brand#54<|COL|>MEDIUM BURNISHED STEEL<|COL|>3<|COL|>4 +Brand#54<|COL|>SMALL BURNISHED STEEL<|COL|>19<|COL|>4 +Brand#54<|COL|>SMALL PLATED BRASS<|COL|>23<|COL|>4 +Brand#54<|COL|>SMALL PLATED TIN<|COL|>14<|COL|>4 +Brand#55<|COL|>LARGE BRUSHED NICKEL<|COL|>9<|COL|>4 +Brand#55<|COL|>LARGE PLATED TIN<|COL|>9<|COL|>4 +Brand#55<|COL|>LARGE POLISHED STEEL<|COL|>36<|COL|>4 +Brand#55<|COL|>MEDIUM BRUSHED TIN<|COL|>45<|COL|>4 +Brand#55<|COL|>PROMO BRUSHED STEEL<|COL|>36<|COL|>4 +Brand#55<|COL|>PROMO BURNISHED STEEL<|COL|>14<|COL|>4 +Brand#55<|COL|>SMALL PLATED COPPER<|COL|>45<|COL|>4 +Brand#55<|COL|>STANDARD ANODIZED BRASS<|COL|>36<|COL|>4 +Brand#55<|COL|>STANDARD BRUSHED COPPER<|COL|>3<|COL|>4 +Brand#55<|COL|>STANDARD BRUSHED STEEL<|COL|>19<|COL|>4 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q17.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q17.out new file mode 100644 index 000000000..0339a4c9e --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q17.out @@ -0,0 +1,2 @@ +1 +2550.622857 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q18.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q18.out new file mode 100644 index 000000000..3cc4aa5a7 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q18.out @@ -0,0 +1,3 @@ +2 +Customer#000000667<|COL|>667<|COL|>29158<|COL|>1995-10-21<|COL|>439687.23<|COL|>305.00 +Customer#000000178<|COL|>178<|COL|>6882<|COL|>1997-04-09<|COL|>422359.65<|COL|>303.00 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q19.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q19.out new file mode 100644 index 000000000..0affe673c --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q19.out @@ -0,0 +1,2 @@ +1 +22923.0280 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q2.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q2.out new file mode 100644 index 000000000..7ac0f8c37 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q2.out @@ -0,0 +1,5 @@ +4 +4186.95<|COL|>Supplier#000000077<|COL|>GERMANY<|COL|>249<|COL|>Manufacturer#4<|COL|>wVtcr0uH3CyrSiWMLsqnB09Syo,UuZxPMeBghlY<|COL|>17-281-345-4863<|COL|>the slyly final asymptotes. blithely pending theodoli +1883.37<|COL|>Supplier#000000086<|COL|>ROMANIA<|COL|>1015<|COL|>Manufacturer#4<|COL|>J1fgg5QaqnN<|COL|>29-903-665-7065<|COL|>cajole furiously special, final requests: furiously spec +1687.81<|COL|>Supplier#000000017<|COL|>ROMANIA<|COL|>1634<|COL|>Manufacturer#2<|COL|>c2d,ESHRSkK3WYnxpgw6aOqN0q<|COL|>29-601-884-9219<|COL|>eep against the furiously bold ideas. fluffily bold packa +287.16<|COL|>Supplier#000000052<|COL|>ROMANIA<|COL|>323<|COL|>Manufacturer#4<|COL|>WCk XCHYzBA1dvJDSol4ZJQQcQN,<|COL|>29-974-934-4713<|COL|>dolites are slyly against the furiously regular packages. ironic, final deposits cajole quickly diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q20.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q20.out new file mode 100644 index 000000000..82dfc5e81 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q20.out @@ -0,0 +1,2 @@ +1 +Supplier#000000013<|COL|>HK71HQyWoqRWOX8GI FpgAifW,2PoH diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q21.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q21.out new file mode 100644 index 000000000..a9797898d --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q21.out @@ -0,0 +1,2 @@ +1 +Supplier#000000074<|COL|>9 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q22.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q22.out new file mode 100644 index 000000000..149736344 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q22.out @@ -0,0 +1,8 @@ +7 +13<|COL|>10<|COL|>75359.29 +17<|COL|>8<|COL|>62288.98 +18<|COL|>14<|COL|>111072.45 +23<|COL|>5<|COL|>40458.86 +29<|COL|>11<|COL|>88722.85 +30<|COL|>17<|COL|>122189.33 +31<|COL|>8<|COL|>66313.16 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q3.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q3.out new file mode 100644 index 000000000..a99412042 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q3.out @@ -0,0 +1,11 @@ +10 +47714<|COL|>267010.5894<|COL|>1995-03-11<|COL|>0 +22276<|COL|>266351.5562<|COL|>1995-01-29<|COL|>0 +32965<|COL|>263768.3414<|COL|>1995-02-25<|COL|>0 +21956<|COL|>254541.1285<|COL|>1995-02-02<|COL|>0 +1637<|COL|>243512.7981<|COL|>1995-02-08<|COL|>0 +10916<|COL|>241320.0814<|COL|>1995-03-11<|COL|>0 +30497<|COL|>208566.6969<|COL|>1995-02-07<|COL|>0 +450<|COL|>205447.4232<|COL|>1995-03-05<|COL|>0 +47204<|COL|>204478.5213<|COL|>1995-03-13<|COL|>0 +9696<|COL|>201502.2188<|COL|>1995-02-20<|COL|>0 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q4.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q4.out new file mode 100644 index 000000000..8d51c20b4 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q4.out @@ -0,0 +1,6 @@ +5 +1-URGENT<|COL|>93 +2-HIGH<|COL|>103 +3-MEDIUM<|COL|>109 +4-NOT SPECIFIED<|COL|>102 +5-LOW<|COL|>128 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q5.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q5.out new file mode 100644 index 000000000..f383c8bbf --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q5.out @@ -0,0 +1,6 @@ +5 +VIETNAM<|COL|>1000926.6999 +CHINA<|COL|>740210.7570 +JAPAN<|COL|>660651.2425 +INDONESIA<|COL|>566379.5276 +INDIA<|COL|>422874.6844 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q6.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q6.out new file mode 100644 index 000000000..5aa2c5a96 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q6.out @@ -0,0 +1,2 @@ +1 +1193053.2253 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q7.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q7.out new file mode 100644 index 000000000..6f9d059fe --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q7.out @@ -0,0 +1,5 @@ +4 +FRANCE<|COL|>GERMANY<|COL|>1995<|COL|>268068.5774 +FRANCE<|COL|>GERMANY<|COL|>1996<|COL|>303862.2980 +GERMANY<|COL|>FRANCE<|COL|>1995<|COL|>621159.4882 +GERMANY<|COL|>FRANCE<|COL|>1996<|COL|>379095.8854 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q8.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q8.out new file mode 100644 index 000000000..5adac6bde --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q8.out @@ -0,0 +1,3 @@ +2 +1995<|COL|>0.000000 +1996<|COL|>0.000000 diff --git a/spark-extension-shims-spark/src/test/resources/tpch-query-results/q9.out b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q9.out new file mode 100644 index 000000000..52ab2eb67 --- /dev/null +++ b/spark-extension-shims-spark/src/test/resources/tpch-query-results/q9.out @@ -0,0 +1,174 @@ +173 +ALGERIA<|COL|>1998<|COL|>97864.5682 +ALGERIA<|COL|>1997<|COL|>368231.6695 +ALGERIA<|COL|>1996<|COL|>196525.8046 +ALGERIA<|COL|>1995<|COL|>341438.6885 +ALGERIA<|COL|>1994<|COL|>677444.0160 +ALGERIA<|COL|>1993<|COL|>458756.9157 +ALGERIA<|COL|>1992<|COL|>549243.9511 +ARGENTINA<|COL|>1998<|COL|>80448.7680 +ARGENTINA<|COL|>1997<|COL|>186279.1618 +ARGENTINA<|COL|>1996<|COL|>154041.8822 +ARGENTINA<|COL|>1995<|COL|>113143.3119 +ARGENTINA<|COL|>1994<|COL|>169680.4239 +ARGENTINA<|COL|>1993<|COL|>116513.8141 +ARGENTINA<|COL|>1992<|COL|>202404.7608 +BRAZIL<|COL|>1998<|COL|>75952.5946 +BRAZIL<|COL|>1997<|COL|>190548.1104 +BRAZIL<|COL|>1996<|COL|>219059.0692 +BRAZIL<|COL|>1995<|COL|>186435.2023 +BRAZIL<|COL|>1994<|COL|>96835.1870 +BRAZIL<|COL|>1993<|COL|>186365.4109 +BRAZIL<|COL|>1992<|COL|>152546.4439 +CANADA<|COL|>1998<|COL|>101030.3336 +CANADA<|COL|>1997<|COL|>101197.3441 +CANADA<|COL|>1996<|COL|>257697.1355 +CANADA<|COL|>1995<|COL|>91474.8820 +CANADA<|COL|>1994<|COL|>249182.7548 +CANADA<|COL|>1993<|COL|>185737.8379 +CANADA<|COL|>1992<|COL|>143371.7465 +CHINA<|COL|>1998<|COL|>508364.5444 +CHINA<|COL|>1997<|COL|>650235.1646 +CHINA<|COL|>1996<|COL|>911366.0698 +CHINA<|COL|>1995<|COL|>797268.4076 +CHINA<|COL|>1994<|COL|>529989.3095 +CHINA<|COL|>1993<|COL|>573864.3972 +CHINA<|COL|>1992<|COL|>751688.7613 +EGYPT<|COL|>1998<|COL|>306325.2842 +EGYPT<|COL|>1997<|COL|>568461.6699 +EGYPT<|COL|>1996<|COL|>465081.9232 +EGYPT<|COL|>1995<|COL|>542886.5087 +EGYPT<|COL|>1994<|COL|>745807.8123 +EGYPT<|COL|>1993<|COL|>381503.2008 +EGYPT<|COL|>1992<|COL|>641866.4367 +ETHIOPIA<|COL|>1998<|COL|>226054.5716 +ETHIOPIA<|COL|>1997<|COL|>585193.2802 +ETHIOPIA<|COL|>1996<|COL|>405412.7741 +ETHIOPIA<|COL|>1995<|COL|>270455.7637 +ETHIOPIA<|COL|>1994<|COL|>567875.4279 +ETHIOPIA<|COL|>1993<|COL|>412302.2871 +ETHIOPIA<|COL|>1992<|COL|>551284.5821 +FRANCE<|COL|>1998<|COL|>135723.4050 +FRANCE<|COL|>1997<|COL|>249664.7578 +FRANCE<|COL|>1996<|COL|>175882.8934 +FRANCE<|COL|>1995<|COL|>116394.7866 +FRANCE<|COL|>1994<|COL|>197695.2438 +FRANCE<|COL|>1993<|COL|>231878.6201 +FRANCE<|COL|>1992<|COL|>199131.2037 +GERMANY<|COL|>1998<|COL|>172741.1024 +GERMANY<|COL|>1997<|COL|>393833.4660 +GERMANY<|COL|>1996<|COL|>335634.5936 +GERMANY<|COL|>1995<|COL|>378106.0763 +GERMANY<|COL|>1994<|COL|>250107.6653 +GERMANY<|COL|>1993<|COL|>327154.9365 +GERMANY<|COL|>1992<|COL|>387240.0885 +INDIA<|COL|>1998<|COL|>347548.7604 +INDIA<|COL|>1997<|COL|>656797.9670 +INDIA<|COL|>1996<|COL|>522759.3529 +INDIA<|COL|>1995<|COL|>574428.6693 +INDIA<|COL|>1994<|COL|>741983.7846 +INDIA<|COL|>1993<|COL|>729948.5341 +INDIA<|COL|>1992<|COL|>661061.1415 +INDONESIA<|COL|>1998<|COL|>91791.5096 +INDONESIA<|COL|>1997<|COL|>183956.4613 +INDONESIA<|COL|>1996<|COL|>415234.7848 +INDONESIA<|COL|>1995<|COL|>427155.3804 +INDONESIA<|COL|>1994<|COL|>286271.2875 +INDONESIA<|COL|>1993<|COL|>551178.8823 +INDONESIA<|COL|>1992<|COL|>274513.2685 +IRAN<|COL|>1998<|COL|>47959.8219 +IRAN<|COL|>1997<|COL|>184335.0615 +IRAN<|COL|>1996<|COL|>223115.2464 +IRAN<|COL|>1995<|COL|>125339.0927 +IRAN<|COL|>1994<|COL|>117228.3122 +IRAN<|COL|>1993<|COL|>208030.3229 +IRAN<|COL|>1992<|COL|>161835.5475 +IRAQ<|COL|>1998<|COL|>161797.4924 +IRAQ<|COL|>1997<|COL|>224876.5436 +IRAQ<|COL|>1996<|COL|>145277.8980 +IRAQ<|COL|>1995<|COL|>467955.2505 +IRAQ<|COL|>1994<|COL|>97455.2990 +IRAQ<|COL|>1993<|COL|>114821.6440 +IRAQ<|COL|>1992<|COL|>213307.1574 +JAPAN<|COL|>1998<|COL|>307594.5980 +JAPAN<|COL|>1997<|COL|>339018.1488 +JAPAN<|COL|>1996<|COL|>649578.3368 +JAPAN<|COL|>1995<|COL|>671644.0911 +JAPAN<|COL|>1994<|COL|>576266.2386 +JAPAN<|COL|>1993<|COL|>514190.8437 +JAPAN<|COL|>1992<|COL|>534914.9339 +JORDAN<|COL|>1996<|COL|>33460.2447 +JORDAN<|COL|>1995<|COL|>20364.1623 +JORDAN<|COL|>1994<|COL|>15528.6088 +JORDAN<|COL|>1993<|COL|>14640.9889 +JORDAN<|COL|>1992<|COL|>10904.2931 +KENYA<|COL|>1998<|COL|>521926.5198 +KENYA<|COL|>1997<|COL|>559632.3408 +KENYA<|COL|>1996<|COL|>772855.7939 +KENYA<|COL|>1995<|COL|>516452.5067 +KENYA<|COL|>1994<|COL|>543665.8154 +KENYA<|COL|>1993<|COL|>866924.8754 +KENYA<|COL|>1992<|COL|>567410.5502 +MOROCCO<|COL|>1998<|COL|>217794.4973 +MOROCCO<|COL|>1997<|COL|>439240.9287 +MOROCCO<|COL|>1996<|COL|>399969.4680 +MOROCCO<|COL|>1995<|COL|>258131.9398 +MOROCCO<|COL|>1994<|COL|>386972.1424 +MOROCCO<|COL|>1993<|COL|>145468.0381 +MOROCCO<|COL|>1992<|COL|>284314.2813 +MOZAMBIQUE<|COL|>1998<|COL|>518693.2238 +MOZAMBIQUE<|COL|>1997<|COL|>613873.2961 +MOZAMBIQUE<|COL|>1996<|COL|>936793.5612 +MOZAMBIQUE<|COL|>1995<|COL|>727204.7718 +MOZAMBIQUE<|COL|>1994<|COL|>1104618.1807 +MOZAMBIQUE<|COL|>1993<|COL|>893266.0530 +MOZAMBIQUE<|COL|>1992<|COL|>1062432.0884 +PERU<|COL|>1998<|COL|>287242.9797 +PERU<|COL|>1997<|COL|>532358.3660 +PERU<|COL|>1996<|COL|>398435.7507 +PERU<|COL|>1995<|COL|>462031.6251 +PERU<|COL|>1994<|COL|>304235.4118 +PERU<|COL|>1993<|COL|>505885.4890 +PERU<|COL|>1992<|COL|>382290.0947 +ROMANIA<|COL|>1998<|COL|>357824.5528 +ROMANIA<|COL|>1997<|COL|>569806.5564 +ROMANIA<|COL|>1996<|COL|>732001.5568 +ROMANIA<|COL|>1995<|COL|>408657.1154 +ROMANIA<|COL|>1994<|COL|>540702.5463 +ROMANIA<|COL|>1993<|COL|>883158.5056 +ROMANIA<|COL|>1992<|COL|>505488.9501 +RUSSIA<|COL|>1998<|COL|>34448.6357 +RUSSIA<|COL|>1997<|COL|>314972.0446 +RUSSIA<|COL|>1996<|COL|>430049.5821 +RUSSIA<|COL|>1995<|COL|>360538.0586 +RUSSIA<|COL|>1994<|COL|>301791.0114 +RUSSIA<|COL|>1993<|COL|>308993.9622 +RUSSIA<|COL|>1992<|COL|>289868.6564 +SAUDI ARABIA<|COL|>1998<|COL|>16502.4100 +SAUDI ARABIA<|COL|>1997<|COL|>61830.9556 +SAUDI ARABIA<|COL|>1996<|COL|>213650.2809 +SAUDI ARABIA<|COL|>1995<|COL|>62668.7250 +SAUDI ARABIA<|COL|>1994<|COL|>94629.1538 +SAUDI ARABIA<|COL|>1993<|COL|>57768.3071 +SAUDI ARABIA<|COL|>1992<|COL|>66520.1093 +UNITED KINGDOM<|COL|>1998<|COL|>80437.6523 +UNITED KINGDOM<|COL|>1997<|COL|>252509.7351 +UNITED KINGDOM<|COL|>1996<|COL|>231152.8582 +UNITED KINGDOM<|COL|>1995<|COL|>181310.8808 +UNITED KINGDOM<|COL|>1994<|COL|>239161.2061 +UNITED KINGDOM<|COL|>1993<|COL|>122103.1142 +UNITED KINGDOM<|COL|>1992<|COL|>60882.3080 +UNITED STATES<|COL|>1998<|COL|>440347.6658 +UNITED STATES<|COL|>1997<|COL|>652958.9371 +UNITED STATES<|COL|>1996<|COL|>1004593.8282 +UNITED STATES<|COL|>1995<|COL|>860144.1029 +UNITED STATES<|COL|>1994<|COL|>807797.4877 +UNITED STATES<|COL|>1993<|COL|>736669.4711 +UNITED STATES<|COL|>1992<|COL|>877851.4103 +VIETNAM<|COL|>1998<|COL|>358248.0159 +VIETNAM<|COL|>1997<|COL|>394817.2842 +VIETNAM<|COL|>1996<|COL|>439390.0836 +VIETNAM<|COL|>1995<|COL|>418626.6325 +VIETNAM<|COL|>1994<|COL|>422644.8168 +VIETNAM<|COL|>1993<|COL|>309063.4020 +VIETNAM<|COL|>1992<|COL|>716126.5378 diff --git a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronTPCHSuite.scala b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronTPCHSuite.scala new file mode 100644 index 000000000..b917b3d43 --- /dev/null +++ b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronTPCHSuite.scala @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.auron + +import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.file.Paths + +import scala.collection.mutable +import scala.util.matching.Regex + +import org.apache.commons.io.FileUtils +import org.apache.spark.SparkConf +import org.apache.spark.sql.{DataFrame, QueryTest, Row} +import org.apache.spark.sql.execution.FormattedMode +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.DoubleType + +abstract class AuronTPCHSuite extends QueryTest with SharedSparkSession { + + protected val regenGoldenFiles: Boolean = + sys.env.getOrElse("REGEN_TPCH_GOLDEN_FILES", "0") == "1" + + protected val baseResourcePath = + Paths.get("src", "test", "resources").toAbsolutePath.toString + protected val tpchDataPath: String = s"$baseResourcePath/tpch-data-parquet" + protected val tpchQueriesPath: String = s"$baseResourcePath/tpch-queries" + protected val tpchResultsPath: String = s"$baseResourcePath/tpch-query-results" + protected val tpchPlanPath: String = s"$baseResourcePath/tpch-plan-stability" + + protected val colSep: String = "<|COL|>" + + protected val tpchQueries: Seq[String] = (1 to 22).map("q" + _) + + protected val tpchTables: Seq[String] = + Seq("customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier") + + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.sql.extensions", "org.apache.spark.sql.auron.AuronSparkSessionExtension") + .set( + "spark.shuffle.manager", + "org.apache.spark.sql.execution.auron.shuffle.AuronShuffleManager") + .set("spark.memory.offHeap.enabled", "false") + .set("spark.ui.enabled", "false") + .set("spark.auron.enable", "true") + } + + override def beforeAll(): Unit = { + super.beforeAll() + spark.sparkContext.setLogLevel("WARN") + createTPCHTables() + } + + protected def createTPCHTables(): Unit = { + tpchTables + .foreach { tableName => + spark.read.parquet(s"$tpchDataPath/$tableName").createOrReplaceTempView(tableName) + tableName -> spark.table(tableName).count() + } + } + + def shouldVerifyPhysicalPlan(): Boolean = { + Shims.get.shimVersion match { + case "spark-3.5" => true + case _ => false // TODO: Support for other Spark versions in the future + } + } + + protected def checkQueryResult(df: DataFrame, queryId: String): Unit = { + val goldenFile = new File(s"$tpchResultsPath/$queryId.out") + val rows = df.collect() + + if (regenGoldenFiles) { + writeGoldenFile(goldenFile, formatResultContent(rows)) + return + } + + if (df.schema.exists(_.dataType == DoubleType)) { + compareDoubleResult(queryId, rows, goldenFile) + } else { + compareResultStr(queryId, rows, goldenFile) + } + } + + private def formatResultContent(rows: Array[Row]): String = { + val rowStrings = rows.map(_.mkString(colSep)) + s"${rows.length}\n${rowStrings.mkString("\n")}\n" + } + + private def writeGoldenFile(file: File, content: String): Unit = { + Option(file.getParentFile).foreach(_.mkdirs()) + FileUtils.writeStringToFile(file, content, StandardCharsets.UTF_8) + } + + protected def compareResultStr(sqlNum: String, rows: Array[Row], goldenFile: File): Unit = { + val actualContent = formatResultContent(rows) + + val expectedResult = FileUtils.readFileToString(goldenFile, StandardCharsets.UTF_8) + if (expectedResult != actualContent) { + fail(s""" + |=== $sqlNum result does NOT match expected === + |[Expected] + |${expectedResult} + |[Actual] + |${actualContent} + |""".stripMargin) + } + } + + protected def compareDoubleResult( + queryId: String, + rows: Array[Row], + goldenFile: File, + tolerance: Double = 1e-6): Unit = { + + val expectedRowIter = FileUtils.readLines(goldenFile, StandardCharsets.UTF_8).iterator() + val expectedRowCount = expectedRowIter.next().toInt + assert( + rows.length == expectedRowCount, + s"Row count mismatch in $queryId: expected $expectedRowCount, got ${rows.length}") + + rows.zipWithIndex.foreach { case (actualRow, rowIdx) => + assert(expectedRowIter.hasNext) + val expectedRow = expectedRowIter.next().split(Regex.quote(colSep)) + + actualRow.schema.zipWithIndex.foreach { case (field, colIdx) => + field.dataType match { + case DoubleType => + val actualValue = actualRow.getDouble(colIdx) + val expectedValue = expectedRow(colIdx).toDouble + val diff = math.abs(actualValue - expectedValue) + assert( + diff < tolerance, + s"Floating-point mismatch in $queryId row $rowIdx col $colIdx: " + + s"expected ${expectedValue}, got ${actualValue} (diff=$diff)") + case _ => + val actualValue = actualRow.get(colIdx).toString + val expectedValue = expectedRow(colIdx) + assert( + actualValue == expectedValue, + s"Mismatch in $queryId row $rowIdx col $colIdx: " + + s"expected $expectedValue, got $actualValue") + } + } + } + } + + private def normalizePhysicalPlan(plan: String): String = { + val exprIdRegex = "#\\d+L?".r + val planIdRegex = "plan_id=\\d+".r + + // Normalize file location + def normalizeLocation(plan: String): String = { + plan.replaceAll("""file:/[^,\s\]\)]+""", "file:/") + } + + // Create a normalized map for regex matches + def createNormalizedMap(regex: Regex, plan: String): Map[String, String] = { + val map = new mutable.HashMap[String, String]() + regex + .findAllMatchIn(plan) + .map(_.toString) + .foreach(map.getOrElseUpdate(_, (map.size + 1).toString)) + map.toMap + } + + // Replace occurrences in the plan using the normalized map + def replaceWithNormalizedValues( + plan: String, + regex: Regex, + normalizedMap: Map[String, String], + format: String): String = { + regex.replaceAllIn(plan, regexMatch => s"$format${normalizedMap(regexMatch.toString)}") + } + + // Normalize the entire plan step by step + val exprIdMap = createNormalizedMap(exprIdRegex, plan) + val exprIdNormalized = replaceWithNormalizedValues(plan, exprIdRegex, exprIdMap, "#") + + val planIdMap = createNormalizedMap(planIdRegex, exprIdNormalized) + val planIdNormalized = + replaceWithNormalizedValues(exprIdNormalized, planIdRegex, planIdMap, "plan_id=") + + // QueryStageExec will take its id as argument, replace it with X + val argumentsNormalized = planIdNormalized + .replaceAll("Arguments: [0-9]+, [0-9]+", "Arguments: X, X") + .replaceAll("Arguments: [0-9]+", "Arguments: X") + + normalizeLocation(argumentsNormalized) + } + + protected def checkPhysicalPlan(df: DataFrame, queryId: String): Unit = { + if (!shouldVerifyPhysicalPlan()) { + return + } + + val goldenPlanFile = new File(s"$tpchPlanPath/$queryId.txt") + val actualPlan = normalizePhysicalPlan(df.queryExecution.explainString(FormattedMode)) + + if (regenGoldenFiles) { + writeGoldenFile(goldenPlanFile, actualPlan) + return + } + + val expectedPlan = FileUtils.readFileToString(goldenPlanFile, StandardCharsets.UTF_8) + if (expectedPlan != actualPlan) { + val actualTempFile = new File(FileUtils.getTempDirectory, s"tpch.actual.plan.$queryId.txt") + FileUtils.writeStringToFile(actualTempFile, actualPlan, StandardCharsets.UTF_8) + fail(s""" + |Physical plan mismatch for query $queryId + |Expected: ${goldenPlanFile.getAbsolutePath} + |Actual : ${actualTempFile.getAbsolutePath} + | + |--- Expected --- + |$expectedPlan + | + |--- Actual --- + |$actualPlan + |""".stripMargin) + } + } + + tpchQueries.foreach { queryId => + test(s"TPC-H $queryId") { + val queryFile = new File(s"$tpchQueriesPath/$queryId.sql") + val sqlText = FileUtils.readFileToString(queryFile, StandardCharsets.UTF_8).trim + + val resultDf = spark.sql(sqlText) + + checkQueryResult(resultDf, queryId) + checkPhysicalPlan(resultDf, queryId) + } + } +} + +/** + * Variant that forces usage of the legacy V1 Parquet reader and disables broadcast joins + * (ensuring sort-merge joins are used). + */ +class AuronTPCHV1Suite extends AuronTPCHSuite { + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.sql.sources.useV1SourceList", "parquet") + .set("spark.sql.autoBroadcastJoinThreshold", "-1") + } +}