Optimizer with operator pool pointer

src/common/symbols.cpp

@@ -15,7 +15,7 @@
 #include "duckdb/main/query_result.hpp"
 #include "duckdb/main/relation.hpp"
 #include "duckdb/main/stream_query_result.hpp"
-#include "duckdb/optimizer/join_order_optimizer.hpp"
+#include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
 #include "duckdb/optimizer/rule.hpp"
 #include "duckdb/parallel/pipeline.hpp"
 #include "duckdb/parallel/meta_pipeline.hpp"

src/include/duckdb/execution/physical_operator.hpp

@@ -14,7 +14,7 @@
 #include "duckdb/common/enums/physical_operator_type.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/execution/execution_context.hpp"
-#include "duckdb/optimizer/join_node.hpp"
+#include "duckdb/optimizer/join_order/join_node.hpp"
 
 namespace duckdb {
 class Event;

src/include/duckdb/optimizer/filter_pushdown.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/optimizer/filter_combiner.hpp"
 #include "duckdb/optimizer/rule.hpp"
+#include "duckdb/optimizer/operator_pool.hpp"
 
 namespace duckdb {
 

src/include/duckdb/optimizer/cardinality_estimator.hpp → src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp

@@ -1,14 +1,14 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/optimizer/cardinality_estimator.hpp
+// duckdb/optimizer/join_order/cardinality_estimator.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 #pragma once
 
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
-#include "duckdb/optimizer/join_node.hpp"
+#include "duckdb/optimizer/join_order/join_node.hpp"
 #include "duckdb/planner/column_binding.hpp"
 #include "duckdb/planner/column_binding_map.hpp"
 #include "duckdb/planner/filter/conjunction_filter.hpp"

src/include/duckdb/optimizer/estimated_properties.hpp → src/include/duckdb/optimizer/join_order/estimated_properties.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/optimizer/estimated_properties.hpp
+// duckdb/optimizer/join_order/estimated_properties.hpp
 //
 //
 //===----------------------------------------------------------------------===//

src/include/duckdb/optimizer/join_node.hpp → src/include/duckdb/optimizer/join_order/join_node.hpp

@@ -1,23 +1,23 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/optimizer/join_node.hpp
+// duckdb/optimizer/join_order/join_node.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/unordered_set.hpp"
-#include "duckdb/optimizer/join_order/query_graph.hpp"
+#include "duckdb/optimizer/join_order/estimated_properties.hpp"
 #include "duckdb/optimizer/join_order/join_relation.hpp"
+#include "duckdb/optimizer/join_order/query_graph.hpp"
 #include "duckdb/parser/expression_map.hpp"
 #include "duckdb/planner/logical_operator_visitor.hpp"
-#include "duckdb/storage/statistics/distinct_statistics.hpp"
 #include "duckdb/planner/table_filter.hpp"
-#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
-#include "duckdb/optimizer/estimated_properties.hpp"
+#include "duckdb/storage/statistics/distinct_statistics.hpp"
 
 namespace duckdb {
 

src/include/duckdb/optimizer/join_order_optimizer.hpp → src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/optimizer/join_order_optimizer.hpp
+// duckdb/optimizer/join_order/join_order_optimizer.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -10,13 +10,13 @@
 
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/unordered_set.hpp"
-#include "duckdb/optimizer/join_order/query_graph.hpp"
 #include "duckdb/optimizer/join_order/join_relation.hpp"
-#include "duckdb/optimizer/join_node.hpp"
+#include "duckdb/optimizer/join_order/cardinality_estimator.hpp"
+#include "duckdb/optimizer/join_order/query_graph.hpp"
+#include "duckdb/optimizer/join_order/join_node.hpp"
 #include "duckdb/parser/expression_map.hpp"
 #include "duckdb/planner/logical_operator.hpp"
 #include "duckdb/planner/logical_operator_visitor.hpp"
-#include "duckdb/optimizer/cardinality_estimator.hpp"
 
 #include <functional>
 

src/include/duckdb/optimizer/operator_pool.hpp

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/operator_pool.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/common/unordered_set.hpp"
+
+namespace duckdb {
+
+class OperatorPool {
+public:
+	OperatorPool() {
+		seen_operators = unordered_set<idx_t>();
+	}
+
+	void AddOperator(LogicalOperator *op);
+	bool InPool(LogicalOperator *op);
+	void EmptyOperatorPool();
+
+private:
+	unordered_set<idx_t> seen_operators;
+};
+} // namespace duckdb

src/include/duckdb/optimizer/optimizer.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/planner/logical_operator.hpp"
 #include "duckdb/planner/logical_operator_visitor.hpp"
 #include "duckdb/common/enums/optimizer_type.hpp"
+#include "duckdb/optimizer/operator_pool.hpp"
 
 #include <functional>
 
@@ -27,6 +28,7 @@ class Optimizer {
 	ClientContext &context;
 	Binder &binder;
 	ExpressionRewriter rewriter;
+	OperatorPool seen_operators;
 
 private:
 	void RunOptimizer(OptimizerType type, const std::function<void()> &callback);

src/include/duckdb/planner/logical_operator.hpp

@@ -9,16 +9,16 @@
 #pragma once
 
 #include "duckdb/catalog/catalog.hpp"
-#include "duckdb/optimizer/estimated_properties.hpp"
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/enums/logical_operator_type.hpp"
+#include "duckdb/optimizer/join_order/estimated_properties.hpp"
+#include "duckdb/planner/column_binding.hpp"
 #include "duckdb/planner/expression.hpp"
 #include "duckdb/planner/logical_operator_visitor.hpp"
-#include "duckdb/planner/column_binding.hpp"
 #include "duckdb/planner/plan_serialization.hpp"
 
-#include <functional>
 #include <algorithm>
+#include <functional>
 
 namespace duckdb {
 

src/optimizer/CMakeLists.txt

@@ -17,11 +17,8 @@ add_library_unity(
   filter_pushdown.cpp
   filter_pullup.cpp
   in_clause_rewriter.cpp
-  join_node.cpp
-  join_order_optimizer.cpp
-  cardinality_estimator.cpp
-  estimated_properties.cpp
   optimizer.cpp
+  operator_pool.cpp
   expression_rewriter.cpp
   regex_range_filter.cpp
   remove_unused_columns.cpp

src/optimizer/filter_pushdown.cpp

@@ -5,6 +5,8 @@
 #include "duckdb/planner/operator/logical_join.hpp"
 #include "duckdb/optimizer/optimizer.hpp"
 
+#include "iostream"
+
 namespace duckdb {
 
 using Filter = FilterPushdown::Filter;
@@ -13,6 +15,11 @@ FilterPushdown::FilterPushdown(Optimizer &optimizer) : optimizer(optimizer), com
 }
 
 unique_ptr<LogicalOperator> FilterPushdown::Rewrite(unique_ptr<LogicalOperator> op) {
+	if (optimizer.seen_operators.InPool(op.get())) {
+		// We've already optimized this operator, so just return.
+		return move(op);
+	}
+	optimizer.seen_operators.AddOperator(op.get());
 	D_ASSERT(!combiner.HasFilters());
 	switch (op->type) {
 	case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY:

src/optimizer/join_order/CMakeLists.txt

@@ -1,5 +1,12 @@
-add_library_unity(duckdb_optimizer_join_order OBJECT query_graph.cpp
-                  join_relation_set.cpp)
+add_library_unity(
+  duckdb_optimizer_join_order
+  OBJECT
+  query_graph.cpp
+  cardinality_estimator.cpp
+  join_relation_set.cpp
+  join_node.cpp
+  estimated_properties.cpp
+  join_order_optimizer.cpp)
 set(ALL_OBJECT_FILES
     ${ALL_OBJECT_FILES} $<TARGET_OBJECTS:duckdb_optimizer_join_order>
     PARENT_SCOPE)

src/optimizer/cardinality_estimator.cpp → src/optimizer/join_order/cardinality_estimator.cpp

@@ -1,12 +1,12 @@
+#include "duckdb/function/table/table_scan.hpp"
+#include "duckdb/optimizer/join_order/join_node.hpp"
+#include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
 #include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
-#include "duckdb/optimizer/join_order_optimizer.hpp"
-#include "duckdb/optimizer/join_node.hpp"
-#include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/storage/statistics/numeric_statistics.hpp"
-#include "duckdb/function/table/table_scan.hpp"
 
 namespace duckdb {
 

src/optimizer/estimated_properties.cpp → src/optimizer/join_order/estimated_properties.cpp

@@ -1,5 +1,5 @@
 
-#include "duckdb/optimizer/estimated_properties.hpp"
+#include "duckdb/optimizer/join_order/estimated_properties.hpp"
 
 namespace duckdb {
 

src/optimizer/join_node.cpp → src/optimizer/join_order/join_node.cpp

@@ -1,7 +1,8 @@
+#include "duckdb/optimizer/join_order/join_node.hpp"
+
 #include "duckdb/common/limits.hpp"
 #include "duckdb/planner/expression/list.hpp"
 #include "duckdb/planner/operator/list.hpp"
-#include "duckdb/optimizer/join_node.hpp"
 
 namespace duckdb {
 

src/optimizer/join_order_optimizer.cpp → src/optimizer/join_order/join_order_optimizer.cpp

@@ -1,7 +1,8 @@
-#include "duckdb/optimizer/join_order_optimizer.hpp"
+#include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
+
+#include "duckdb/common/limits.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/main/client_context.hpp"
-#include "duckdb/common/limits.hpp"
 #include "duckdb/planner/expression/list.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/operator/list.hpp"

src/optimizer/operator_pool.cpp

@@ -0,0 +1,23 @@
+//
+// Created by Tom Ebergen on 06/12/2022.
+//
+
+#include "duckdb/optimizer/operator_pool.hpp"
+
+namespace duckdb {
+
+void OperatorPool::AddOperator(LogicalOperator *op) {
+	D_ASSERT(!InPool(op));
+	seen_operators.insert((idx_t)op);
+}
+
+bool OperatorPool::InPool(LogicalOperator *op) {
+	auto it = seen_operators.find((idx_t)op);
+	return it != seen_operators.end();
+}
+
+void OperatorPool::EmptyOperatorPool() {
+	seen_operators.clear();
+}
+
+} // namespace duckdb

src/optimizer/optimizer.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/optimizer/optimizer.hpp"
 
+#include "duckdb/execution/column_binding_resolver.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/config.hpp"
@@ -12,7 +13,7 @@
 #include "duckdb/optimizer/filter_pullup.hpp"
 #include "duckdb/optimizer/filter_pushdown.hpp"
 #include "duckdb/optimizer/in_clause_rewriter.hpp"
-#include "duckdb/optimizer/join_order_optimizer.hpp"
+#include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
 #include "duckdb/optimizer/regex_range_filter.hpp"
 #include "duckdb/optimizer/remove_unused_columns.hpp"
 #include "duckdb/optimizer/rule/equal_or_null_simplification.hpp"
@@ -22,11 +23,11 @@
 #include "duckdb/optimizer/topn_optimizer.hpp"
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/planner.hpp"
-#include "duckdb/execution/column_binding_resolver.hpp"
 
 namespace duckdb {
 
-Optimizer::Optimizer(Binder &binder, ClientContext &context) : context(context), binder(binder), rewriter(context) {
+Optimizer::Optimizer(Binder &binder, ClientContext &context)
+    : context(context), binder(binder), rewriter(context), seen_operators() {
 	rewriter.rules.push_back(make_unique<ConstantFoldingRule>(rewriter));
 	rewriter.rules.push_back(make_unique<DistributivityRule>(rewriter));
 	rewriter.rules.push_back(make_unique<ArithmeticSimplificationRule>(rewriter));
@@ -60,6 +61,7 @@ void Optimizer::RunOptimizer(OptimizerType type, const std::function<void()> &ca
 	profiler.StartPhase(OptimizerTypeToString(type));
 	callback();
 	profiler.EndPhase();
+	seen_operators.EmptyOperatorPool();
 	if (plan) {
 		Verify(*plan);
 	}

src/optimizer/pushdown/pushdown_left_join.cpp

@@ -8,6 +8,8 @@
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_filter.hpp"
 
+#include "iostream"
+
 namespace duckdb {
 
 using Filter = FilterPushdown::Filter;
@@ -120,8 +122,11 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
 		}
 	});
 	right_pushdown.GenerateFilters();
+
 	op->children[0] = left_pushdown.Rewrite(move(op->children[0]));
 	op->children[1] = right_pushdown.Rewrite(move(op->children[1]));
+	// Need to add the operators here because they are optimized with different
+	// filter_pushdown objects (i.e. left_pushdown and right_pushdown)
 	return FinishPushdown(move(op));
 }