From 243fc2c486b59cad2ce5d4fd94d490a9736be71a Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Fri, 15 Jan 2016 15:04:53 +0900 Subject: [PATCH 01/17] initial commit --- .../org/apache/tajo/algebra/CreateTable.java | 9 +-- .../tajo/catalog/AbstractCatalogClient.java | 10 +-- .../tajo/catalog/store/HiveCatalogStore.java | 21 ++---- .../apache/tajo/catalog/CatalogServer.java | 37 +++------- .../InfoSchemaMetadataDictionary.java | 9 +-- .../store/XMLCatalogSchemaManager.java | 9 +-- .../tajo/cli/tsql/CliClientParamsFactory.java | 8 +-- .../tsql/commands/DescFunctionCommand.java | 8 +-- .../org/apache/tajo/QueryTestCaseBase.java | 5 +- .../org/apache/tajo/TajoTestingCluster.java | 4 +- .../apache/tajo/client/TestTajoClient.java | 6 +- .../engine/planner/TestLogicalPlanner.java | 50 ++++++------- .../planner/physical/TestPhysicalPlanner.java | 8 +-- .../apache/tajo/master/TestRepartitioner.java | 4 +- .../tajo/engine/planner/enforce/Enforcer.java | 4 +- .../planner/global/ExecutionBlockCursor.java | 13 ++-- .../engine/planner/global/GlobalPlanner.java | 10 +-- .../engine/planner/global/MasterPlan.java | 6 +- .../global/ParallelExecutionQueue.java | 8 +-- .../builder/DistinctGroupbyBuilder.java | 32 ++------- .../rewriter/rules/BroadcastJoinRule.java | 14 ++-- .../rewriter/rules/GlobalPlanRewriteUtil.java | 4 +- .../planner/physical/BSTIndexScanExec.java | 8 +-- .../DistinctGroupbyFirstAggregationExec.java | 4 +- .../DistinctGroupbyHashAggregationExec.java | 8 +-- .../physical/HashShuffleFileWriteExec.java | 4 +- .../engine/planner/physical/SeqScanExec.java | 6 +- .../tajo/engine/query/TaskRequestImpl.java | 5 +- .../apache/tajo/engine/utils/TableCache.java | 7 +- .../org/apache/tajo/master/QueryManager.java | 4 +- .../ExplainGlobalPlanPreprocessorForTest.java | 12 ++-- .../NonForwardQueryResultSystemScanner.java | 11 ++- .../tajo/master/exec/QueryExecutor.java | 4 +- .../apache/tajo/parser/sql/SQLAnalyzer.java | 11 +-- .../querymaster/DefaultTaskScheduler.java | 12 ++-- .../org/apache/tajo/querymaster/Query.java | 18 ++--- .../apache/tajo/querymaster/QueryMaster.java | 22 +++--- .../tajo/querymaster/Repartitioner.java | 14 ++-- .../org/apache/tajo/querymaster/Stage.java | 6 +- .../org/apache/tajo/querymaster/Task.java | 21 ++---- .../tajo/util/history/HistoryWriter.java | 9 +-- .../tajo/util/history/QueryHistory.java | 5 +- .../util/metrics/RegexpMetricsFilter.java | 5 +- .../tajo/util/metrics/TajoSystemMetrics.java | 4 +- .../tajo/webapp/QueryExecutorServlet.java | 14 ++-- .../tajo/worker/ExecutionBlockContext.java | 18 +++-- .../org/apache/tajo/worker/TaskExecutor.java | 4 +- .../java/org/apache/tajo/worker/TaskImpl.java | 23 +++--- .../org/apache/tajo/worker/TaskManager.java | 18 ++--- .../tajo/ws/rs/resources/ClusterResource.java | 9 +-- .../ws/rs/resources/FunctionsResource.java | 5 +- .../tajo/ws/rs/resources/QueryResource.java | 9 +-- .../apache/tajo/plan/LogicalOptimizer.java | 11 +-- .../org/apache/tajo/plan/LogicalPlan.java | 11 +-- .../apache/tajo/plan/NamedExprsManager.java | 5 +- .../GreedyHeuristicJoinOrderAlgorithm.java | 6 +- .../tajo/plan/joinorder/JoinGraphContext.java | 33 +++------ .../tajo/plan/joinorder/JoinOrderingUtil.java | 10 +-- .../plan/logical/DistinctGroupbyNode.java | 12 ++-- .../tajo/plan/nameresolver/NameResolver.java | 28 +++----- .../rules/CommonConditionReduceRule.java | 7 +- .../rewrite/rules/FilterPushDownRule.java | 71 +++++-------------- .../rewrite/rules/ProjectionPushDownRule.java | 24 ++----- .../plan/verifier/PreLogicalPlanVerifier.java | 15 ++-- .../tajo/storage/OldStorageManager.java | 4 +- .../tajo/storage/hbase/HBaseTablespace.java | 4 +- .../storage/HashShuffleAppenderWrapper.java | 4 +- 67 files changed, 263 insertions(+), 561 deletions(-) diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java index 6131b8d8fd..4681250d5a 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java @@ -30,6 +30,7 @@ import java.lang.reflect.Type; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class CreateTable extends Expr { @Expose @SerializedName("IsExternal") @@ -419,9 +420,7 @@ public Object clone() throws CloneNotSupportedException { hash.quantity = quantity; if (specifiers != null) { hash.specifiers = new ArrayList<>(); - for (PartitionSpecifier specifier : specifiers) { - hash.specifiers.add(specifier); - } + hash.specifiers.addAll(specifiers.stream().collect(Collectors.toList())); } return hash; } @@ -472,9 +471,7 @@ public Object clone() throws CloneNotSupportedException { } if (specifiers != null) { listPartition.specifiers = new ArrayList<>(); - for (ListPartitionSpecifier specifier : specifiers) { - listPartition.specifiers.add(specifier); - } + listPartition.specifiers.addAll(specifiers.stream().collect(Collectors.toList())); } return listPartition; } diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 2c4d41e72b..10f59b60f4 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; import static org.apache.tajo.catalog.CatalogUtil.buildTableIdentifier; import static org.apache.tajo.error.Errors.ResultCode.*; @@ -522,9 +523,7 @@ public void addPartitions(String databaseName, String tableName, List getAllIndexesByTable(final String databaseNam final IndexListResponse response = stub.getAllIndexesByTable(null, proto); ensureOk(response.getState()); - List indexDescs = new ArrayList<>(); - for (IndexDescProto descProto : response.getIndexDescList()) { - indexDescs.add(new IndexDesc(descProto)); - } + List indexDescs = response.getIndexDescList().stream().map(IndexDesc::new).collect(Collectors.toList()); return indexDescs; } catch (ServiceException e) { throw new RuntimeException(e); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index b855c77733..f7dcb94ae2 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -65,6 +65,7 @@ import java.io.File; import java.util.*; +import java.util.stream.Collectors; public class HiveCatalogStore extends CatalogConstants implements CatalogStore { protected final Log LOG = LogFactory.getLog(getClass()); @@ -689,11 +690,9 @@ private void renameColumn(String databaseName, String tableName, CatalogProtos.A Table table = client.getHiveClient().getTable(databaseName, tableName); List columns = table.getSd().getCols(); - for (final FieldSchema currentColumn : columns) { - if (currentColumn.getName().equalsIgnoreCase(alterColumnProto.getOldColumnName())) { - currentColumn.setName(alterColumnProto.getNewColumnName()); - } - } + columns.stream().filter(currentColumn -> currentColumn.getName().equalsIgnoreCase(alterColumnProto.getOldColumnName())).forEach(currentColumn -> { + currentColumn.setName(alterColumnProto.getNewColumnName()); + }); client.getHiveClient().alter_table(databaseName, tableName, table); } catch (NoSuchObjectException nsoe) { @@ -745,9 +744,7 @@ private void addPartition(String databaseName, String tableName, CatalogProtos.P partition.setParameters(params); List values = Lists.newArrayList(); - for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { - values.add(keyProto.getPartitionValue()); - } + values.addAll(partitionDescProto.getPartitionKeysList().stream().map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); partition.setValues(values); Table table = client.getHiveClient().getTable(databaseName, tableName); @@ -773,9 +770,7 @@ private void dropPartition(String databaseName, String tableName, CatalogProtos. client = clientPool.getClient(); List values = Lists.newArrayList(); - for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { - values.add(keyProto.getPartitionValue()); - } + values.addAll(partitionDescProto.getPartitionKeysList().stream().map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); client.getHiveClient().dropPartition(databaseName, tableName, values, true); } catch (Exception e) { throw new TajoInternalError(e); @@ -1250,9 +1245,7 @@ public void addPartitions(String databaseName, String tableName, List values = Lists.newArrayList(); - for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { - values.add(keyProto.getPartitionValue()); - } + values.addAll(partitionDescProto.getPartitionKeysList().stream().map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); partition.setValues(values); Table table = client.getHiveClient().getTable(databaseName, tableName); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 6583d4eb01..50d31bcf51 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -63,6 +63,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; import static org.apache.tajo.exception.ExceptionUtil.printStackTraceIfError; @@ -1032,9 +1033,7 @@ public GetPartitionsResponse getPartitionsByTableName(RpcController controller, List partitions = store.getPartitionsOfTable(dbName, tbName); GetPartitionsResponse.Builder builder = GetPartitionsResponse.newBuilder(); - for (PartitionDescProto partition : partitions) { - builder.addPartition(partition); - } + partitions.forEach(builder::addPartition); builder.setState(OK); return builder.build(); @@ -1414,12 +1413,8 @@ private FunctionDescProto findFunction(String signature, List candidates = Lists.newArrayList(); if (functions.containsKey(signature)) { - for (FunctionDescProto func : functions.get(signature)) { - if (func.getSignature().getParameterTypesList() != null && - func.getSignature().getParameterTypesList().equals(params)) { - candidates.add(func); - } - } + candidates.addAll(functions.get(signature).stream().filter(func -> func.getSignature().getParameterTypesList() != null && + func.getSignature().getParameterTypesList().equals(params)).collect(Collectors.toList())); } /* @@ -1432,12 +1427,8 @@ private FunctionDescProto findFunction(String signature, List func.getSignature().getParameterTypesList() != null && + CatalogUtil.isMatchedFunction(func.getSignature().getParameterTypesList(), params)).collect(Collectors.toList())); // if there are more than one function candidates, we choose the nearest matched function. if (candidates.size() > 0) { @@ -1456,19 +1447,11 @@ private FunctionDescProto findFunction(String signature, FunctionType type, List if (functions.containsKey(signature)) { if (strictTypeCheck) { - for (FunctionDescProto func : functions.get(signature)) { - if (func.getSignature().getType() == type && - func.getSignature().getParameterTypesList().equals(params)) { - candidates.add(func); - } - } + candidates.addAll(functions.get(signature).stream().filter(func -> func.getSignature().getType() == type && + func.getSignature().getParameterTypesList().equals(params)).collect(Collectors.toList())); } else { - for (FunctionDescProto func : functions.get(signature)) { - if (func.getSignature().getParameterTypesList() != null && - CatalogUtil.isMatchedFunction(func.getSignature().getParameterTypesList(), params)) { - candidates.add(func); - } - } + candidates.addAll(functions.get(signature).stream().filter(func -> func.getSignature().getParameterTypesList() != null && + CatalogUtil.isMatchedFunction(func.getSignature().getParameterTypesList(), params)).collect(Collectors.toList())); } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java index d9008aa1b7..86aebf4d17 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import org.apache.tajo.exception.UndefinedTableException; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -78,12 +79,8 @@ public String getSystemDatabaseName() { } public List getAllSystemTables() { - List systemTableNames = new ArrayList<>(); - - for (TableDescriptor descriptor: schemaInfoTableDescriptors) { - systemTableNames.add(descriptor.getTableNameString()); - } - + List systemTableNames = schemaInfoTableDescriptors.stream().map(TableDescriptor::getTableNameString).collect(Collectors.toList()); + return systemTableNames; } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index ea15c079ea..12814b2805 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -43,6 +43,7 @@ import java.util.*; import java.util.jar.JarEntry; import java.util.jar.JarFile; +import java.util.stream.Collectors; public class XMLCatalogSchemaManager { protected final Log LOG = LogFactory.getLog(getClass()); @@ -304,12 +305,8 @@ public void upgradeBaseSchema(Connection conn, int currentVersion) { final List candidatePatches = new ArrayList<>(); Statement stmt; - - for (SchemaPatch patch: this.catalogStore.getPatches()) { - if (currentVersion >= patch.getPriorVersion()) { - candidatePatches.add(patch); - } - } + + candidatePatches.addAll(this.catalogStore.getPatches().stream().filter(patch -> currentVersion >= patch.getPriorVersion()).collect(Collectors.toList())); Collections.sort(candidatePatches); try { diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java index a4b0c1d640..c6b997fc0c 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java @@ -41,11 +41,9 @@ class CliClientParamsFactory { public static Properties get(@Nullable Properties connParam) { Properties copy = connParam == null ? new Properties() : (Properties) connParam.clone(); - for (Map.Entry entry : DEFAULT_PARAMS.entrySet()) { - if (!copy.contains(entry.getKey())) { - copy.setProperty(entry.getKey(), entry.getValue()); - } - } + DEFAULT_PARAMS.entrySet().stream().filter(entry -> !copy.contains(entry.getKey())).forEach(entry -> { + copy.setProperty(entry.getKey(), entry.getValue()); + }); return copy; } } diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java index 944ad0fb5f..c62999f439 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java @@ -92,11 +92,9 @@ public void invoke(String[] cmd) throws Exception { Map functionMap = new HashMap<>(); - for (CatalogProtos.FunctionDescProto eachFunction: functions) { - if (!functionMap.containsKey(eachFunction.getSupplement().getShortDescription())) { - functionMap.put(eachFunction.getSupplement().getShortDescription(), eachFunction); - } - } + functions.stream().filter(eachFunction -> !functionMap.containsKey(eachFunction.getSupplement().getShortDescription())).forEach(eachFunction -> { + functionMap.put(eachFunction.getSupplement().getShortDescription(), eachFunction); + }); for (CatalogProtos.FunctionDescProto eachFunction: functionMap.values()) { String signature = eachFunction.getSignature().getReturnType().getType() + " " + diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 315f954e0a..23e61073b3 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -73,6 +73,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.*; +import java.util.stream.Collectors; import static org.junit.Assert.*; @@ -518,9 +519,7 @@ protected Collection getBatchQueries(Collection paths) throws IOEx List queries = Lists.newArrayList(); for (Path p : paths) { - for (ParsedResult statement: SimpleParser.parseScript(FileUtil.readTextFile(new File(p.toUri())))) { - queries.add(statement.getStatement()); - } + queries.addAll(SimpleParser.parseScript(FileUtil.readTextFile(new File(p.toUri()))).stream().map(ParsedResult::getStatement).collect(Collectors.toList())); } return queries; diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 4b89c8e274..d43c84470c 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -471,9 +471,7 @@ public void shutdownMiniTajoCluster() { if(this.tajoMaster != null) { this.tajoMaster.stop(); } - for(TajoWorker eachWorker: tajoWorkers) { - eachWorker.stopWorkerForce(); - } + tajoWorkers.forEach(TajoWorker::stopWorkerForce); tajoWorkers.clear(); this.tajoMaster= null; } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java index 828f60c7b7..d5f6e136d1 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -57,6 +57,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.*; +import java.util.stream.Collectors; import static org.junit.Assert.*; @@ -175,10 +176,7 @@ public final void testSessionVariables() throws IOException, TajoException, Inte String prefixName = "key_"; String prefixValue = "val_"; - List unsetList = new ArrayList<>(); - for(Map.Entry entry: client.getAllSessionVariables().entrySet()) { - unsetList.add(entry.getKey()); - } + List unsetList = client.getAllSessionVariables().entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList()); client.unsetSessionVariables(unsetList); for (int i = 0; i < 10; i++) { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 142b2c3836..d8b5a8af59 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -526,12 +526,10 @@ public final void testJoinWithMultipleJoinQual1() throws IOException, TajoExcept } } - for (Map.Entry entry : qualMap.entrySet()) { - if (!entry.getValue()) { - Preconditions.checkArgument(false, - "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); - } - } + qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + Preconditions.checkArgument(false, + "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); + }); } @Test @@ -566,12 +564,10 @@ public final void testJoinWithMultipleJoinQual2() throws IOException, TajoExcept } } - for (Map.Entry entry : qualMap.entrySet()) { - if (!entry.getValue()) { - Preconditions.checkArgument(false, - "SelectionQual not found. -> required JoinQual:" + entry.getKey().toJson()); - } - } + qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + Preconditions.checkArgument(false, + "SelectionQual not found. -> required JoinQual:" + entry.getKey().toJson()); + }); } @Test @@ -611,12 +607,10 @@ public final void testJoinWithMultipleJoinQual3() throws IOException, TajoExcept } } - for (Map.Entry entry : qualMap.entrySet()) { - if (!entry.getValue()) { - Preconditions.checkArgument(false, - "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); - } - } + qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + Preconditions.checkArgument(false, + "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); + }); } @@ -680,19 +674,15 @@ public final void testJoinWithMultipleJoinQual4() throws IOException, TajoExcept } - for (Map.Entry entry : joinQualMap.entrySet()) { - if (!entry.getValue()) { - Preconditions.checkArgument(false, - "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); - } - } + joinQualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + Preconditions.checkArgument(false, + "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); + }); - for (Map.Entry entry : scanMap.entrySet()) { - if (!entry.getValue()) { - Preconditions.checkArgument(false, - "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); - } - } + scanMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + Preconditions.checkArgument(false, + "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); + }); } static void testQuery7(LogicalNode plan) { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 14f3e9803a..c4fb292ecd 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -844,9 +844,7 @@ public final void testAggregationFunction() throws IOException, TajoException { // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); - for (AggregationFunctionCallEval function : groupbyNode.getAggFunctions()) { - function.setFirstPhase(); - } + groupbyNode.getAggFunctions().forEach(AggregationFunctionCallEval::setFirstPhase); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); @@ -875,9 +873,7 @@ public final void testCountFunction() throws IOException, TajoException { // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); - for (AggregationFunctionCallEval function : groupbyNode.getAggFunctions()) { - function.setFirstPhase(); - } + groupbyNode.getAggFunctions().forEach(AggregationFunctionCallEval::setFirstPhase); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestRepartitioner.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestRepartitioner.java index abec6a0443..a4d2d7cd61 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestRepartitioner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestRepartitioner.java @@ -505,9 +505,7 @@ private static void assertFetchProto(FetchProto [] expected, Map resultURLs = Sets.newHashSet(); for (Map> e : result) { - for (List list : e.values()) { - resultURLs.addAll(list); - } + e.values().forEach(resultURLs::addAll); } assertEquals(expectedURLs.size(), resultURLs.size()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java index 66a1791945..33b386b21f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java @@ -214,9 +214,7 @@ public List getProperties() { return proto.getPropertiesList(); } else { List list = new ArrayList<>(); - for (List propertyList : properties.values()) { - list.addAll(propertyList); - } + properties.values().forEach(list::addAll); return list; } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java index 10f6e54722..cff6cf4bb7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java @@ -19,6 +19,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; /** * A distributed execution plan (DEP) is a direct acyclic graph (DAG) of ExecutionBlocks. @@ -103,9 +104,7 @@ public String toString() { // Add all execution blocks in a depth first and postfix order private void buildDepthFirstOrder(ExecutionBlock current) { if (!masterPlan.isLeaf(current.getId())) { - for (ExecutionBlock execBlock : masterPlan.getChilds(current)) { - buildDepthFirstOrder(execBlock); - } + masterPlan.getChilds(current).forEach(this::buildDepthFirstOrder); } orderedBlocks.add(current); } @@ -136,9 +135,7 @@ In the case of the upper plan, buildDepthFirstOrder() makes the following order orderRequiredChildCountMap.get(eachItem.parentEB.getId()).decrementAndGet(); } else { if (eachItem.allSiblingsOrdered()) { - for (BuildOrderItem eachSiblingItem: notOrderedSiblingBlocks) { - orderedBlocks.add(eachSiblingItem.eb); - } + orderedBlocks.addAll(notOrderedSiblingBlocks.stream().map(eachSiblingItem -> eachSiblingItem.eb).collect(Collectors.toList())); orderedBlocks.add(eachItem.eb); notOrderedSiblingBlocks.clear(); } else { @@ -162,9 +159,7 @@ private void preExecutionOrder(BuildOrderItem current) { stack.push(item); } } - for (BuildOrderItem eachItem : stack) { - preExecutionOrder(eachItem); - } + stack.forEach(this::preExecutionOrder); } executionOrderedBlocks.add(current); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index fd34c46f7d..1ed89c089d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -56,6 +56,7 @@ import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; import static org.apache.tajo.conf.TajoConf.ConfVars; import static org.apache.tajo.conf.TajoConf.ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS; @@ -547,13 +548,8 @@ private ExecutionBlock buildGroupByIncludingDistinctFunctionsMultiStage(GlobalPl } } - List firstStageTargets = new ArrayList<>(); - for (Column column : firstStageGroupingColumns) { - firstStageTargets.add(new Target(new FieldEval(column))); - } - for (Target target : firstPhaseEvalNodeTargets) { - firstStageTargets.add(target); - } + List firstStageTargets = firstStageGroupingColumns.stream().map(column -> new Target(new FieldEval(column))).collect(Collectors.toList()); + firstStageTargets.addAll(firstPhaseEvalNodeTargets.stream().collect(Collectors.toList())); // Create the groupby node for the first stage and set all necessary descriptions GroupbyNode firstStageGroupby = new GroupbyNode(context.plan.getLogicalPlan().newPID()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java index a7b03e73fb..670042b082 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java @@ -36,6 +36,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; public class MasterPlan { private final QueryId queryId; @@ -236,10 +237,7 @@ public List getChilds(ExecutionBlock execBlock) { } public List getChilds(ExecutionBlockId id) { - List childBlocks = new ArrayList<>(); - for (ExecutionBlockId cid : execBlockGraph.getChilds(id)) { - childBlocks.add(execBlockMap.get(cid)); - } + List childBlocks = execBlockGraph.getChilds(id).stream().map(cid -> execBlockMap.get(cid)).collect(Collectors.toList()); return childBlocks; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java index d197bda7ef..efb6f47f5d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java @@ -76,11 +76,9 @@ public synchronized int size() { public synchronized ExecutionBlock[] first() { int max = Math.min(maximum, executable.size()); List result = new ArrayList<>(); - for (Deque queue : executable) { - if (result.size() < max && isExecutableNow(queue.peekLast())) { - result.add(queue.removeLast()); - } - } + executable.stream().filter(queue -> result.size() < max && isExecutableNow(queue.peekLast())).forEach(queue -> { + result.add(queue.removeLast()); + }); LOG.info("Initial executable blocks " + result); return result.toArray(new ExecutionBlock[result.size()]); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index 720c337eb4..4baf58d25f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -51,6 +51,7 @@ import org.apache.tajo.util.TUtil; import java.util.*; +import java.util.stream.Collectors; import static org.apache.tajo.plan.serder.PlanProto.ShuffleType.HASH_SHUFFLE; @@ -226,11 +227,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou // Grouping columns are GROUP BY clause's column + Distinct column. List groupingColumns = new ArrayList<>(); - for (Column eachGroupingColumn: groupbyUniqColumns) { - if (!groupingColumns.contains(eachGroupingColumn)) { - groupingColumns.add(eachGroupingColumn); - } - } + groupbyUniqColumns.stream().filter(eachGroupingColumn -> !groupingColumns.contains(eachGroupingColumn)).forEach(groupingColumns::add); distinctGroupbyNode.setGroupingColumns(groupingColumns.toArray(new Column[groupingColumns.size()])); } buildInfo.addAggFunction(aggFunction); @@ -243,9 +240,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou List baseGroupByTargets = new ArrayList<>(); baseGroupByTargets.add(new Target(new FieldEval(new Column("?distinctseq", Type.INT2)))); - for (Column column : originalGroupingColumns) { - baseGroupByTargets.add(new Target(new FieldEval(column))); - } + baseGroupByTargets.addAll(originalGroupingColumns.stream().map(column -> new Target(new FieldEval(column))).collect(Collectors.toList())); //Add child groupby node for each Distinct clause for (DistinctGroupbyNodeBuildInfo buildInfo: distinctNodeBuildInfos.values()) { @@ -391,11 +386,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou // Grouping columns are GROUP BY clause's column + Distinct column. List groupingColumns = new ArrayList<>(originalGroupingColumns); - for (Column eachGroupingColumn: groupbyUniqColumns) { - if (!groupingColumns.contains(eachGroupingColumn)) { - groupingColumns.add(eachGroupingColumn); - } - } + groupbyUniqColumns.stream().filter(eachGroupingColumn -> !groupingColumns.contains(eachGroupingColumn)).forEach(groupingColumns::add); distinctGroupbyNode.setGroupingColumns(groupingColumns.toArray(new Column[groupingColumns.size()])); } buildInfo.addAggFunction(aggFunction); @@ -431,10 +422,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou // finally this aggregation output tuple's order is GROUP_BY_COL1, COL2, .... + AGG_VALUE, SUM_VALUE, ... GroupbyNode otherGroupbyNode = new GroupbyNode(context.getPlan().getLogicalPlan().newPID()); - List targets = new ArrayList<>(); - for (Column column : originalGroupingColumns) { - targets.add(new Target(new FieldEval(column))); - } + List targets = originalGroupingColumns.stream().map(column -> new Target(new FieldEval(column))).collect(Collectors.toList()); targets.addAll(otherAggregationFunctionTargets); otherGroupbyNode.setTargets(targets); @@ -532,11 +520,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou LinkedHashSet distinctColumns = EvalTreeUtil.findUniqueColumns(secondStageGroupbyNode.getAggFunctions().get(0)); List uniqueDistinctColumn = new ArrayList<>(); // remove origin group by column from distinctColumns - for (Column eachColumn: distinctColumns) { - if (!originGroupColumns.contains(eachColumn)) { - uniqueDistinctColumn.add(eachColumn); - } - } + uniqueDistinctColumn.addAll(distinctColumns.stream().filter(eachColumn -> !originGroupColumns.contains(eachColumn)).collect(Collectors.toList())); for (int i = 0; i < originGroupColumns.size(); i++) { secondGroupbyTargets.add(oldTargets.get(i)); if (grpIdx > 0) { @@ -652,9 +636,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou int index = 0; for(GroupbyNode eachNode: secondStageDistinctNode.getSubPlans()) { eachNode.setInSchema(firstStageDistinctNode.getOutSchema()); - for (Column column: eachNode.getOutSchema().getRootColumns()) { - secondStageInSchema.add(column); - } + eachNode.getOutSchema().getRootColumns().forEach(secondStageInSchema::add); } secondStageDistinctNode.setInSchema(secondStageInSchema.build()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java index d390740187..4da6701699 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java @@ -224,12 +224,10 @@ private void visitNonLeafNode(Context context, ExecutionBlock current) throws Ta } JoinType joinType = ((JoinNode)found).getJoinType(); - for (ExecutionBlock child : childs) { - if (!child.isPreservedRow()) { - updateBroadcastableRelForChildEb(context, child, joinType); - updateInputBasedOnChildEb(child, current); - } - } + childs.stream().filter(child -> !child.isPreservedRow()).forEach(child -> { + updateBroadcastableRelForChildEb(context, child, joinType); + updateInputBasedOnChildEb(child, current); + }); if (current.hasBroadcastRelation()) { // The current execution block and its every child are able to be merged. @@ -245,9 +243,7 @@ private void visitNonLeafNode(Context context, ExecutionBlock current) throws Ta } } else { List relations = new ArrayList<>(current.getBroadcastRelations()); - for (ScanNode eachRelation : relations) { - current.removeBroadcastRelation(eachRelation); - } + relations.forEach(current::removeBroadcastRelation); } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java index b13cb0f1a8..566d417f64 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java @@ -36,9 +36,7 @@ public class GlobalPlanRewriteUtil { * @return */ public static ExecutionBlock mergeExecutionBlocks(MasterPlan plan, ExecutionBlock child, ExecutionBlock parent) { - for (ScanNode broadcastable : child.getBroadcastRelations()) { - parent.addBroadcastRelation(broadcastable); - } + child.getBroadcastRelations().forEach(parent::addBroadcastRelation); // connect parent and grand children List grandChilds = plan.getChilds(child); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index 7ab0943915..bb9da9695f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@ -99,11 +99,9 @@ private static Schema mergeSubSchemas(Schema originalSchema, Schema subSchema, L } SchemaBuilder mergedSchema = SchemaBuilder.builder(); - for (Column column : originalSchema.getRootColumns()) { - if (subSchema.contains(column) || qualAndTargets.contains(column)) { - mergedSchema.add(column); - } - } + originalSchema.getRootColumns().stream() + .filter(column -> subSchema.contains(column) || qualAndTargets.contains(column)) + .forEach(mergedSchema::add); return mergedSchema.build(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java index 18c6bc969d..a0bf345c9c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java @@ -363,9 +363,7 @@ public void rescan() { } public void close() throws IOException { - for (TupleSet set : distinctAggrDatas.values()) { - set.clear(); - } + distinctAggrDatas.values().forEach(TupleSet::clear); distinctAggrDatas.clear(); distinctAggrDatas = null; currentGroupingTuples = null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java index 6607416e09..05b58a8476 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java @@ -261,9 +261,7 @@ public Tuple next() throws IOException { listIndex++; } - for (TupleList eachList : tupleSlots) { - eachList.clear(); - } + tupleSlots.forEach(TupleList::clear); tupleSlots.clear(); currentAggregatedTupleIndex = 0; @@ -440,9 +438,7 @@ public TupleList aggregate(Map groupTuples) { } public void close() throws IOException { - for (TupleMap map : hashTable.values()) { - map.clear(); - } + hashTable.values().forEach(TupleMap::clear); hashTable.clear(); hashTable = null; iterator = null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java index 0da1aa6e05..3cd9671ed9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java @@ -234,9 +234,7 @@ public void rescan() throws IOException { @Override public void close() throws IOException{ if (partitionMemoryMap.size() > 0) { - for (RowBlock rowBlock : partitionMemoryMap.values()) { - rowBlock.release(); - } + partitionMemoryMap.values().forEach(RowBlock::release); partitionMemoryMap.clear(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index 52cb080edd..48d45f693d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -153,11 +153,7 @@ public static Schema getProjectSchema(Schema inSchema, Schema outSchema, columnSet.addAll(EvalTreeUtil.findUniqueColumns(t.getEvalTree())); } - for (Column column : inSchema.getAllColumns()) { - if (columnSet.contains(column)) { - projected.add(column); - } - } + inSchema.getAllColumns().stream().filter(columnSet::contains).forEach(projected::add); return projected.build(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java index 65c23f42ba..5f6f6d0bfd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; @@ -222,9 +223,7 @@ private void initFetches() { } TaskRequestProtoOrBuilder p = viaProto ? proto : builder; this.fetches = new ArrayList<>(); - for(FetchProto fetch : p.getFetchesList()) { - fetches.add(fetch); - } + fetches.addAll(p.getFetchesList().stream().collect(Collectors.toList())); } private void maybeInitBuilder() { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java index f2a2217445..ba7827e6d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * This is a simple TableCache which just added CacheHolder as needed. @@ -61,11 +62,7 @@ public synchronized void releaseCache(ExecutionBlockId ebId) { public synchronized List getCacheKeyByExecutionBlockId(ExecutionBlockId ebId) { List keys = Lists.newArrayList(); - for (TableCacheKey eachKey : cacheMap.keySet()) { - if (eachKey.ebId.equals(ebId.toString())) { - keys.add(eachKey); - } - } + keys.addAll(cacheMap.keySet().stream().filter(eachKey -> eachKey.ebId.equals(ebId.toString())).collect(Collectors.toList())); return keys; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java index b4f1d66ac7..8f80b825dd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java @@ -95,9 +95,7 @@ public void serviceInit(Configuration conf) throws Exception { @Override public void serviceStop() throws Exception { - for(QueryInProgress eachQueryInProgress: runningQueries.values()) { - eachQueryInProgress.stopProgress(); - } + runningQueries.values().forEach(QueryInProgress::stopProgress); super.serviceStop(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java index 78cd0159c4..1ec99f5a1c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java @@ -47,13 +47,11 @@ public void prepareTest(MasterPlan plan) { for (ExecutionBlock block : cursor) { List outgoingChannels = plan.getOutgoingChannels(block.getId()); if (outgoingChannels != null) { - for (DataChannel channel : outgoingChannels) { - if (channel.hasShuffleKeys()) { - Column[] shuffleKeys = channel.getShuffleKeys(); - Arrays.sort(shuffleKeys, columnComparator); - channel.setShuffleKeys(shuffleKeys); - } - } + outgoingChannels.stream().filter(channel -> channel.hasShuffleKeys()).forEach(channel -> { + Column[] shuffleKeys = channel.getShuffleKeys(); + Arrays.sort(shuffleKeys, columnComparator); + channel.setShuffleKeys(shuffleKeys); + }); } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index ac917ef006..50736be75e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -70,6 +70,7 @@ import java.util.List; import java.util.Map; import java.util.Stack; +import java.util.stream.Collectors; public class NonForwardQueryResultSystemScanner implements NonForwardQueryResultScanner { @@ -527,13 +528,9 @@ private List getClusterInfo(Schema outSchema) { } tuples = new ArrayList<>(queryMasterList.size() + nodeStatusList.size()); - for (NodeStatus queryMaster: queryMasterList) { - tuples.add(getQueryMasterTuple(outSchema, queryMaster)); - } - - for (NodeStatus nodeStatus : nodeStatusList) { - tuples.add(getWorkerTuple(outSchema, nodeStatus)); - } + tuples.addAll(queryMasterList.stream().map(queryMaster -> getQueryMasterTuple(outSchema, queryMaster)).collect(Collectors.toList())); + + tuples.addAll(nodeStatusList.stream().map(nodeStatus -> getWorkerTuple(outSchema, nodeStatus)).collect(Collectors.toList())); return tuples; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index 58f4ec013f..d7a08aec27 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -402,9 +402,7 @@ public static void startScriptExecutors(QueryContext queryContext, EvalContext e } public static void stopScriptExecutors(EvalContext evalContext) { - for (TajoScriptEngine executor : evalContext.getAllScriptEngines()) { - executor.shutdown(); - } + evalContext.getAllScriptEngines().forEach(TajoScriptEngine::shutdown); } /** diff --git a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java index 6ffe65b059..1a20028c68 100644 --- a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java @@ -1351,11 +1351,7 @@ public Expr visitCreate_table_statement(Create_table_statementContext ctx) { @Override public Expr visitTruncate_table_statement(@NotNull Truncate_table_statementContext ctx) { List tableNameContexts = ctx.table_name(); - List tableNames = new ArrayList<>(); - - for (Table_nameContext eachTableNameContext: tableNameContexts) { - tableNames.add(buildIdentifierChain(eachTableNameContext.identifier())); - } + List tableNames = tableNameContexts.stream().map(eachTableNameContext -> buildIdentifierChain(eachTableNameContext.identifier())).collect(Collectors.toList()); return new TruncateTable(tableNames); } @@ -1404,10 +1400,7 @@ public PartitionMethodDescExpr parseTablePartitioningClause(Table_partitioning_c } else { // ( PARTITION part_name , ...) List specifiers = Lists.newArrayList(); - for (Individual_hash_partitionContext partition : - hashPartitions.individual_hash_partitions().individual_hash_partition()) { - specifiers.add(new CreateTable.PartitionSpecifier(partition.partition_name().getText())); - } + specifiers.addAll(hashPartitions.individual_hash_partitions().individual_hash_partition().stream().map(partition -> new PartitionSpecifier(partition.partition_name().getText())).collect(Collectors.toList())); return new HashPartition(buildColumnReferenceList(hashPartitions.column_reference_list()), specifiers); } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index 401ba0a3a6..07fe50b884 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -63,6 +63,8 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.apache.tajo.ResourceProtos.*; @@ -288,11 +290,7 @@ private Set getWorkerIds(Collection hosts){ Set workerIds = Sets.newHashSet(); if(hosts.isEmpty()) return workerIds; - for (WorkerConnectionInfo worker : stage.getContext().getWorkerMap().values()) { - if(hosts.contains(worker.getHost())){ - workerIds.add(worker.getId()); - } - } + workerIds.addAll(stage.getContext().getWorkerMap().values().stream().filter(worker -> hosts.contains(worker.getHost())).map((Function) WorkerConnectionInfo::getId).collect(Collectors.toList())); return workerIds; } @@ -328,9 +326,7 @@ protected LinkedList createTaskRequest(final int incompleteTas masterClientService.reserveNodeResources(callBack.getController(), request.build(), callBack); NodeResourceResponse response = callBack.get(); - for (AllocationResourceProto resource : response.getResourceList()) { - taskRequestEvents.add(new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId())); - } + taskRequestEvents.addAll(response.getResourceList().stream().map(resource -> new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId())).collect(Collectors.toList())); return taskRequestEvents; } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 289d933bd1..0a34b69d61 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -62,6 +62,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; public class Query implements EventHandler { private static final Log LOG = LogFactory.getLog(Query.class); @@ -304,10 +305,7 @@ public QueryHistory getQueryHistory() { } private List makeStageHistories() { - List stageHistories = new ArrayList<>(); - for(Stage eachStage : getStages()) { - stageHistories.add(eachStage.getStageHistory()); - } + List stageHistories = getStages().stream().map(Stage::getStageHistory).collect(Collectors.toList()); return stageHistories; } @@ -322,11 +320,9 @@ private QueryHistory makeQueryHistory() { queryHistory.setDistributedPlan(plan.toString()); List sessionVariables = new ArrayList<>(); - for(Map.Entry entry: plan.getContext().getAllKeyValus().entrySet()) { - if (SessionVars.exists(entry.getKey()) && SessionVars.isPublic(SessionVars.get(entry.getKey()))) { - sessionVariables.add(new String[]{entry.getKey(), entry.getValue()}); - } - } + plan.getContext().getAllKeyValus().entrySet().stream().filter(entry -> SessionVars.exists(entry.getKey()) && SessionVars.isPublic(SessionVars.get(entry.getKey()))).forEach(entry -> { + sessionVariables.add(new String[]{entry.getKey(), entry.getValue()}); + }); queryHistory.setSessionVariables(sessionVariables); return queryHistory; @@ -341,9 +337,7 @@ public List getPartitions() { } public void clearPartitions() { - for(Stage eachStage : getStages()) { - eachStage.clearPartitions(); - } + getStages().forEach(Stage::clearPartitions); } public SerializedException getFailureReason() { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index adc7b089e8..ddefc19ab9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -447,20 +447,18 @@ public void run() { List tempTasks = new ArrayList<>(); tempTasks.addAll(queryMasterTasks.values()); - for(QueryMasterTask eachTask: tempTasks) { - if(!eachTask.isStopped()) { - try { - long lastHeartbeat = eachTask.getLastClientHeartbeat(); - long time = System.currentTimeMillis() - lastHeartbeat; - if(lastHeartbeat > 0 && time > querySessionTimeout * 1000) { - LOG.warn("Query " + eachTask.getQueryId() + " stopped cause query session timeout: " + time + " ms"); - eachTask.expireQuerySession(); - } - } catch (Exception e) { - LOG.error(eachTask.getQueryId() + ":" + e.getMessage(), e); + tempTasks.stream().filter(eachTask -> !eachTask.isStopped()).forEach(eachTask -> { + try { + long lastHeartbeat = eachTask.getLastClientHeartbeat(); + long time = System.currentTimeMillis() - lastHeartbeat; + if (lastHeartbeat > 0 && time > querySessionTimeout * 1000) { + LOG.warn("Query " + eachTask.getQueryId() + " stopped cause query session timeout: " + time + " ms"); + eachTask.expireQuerySession(); } + } catch (Exception e) { + LOG.error(eachTask.getQueryId() + ":" + e.getMessage(), e); } - } + }); } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index c264e3e75f..28b8fab302 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -514,14 +514,12 @@ private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext sch private static void addJoinShuffle(Stage stage, int partitionId, Map> grouppedPartitions) { Map> fetches = new HashMap<>(); - for (ExecutionBlock execBlock : stage.getMasterPlan().getChilds(stage.getId())) { - if (grouppedPartitions.containsKey(execBlock.getId())) { - String name = execBlock.getId().toString(); - List requests = mergeShuffleRequest(name, partitionId, HASH_SHUFFLE, - grouppedPartitions.get(execBlock.getId())); - fetches.put(name, requests); - } - } + stage.getMasterPlan().getChilds(stage.getId()).stream().filter(execBlock -> grouppedPartitions.containsKey(execBlock.getId())).forEach(execBlock -> { + String name = execBlock.getId().toString(); + List requests = mergeShuffleRequest(name, partitionId, HASH_SHUFFLE, + grouppedPartitions.get(execBlock.getId())); + fetches.put(name, requests); + }); if (fetches.isEmpty()) { LOG.info(stage.getId() + "'s " + partitionId + " partition has empty result."); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index c055d119e4..e308524ce6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -78,6 +78,8 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.apache.tajo.ResourceProtos.*; import static org.apache.tajo.conf.TajoConf.ConfVars; @@ -493,9 +495,7 @@ private StageHistory makeStageHistory() { } Set partitions = Sets.newHashSet(); - for (IntermediateEntry entry : getHashShuffleIntermediateEntries()) { - partitions.add(entry.getPartId()); - } + partitions.addAll(getHashShuffleIntermediateEntries().stream().map((Function) IntermediateEntry::getPartId).collect(Collectors.toList())); stageHistory.setTotalInputBytes(totalInputBytes); stageHistory.setTotalReadBytes(totalReadBytes); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java index f1ad931e47..62d7cdde0d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java @@ -56,6 +56,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; import static org.apache.tajo.ResourceProtos.*; import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; @@ -298,10 +299,7 @@ private TaskHistory makeTaskHistory() { taskHistory.setFetchs(fetchList.toArray(new String[][]{})); - List dataLocationList = new ArrayList<>(); - for(DataLocation eachLocation: getDataLocations()) { - dataLocationList.add(eachLocation.toString()); - } + List dataLocationList = getDataLocations().stream().map(DataLocation::toString).collect(Collectors.toList()); taskHistory.setDataLocations(dataLocationList.toArray(new String[dataLocationList.size()])); return taskHistory; @@ -388,9 +386,7 @@ public void setFetches(Map> fetches) { public Collection getAllFragments() { Set fragmentProtos = new HashSet<>(); - for (Set eachFragmentSet : fragMap.values()) { - fragmentProtos.addAll(eachFragmentSet); - } + fragMap.values().forEach(fragmentProtos::addAll); return fragmentProtos; } @@ -786,16 +782,11 @@ public IntermediateEntry(IntermediateEntryProto proto) { this.volume = proto.getVolume(); failureRowNums = new ArrayList<>(); - for (FailureIntermediateProto eachFailure: proto.getFailuresList()) { - - failureRowNums.add(new Pair(eachFailure.getPagePos(), - new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum()))); - } + failureRowNums.addAll(proto.getFailuresList().stream().map(eachFailure -> new Pair(eachFailure.getPagePos(), + new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum()))).collect(Collectors.toList())); pages = new ArrayList<>(); - for (IntermediateEntryProto.PageProto eachPage: proto.getPagesList()) { - pages.add(new Pair(eachPage.getPos(), eachPage.getLength())); - } + pages.addAll(proto.getPagesList().stream().map(eachPage -> new Pair(eachPage.getPos(), eachPage.getLength())).collect(Collectors.toList())); } public IntermediateEntry(int taskId, int attemptId, int partId, PullHost host) { diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java index 668940c4c0..91387451ef 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java @@ -39,6 +39,7 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; /** * History directory structure @@ -268,13 +269,7 @@ public void run() { Calendar cal = Calendar.getInstance(); cal.add(Calendar.HOUR_OF_DAY, -2); String closeTargetTime = df.format(cal.getTime()); - List closingTargets = new ArrayList<>(); - - for (String eachWriterTime : taskWriters.keySet()) { - if (eachWriterTime.compareTo(closeTargetTime) <= 0) { - closingTargets.add(eachWriterTime); - } - } + List closingTargets = taskWriters.keySet().stream().filter(eachWriterTime -> eachWriterTime.compareTo(closeTargetTime) <= 0).collect(Collectors.toList()); for (String eachWriterTime : closingTargets) { WriterHolder writerHolder; diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java index bc65cae53e..4c18d1dc23 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; public class QueryHistory implements GsonObject, History { @Expose @@ -140,9 +141,7 @@ public QueryHistoryProto getProto() { List stageHistoryProtos = new ArrayList<>(); if (stageHistories != null) { - for (StageHistory eachStage: stageHistories) { - stageHistoryProtos.add((eachStage.getProto())); - } + stageHistoryProtos.addAll(stageHistories.stream().map(eachStage -> (eachStage.getProto())).collect(Collectors.toList())); } builder.addAllStageHistories(stageHistoryProtos); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java index ef74a4e83e..3804347893 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java @@ -25,14 +25,13 @@ import java.util.Collection; import java.util.List; import java.util.regex.Pattern; +import java.util.stream.Collectors; public class RegexpMetricsFilter implements MetricFilter { List filterPatterns = new ArrayList<>(); public RegexpMetricsFilter(Collection filterExpressions) { - for(String eachExpression: filterExpressions) { - filterPatterns.add(Pattern.compile(eachExpression)); - } + filterPatterns.addAll(filterExpressions.stream().map(Pattern::compile).collect(Collectors.toList())); } @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java index 260d4147f7..c255f9a176 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java @@ -104,9 +104,7 @@ public void stop() { protected void stopAndClearReporter() { synchronized(metricsReporters) { - for(TajoMetricsScheduledReporter eachReporter: metricsReporters) { - eachReporter.close(); - } + metricsReporters.forEach(TajoMetricsScheduledReporter::close); metricsReporters.clear(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java index 708c552207..2863a1d2e3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java @@ -210,9 +210,7 @@ public void service(HttpServletRequest request, } } else if("clearAllQueryRunner".equals(action)) { synchronized(queryRunners) { - for(QueryRunner eachQueryRunner: queryRunners.values()) { - eachQueryRunner.setStop(); - } + queryRunners.values().forEach(QueryRunner::setStop); queryRunners.clear(); } } else if("killQuery".equals(action)) { @@ -267,12 +265,10 @@ public void run() { List queryRunnerList; synchronized(queryRunners) { queryRunnerList = new ArrayList<>(queryRunners.values()); - for(QueryRunner eachQueryRunner: queryRunnerList) { - if(!eachQueryRunner.running.get() && - (System.currentTimeMillis() - eachQueryRunner.finishTime > 180 * 1000)) { - queryRunners.remove(eachQueryRunner.queryRunnerId); - } - } + queryRunnerList.stream().filter(eachQueryRunner -> !eachQueryRunner.running.get() && + (System.currentTimeMillis() - eachQueryRunner.finishTime > 180 * 1000)).forEach(eachQueryRunner -> { + queryRunners.remove(eachQueryRunner.queryRunnerId); + }); } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java index 4ab6627e8c..3a6f6e1dcf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java @@ -187,17 +187,15 @@ public void stop(){ } // If ExecutionBlock is stopped, all running or pending tasks will be marked as failed. - for (Task task : tasks.values()) { - if (task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_PENDING || - task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_RUNNING) { - - try{ - task.abort(); - } catch (Throwable e){ - LOG.error(e, e); - } + tasks.values().stream().filter(task -> task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_PENDING || + task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_RUNNING).forEach(task -> { + + try { + task.abort(); + } catch (Throwable e) { + LOG.error(e, e); } - } + }); tasks.clear(); taskHistories.clear(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java index d1271f88fc..ff748f5542 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java @@ -102,9 +102,7 @@ protected void serviceStop() throws Exception { isStopped = true; threadPool.shutdown(); - for (ExecutorService fetcherThreadPool : fetcherThreadPoolList) { - fetcherThreadPool.shutdown(); - } + fetcherThreadPoolList.forEach(ExecutorService::shutdown); super.serviceStop(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java index 920dfe5205..87c13a8059 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java @@ -68,6 +68,7 @@ import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; import static org.apache.tajo.ResourceProtos.*; @@ -190,9 +191,7 @@ private void startScriptExecutors() throws IOException { } private void stopScriptExecutors() { - for (TajoScriptEngine executor : context.getEvalContext().getAllScriptEngines()) { - executor.shutdown(); - } + context.getEvalContext().getAllScriptEngines().forEach(TajoScriptEngine::shutdown); } @Override @@ -384,9 +383,7 @@ private void waitForFetch() throws InterruptedException, IOException { Set broadcastTableNames = new HashSet<>(); List broadcasts = context.getEnforcer().getEnforceProperties(EnforceType.BROADCAST); if (broadcasts != null) { - for (EnforceProperty eachBroadcast : broadcasts) { - broadcastTableNames.add(eachBroadcast.getBroadcast().getTableName()); - } + broadcastTableNames.addAll(broadcasts.stream().map(eachBroadcast -> eachBroadcast.getBroadcast().getTableName()).collect(Collectors.toList())); } // localize the fetched data and skip the broadcast table @@ -613,15 +610,13 @@ public void run() { try { List fetched = fetcher.get(); if (fetcher.getState() == FetcherState.FETCH_DATA_FINISHED) { - for (FileChunk eachFetch : fetched) { - if (eachFetch.getFile() != null) { - if (!eachFetch.fromRemote()) { - localChunks.add(eachFetch); - } else { - remoteChunks.add(eachFetch); - } + fetched.stream().filter(eachFetch -> eachFetch.getFile() != null).forEach(eachFetch -> { + if (!eachFetch.fromRemote()) { + localChunks.add(eachFetch); + } else { + remoteChunks.add(eachFetch); } - } + }); break; } } catch (Throwable e) { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java index efdda9abf6..6d677c5689 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java @@ -91,9 +91,7 @@ protected void serviceInit(Configuration conf) throws Exception { @Override protected void serviceStop() throws Exception { - for(ExecutionBlockContext context: executionBlockContextMap.values()) { - context.stop(); - } + executionBlockContextMap.values().forEach(ExecutionBlockContext::stop); super.serviceStop(); } @@ -216,15 +214,13 @@ public void handle(TaskManagerEvent event) { QueryStopEvent queryStopEvent = TUtil.checkTypeAndGet(event, QueryStopEvent.class); //cleanup failure ExecutionBlock - for (ExecutionBlockId ebId : executionBlockContextMap.keySet()) { - if (ebId.getQueryId().equals(queryStopEvent.getQueryId())) { - try { - executionBlockContextMap.remove(ebId).stop(); - } catch (Exception e) { - LOG.fatal(e.getMessage(), e); - } + executionBlockContextMap.keySet().stream().filter(ebId -> ebId.getQueryId().equals(queryStopEvent.getQueryId())).forEach(ebId -> { + try { + executionBlockContextMap.remove(ebId).stop(); + } catch (Exception e) { + LOG.fatal(e.getMessage(), e); } - } + }); workerContext.cleanup(queryStopEvent.getQueryId().toString()); break; } diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java index 9311139c41..730cc55a9f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -100,12 +101,8 @@ public Response run(JerseyResourceDelegateContext context) { MasterContext masterContext = context.get(masterContextKey); Map workerMap = masterContext.getResourceManager().getNodes(); - List workerList = new ArrayList<>(); - - for (NodeStatus nodeStatus : workerMap.values()) { - workerList.add(new WorkerResponse(nodeStatus)); - } - + List workerList = workerMap.values().stream().map(WorkerResponse::new).collect(Collectors.toList()); + Map> workerResponseMap = new HashMap<>(); workerResponseMap.put(workersName, workerList); diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java index 871328d66d..3a67134d81 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; @Path("/functions") public class FunctionsResource { @@ -91,9 +92,7 @@ public Response run(JerseyResourceDelegateContext context) { if (functionDescriptors.size() > 0) { List functionSignature = new ArrayList<>(functionDescriptors.size()); - for (FunctionDesc functionDesc : functionDescriptors) { - functionSignature.add(functionDesc.getSignature()); - } + functionSignature.addAll(functionDescriptors.stream().map(FunctionDesc::getSignature).collect(Collectors.toList())); return Response.ok(functionSignature).build(); } else { diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java index defb342ade..2e42d5fcbe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java @@ -44,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Path("/queries") public class QueryResource { @@ -173,12 +174,8 @@ public Response run(JerseyResourceDelegateContext context) { private List selectQueriesInfoByState(List queriesInfo, TajoProtos.QueryState state) { List resultQueriesInfo = new ArrayList<>(queriesInfo.size() / 2); - - for (QueryInfo queryInfo: queriesInfo) { - if (state.equals(queryInfo.getQueryState())) { - resultQueriesInfo.add(queryInfo); - } - } + + resultQueriesInfo.addAll(queriesInfo.stream().filter(queryInfo -> state.equals(queryInfo.getQueryState())).collect(Collectors.toList())); return resultQueriesInfo; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index 65d175915e..c3b38c01fd 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -43,6 +43,7 @@ import org.apache.tajo.util.graph.DirectedGraphCursor; import java.util.*; +import java.util.stream.Collectors; import static org.apache.tajo.plan.LogicalPlan.BlockEdge; import static org.apache.tajo.plan.joinorder.GreedyHeuristicJoinOrderAlgorithm.getCost; @@ -161,11 +162,7 @@ private static LogicalNode handleRemainingFiltersIfNecessary(JoinGraphContext jo markAsEvaluated.addAll(joinGraphContext.getEvaluatedJoinFilters()); Set remainingQuals = new HashSet<>(joinGraphContext.getCandidateJoinFilters()); for (JoinEdge eachEdge : joinEdges) { - for (EvalNode eachQual : eachEdge.getJoinQual()) { - if (!markAsEvaluated.contains(eachQual)) { - remainingQuals.add(eachQual); - } - } + remainingQuals.addAll(eachEdge.getJoinQual().stream().filter(eachQual -> !markAsEvaluated.contains(eachQual)).collect(Collectors.toList())); } if (!remainingQuals.isEmpty()) { @@ -199,9 +196,7 @@ public LogicalNode visitJoin(Set ctx, LogicalPlan plan, LogicalPlan.Quer super.visitJoin(ctx, plan, block, node, stack); if (node.hasTargets()) { - for (Target target : node.getTargets()) { - ctx.add(target); - } + ctx.addAll(node.getTargets().stream().collect(Collectors.toList())); } return node; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index f321c76c83..b6e961a1bc 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -44,6 +44,7 @@ import java.lang.reflect.Constructor; import java.util.*; +import java.util.stream.Collectors; /** * This represents and keeps every information about a query plan for a query. @@ -266,10 +267,7 @@ public QueryBlock getBlock(LogicalNode node) { public void removeBlock(QueryBlock block) { queryBlocks.remove(block.getName()); - List tobeRemoved = new ArrayList<>(); - for (Map.Entry entry : queryBlockByPID.entrySet()) { - tobeRemoved.add(entry.getKey()); - } + List tobeRemoved = queryBlockByPID.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList()); for (Integer rn : tobeRemoved) { queryBlockByPID.remove(rn); } @@ -288,10 +286,7 @@ public QueryBlock getParentBlock(QueryBlock block) { } public List getChildBlocks(QueryBlock block) { - List childBlocks = new ArrayList<>(); - for (String blockName : queryBlockGraph.getChilds(block.getName())) { - childBlocks.add(queryBlocks.get(blockName)); - } + List childBlocks = queryBlockGraph.getChilds(block.getName()).stream().map(blockName -> queryBlocks.get(blockName)).collect(Collectors.toList()); return childBlocks; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java index ee47700d1b..049e647aa6 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java @@ -33,6 +33,7 @@ import org.apache.tajo.util.TUtil; import java.util.*; +import java.util.stream.Collectors; import static org.apache.tajo.catalog.TypeConverter.convert; @@ -223,9 +224,7 @@ public String addNamedExpr(NamedExpr namedExpr) { public Collection getAllNamedExprs() { List namedExprList = Lists.newArrayList(); - for (Map.Entry entry: idToExprBiMap.entrySet()) { - namedExprList.add(new NamedExpr(entry.getValue(), idToNamesMap.get(entry.getKey()).get(0))); - } + namedExprList.addAll(idToExprBiMap.entrySet().stream().map(entry -> new NamedExpr(entry.getValue(), idToNamesMap.get(entry.getKey()).get(0))).collect(Collectors.toList())); return namedExprList; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java index c158f5744b..10a1adde67 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java @@ -32,6 +32,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * This is a greedy heuristic algorithm to find a bushy join tree. This algorithm finds @@ -46,10 +47,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm { public FoundJoinOrder findBestOrder(LogicalPlan plan, LogicalPlan.QueryBlock block, JoinGraphContext graphContext) throws TajoException { - Set vertexes = new HashSet<>(); - for (RelationNode relationNode : block.getRelations()) { - vertexes.add(new RelationVertex(relationNode)); - } + Set vertexes = block.getRelations().stream().map(RelationVertex::new).collect(Collectors.toSet()); // As illustrated at LogicalOptimizer.JoinGraphBuilder, the join graph initially forms a kind of tree. // This join graph can be updated by adding new join edges or removing existing join edges diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java index b0a22dba93..bfbe28e8ca 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; public class JoinGraphContext { private Set rootVertexes = new HashSet<>(); // most left vertex in the join plan @@ -47,19 +48,11 @@ public JoinGraph getJoinGraph() { } public void addCandidateJoinConditions(Collection candidates) { - for (EvalNode eachCandidate : candidates) { - if (!evaluatedJoinConditions.contains(eachCandidate)) { - candidateJoinConditions.add(eachCandidate); - } - } + candidateJoinConditions.addAll(candidates.stream().filter(eachCandidate -> !evaluatedJoinConditions.contains(eachCandidate)).collect(Collectors.toList())); } public void addCandidateJoinFilters(Collection candidates) { - for (EvalNode eachCandidate : candidates) { - if (!evaluatedJoinFilters.contains(eachCandidate)) { - candidateJoinFilters.add(eachCandidate); - } - } + candidateJoinFilters.addAll(candidates.stream().filter(eachCandidate -> !evaluatedJoinFilters.contains(eachCandidate)).collect(Collectors.toList())); } public void removeCandidateJoinConditions(Collection willBeRemoved) { @@ -71,21 +64,17 @@ public void removeCandidateJoinFilters(Collection willBeRemoved) { } public void markAsEvaluatedJoinConditions(Collection willBeMarked) { - for (EvalNode eachEval : willBeMarked) { - if (candidateJoinConditions.contains(eachEval)) { - candidateJoinConditions.remove(eachEval); - evaluatedJoinConditions.add(eachEval); - } - } + willBeMarked.stream().filter(eachEval -> candidateJoinConditions.contains(eachEval)).forEach(eachEval -> { + candidateJoinConditions.remove(eachEval); + evaluatedJoinConditions.add(eachEval); + }); } public void markAsEvaluatedJoinFilters(Collection willBeMarked) { - for (EvalNode eachEval : willBeMarked) { - if (candidateJoinFilters.contains(eachEval)) { - candidateJoinFilters.remove(eachEval); - evaluatedJoinFilters.add(eachEval); - } - } + willBeMarked.stream().filter(eachEval -> candidateJoinFilters.contains(eachEval)).forEach(eachEval -> { + candidateJoinFilters.remove(eachEval); + evaluatedJoinFilters.add(eachEval); + }); } public Set getCandidateJoinConditions() { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java index b4191391cb..45f38dcbd7 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java @@ -28,6 +28,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; public class JoinOrderingUtil { @@ -42,13 +43,8 @@ public class JoinOrderingUtil { */ public static Set findJoinConditionForJoinVertex(Set candidates, JoinEdge edge, boolean isOnPredicates) { - Set conditionsForThisJoin = new HashSet<>(); - for (EvalNode predicate : candidates) { - if (EvalTreeUtil.isJoinQual(predicate, false) - && checkIfEvaluatedAtEdge(predicate, edge, isOnPredicates)) { - conditionsForThisJoin.add(predicate); - } - } + Set conditionsForThisJoin = candidates.stream().filter(predicate -> EvalTreeUtil.isJoinQual(predicate, false) + && checkIfEvaluatedAtEdge(predicate, edge, isOnPredicates)).collect(Collectors.toSet()); return conditionsForThisJoin; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java index c9d5373441..ee691dba1a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java @@ -251,15 +251,13 @@ public Column[] getFirstStageShuffleKeyColumns() { } } } - for (GroupbyNode eachGroupbyNode: subGroupbyPlan) { - if (eachGroupbyNode.getGroupingColumns() != null && eachGroupbyNode.getGroupingColumns().length > 0) { - for (Column eachColumn: eachGroupbyNode.getGroupingColumns()) { - if (!shuffleKeyColumns.contains(eachColumn)) { - shuffleKeyColumns.add(eachColumn); - } + subGroupbyPlan.stream().filter(eachGroupbyNode -> eachGroupbyNode.getGroupingColumns() != null && eachGroupbyNode.getGroupingColumns().length > 0).forEach(eachGroupbyNode -> { + for (Column eachColumn : eachGroupbyNode.getGroupingColumns()) { + if (!shuffleKeyColumns.contains(eachColumn)) { + shuffleKeyColumns.add(eachColumn); } } - } + }); return shuffleKeyColumns.toArray(new Column[shuffleKeyColumns.size()]); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java index 95f8f98090..5d23cf9fa3 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java @@ -35,6 +35,7 @@ import org.apache.tajo.util.StringUtils; import java.util.*; +import java.util.stream.Collectors; /** * Column name resolution utility. A SQL statement can include many kinds of column names, @@ -125,13 +126,7 @@ public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tabl */ public static Collection lookupTableByColumns(LogicalPlan.QueryBlock block, String columnName) { - Set found = new HashSet<>(); - - for (RelationNode rel : block.getRelations()) { - if (rel.getLogicalSchema().contains(columnName)) { - found.add(rel); - } - } + Set found = block.getRelations().stream().filter(rel -> rel.getLogicalSchema().contains(columnName)).collect(Collectors.toSet()); return found; } @@ -246,25 +241,18 @@ static Column lookupColumnFromAllRelsInBlock(LogicalPlan.QueryBlock block, List candidates = new ArrayList<>(); - for (RelationNode rel : block.getRelations()) { - if (rel.isNameResolveBase()) { - Column found = rel.getLogicalSchema().getColumn(columnName); - if (found != null) { - candidates.add(found); - } + block.getRelations().stream().filter(rel -> rel.isNameResolveBase()).forEach(rel -> { + Column found = rel.getLogicalSchema().getColumn(columnName); + if (found != null) { + candidates.add(found); } - } + }); if (!candidates.isEmpty()) { return ensureUniqueColumn(candidates); } else { if (includeSelfDescTable) { - List candidateRels = new ArrayList<>(); - for (RelationNode rel : block.getRelations()) { - if (describeSchemaByItself(rel)) { - candidateRels.add(rel); - } - } + List candidateRels = block.getRelations().stream().filter(rel -> describeSchemaByItself(rel)).collect(Collectors.toList()); if (candidateRels.size() == 1) { return guessColumn(IdentifierUtil.buildFQName(candidateRels.get(0).getCanonicalName(), columnName)); } else if (candidateRels.size() > 1) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java index 114704414c..c0c4285566 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java @@ -33,6 +33,7 @@ import java.util.HashSet; import java.util.Set; import java.util.Stack; +import java.util.stream.Collectors; /** * Condition reduce rule reduces the query predicate based on distributivity. @@ -181,11 +182,7 @@ private EvalNode rewrite(BinaryEval evalNode) { new HashSet<>(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(rightChild))) : new HashSet<>(Arrays.asList(AlgebraicUtil.toDisjunctiveNormalFormArray(rightChild))); - for (EvalNode eachLeftChildSplit : leftChildSplits) { - if (rightChildSplits.contains(eachLeftChildSplit)) { - commonQuals.add(eachLeftChildSplit); - } - } + commonQuals.addAll(leftChildSplits.stream().filter(eachLeftChildSplit -> rightChildSplits.contains(eachLeftChildSplit)).collect(Collectors.toList())); if (leftChildSplits.size() == rightChildSplits.size() && commonQuals.size() == leftChildSplits.size()) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index eb45af1eee..e31a2654c1 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -45,6 +45,7 @@ import org.apache.tajo.schema.IdentifierUtil; import java.util.*; +import java.util.stream.Collectors; /** * This rule tries to push down all filter conditions into logical nodes as lower as possible. @@ -150,12 +151,7 @@ public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, } else { // if there remain search conditions // check if it can be evaluated here - Set matched = new HashSet<>(); - for (EvalNode eachEval : context.pushingDownFilters) { - if (LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)) { - matched.add(eachEval); - } - } + Set matched = context.pushingDownFilters.stream().filter(eachEval -> LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)).collect(Collectors.toSet()); // if there are search conditions which can be evaluated here, // push down them and remove them from context.pushingDownFilters. @@ -216,13 +212,9 @@ public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, Lo // In this case, this join is the top most one within a query block. boolean isTopMostJoin = stack.isEmpty() ? true : stack.peek().getType() != NodeType.JOIN; - for (EvalNode evalNode : context.pushingDownFilters) { - // TODO: currently, non-equi theta join is not supported yet. - if (LogicalPlanner.isEvaluatableJoinQual(block, evalNode, joinNode, onPredicates.contains(evalNode), - isTopMostJoin)) { - matched.add(evalNode); - } - } + // TODO: currently, non-equi theta join is not supported yet. + matched.addAll(context.pushingDownFilters.stream().filter(evalNode -> LogicalPlanner.isEvaluatableJoinQual(block, evalNode, joinNode, onPredicates.contains(evalNode), + isTopMostJoin)).collect(Collectors.toList())); EvalNode qual = null; if (matched.size() > 1) { @@ -364,19 +356,11 @@ private static Set extractNonPushableOuterJoinQuals(final LogicalPlan Set nonPushableQuals = new HashSet<>(); for (EvalNode eachQual : onPredicates) { - for (String relName : preservedTableNameSet) { - if (isEvalNeedRelation(eachQual, relName)) { - nonPushableQuals.add(eachQual); - } - } + nonPushableQuals.addAll(preservedTableNameSet.stream().filter(relName -> isEvalNeedRelation(eachQual, relName)).map(relName -> eachQual).collect(Collectors.toList())); } for (EvalNode eachQual : wherePredicates) { - for (String relName : nullSupplyingTableNameSet) { - if (isEvalNeedRelation(eachQual, relName)) { - nonPushableQuals.add(eachQual); - } - } + nonPushableQuals.addAll(nullSupplyingTableNameSet.stream().filter(relName -> isEvalNeedRelation(eachQual, relName)).map(relName -> eachQual).collect(Collectors.toList())); } return nonPushableQuals; @@ -414,12 +398,7 @@ private static boolean isNonEquiThetaJoinQual(final LogicalPlan.QueryBlock block private static List extractNonEquiThetaJoinQuals(final Set predicates, final LogicalPlan.QueryBlock block, final JoinNode joinNode) { - List nonEquiThetaJoinQuals = new ArrayList<>(); - for (EvalNode eachEval: predicates) { - if (isNonEquiThetaJoinQual(block, joinNode, eachEval)) { - nonEquiThetaJoinQuals.add(eachEval); - } - } + List nonEquiThetaJoinQuals = predicates.stream().filter(eachEval -> isNonEquiThetaJoinQual(block, joinNode, eachEval)).collect(Collectors.toList()); return nonEquiThetaJoinQuals; } @@ -473,11 +452,9 @@ private Map transformEvalsWidthByPassNode( } Set columns = EvalTreeUtil.findUniqueColumns(copy); - for (Column c : columns) { - if (c.hasQualifier()) { - EvalTreeUtil.changeColumnRef(copy, c.getQualifiedName(), c.getSimpleName()); - } - } + columns.stream().filter(c -> c.hasQualifier()).forEach(c -> { + EvalTreeUtil.changeColumnRef(copy, c.getQualifiedName(), c.getSimpleName()); + }); transformedMap.put(copy, eval); } @@ -608,9 +585,7 @@ public LogicalNode visitProjection(FilterPushDownContext context, stack.pop(); // find not matched after visiting child - for (EvalNode eval: context.pushingDownFilters) { - notMatched.add(transformedMap.get(eval)); - } + notMatched.addAll(context.pushingDownFilters.stream().map(transformedMap::get).collect(Collectors.toList())); EvalNode qual = null; if (notMatched.size() > 1) { @@ -647,10 +622,7 @@ public LogicalNode visitProjection(FilterPushDownContext context, } private Collection reverseTransform(BiMap map, Set remainFilters) { - Set reversed = new HashSet<>(); - for (EvalNode evalNode : remainFilters) { - reversed.add(map.get(evalNode)); - } + Set reversed = remainFilters.stream().map(map::get).collect(Collectors.toSet()); return reversed; } @@ -785,12 +757,7 @@ private List addHavingNode(FilterPushDownContext context, LogicalPlan GroupbyNode groupByNode) throws TajoException { // find aggregation column Set groupingColumns = new HashSet<>(Arrays.asList(groupByNode.getGroupingColumns())); - Set aggrFunctionOutColumns = new HashSet<>(); - for (Column column : groupByNode.getOutSchema().getRootColumns()) { - if (!groupingColumns.contains(column)) { - aggrFunctionOutColumns.add(column.getQualifiedName()); - } - } + Set aggrFunctionOutColumns = groupByNode.getOutSchema().getRootColumns().stream().filter(column -> !groupingColumns.contains(column)).map(Column::getQualifiedName).collect(Collectors.toSet()); List aggrEvalOrigins = new ArrayList<>(); List aggrEvals = new ArrayList<>(); @@ -949,11 +916,7 @@ public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, Map transformed = findCanPushdownAndTransform(context, block, scanNode, null, notMatched, partitionColumns, 0); - for (EvalNode eval : transformed.keySet()) { - if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, scanNode)) { - matched.add(eval); - } - } + matched.addAll(transformed.keySet().stream().filter(eval -> LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, scanNode)).collect(Collectors.toList())); EvalNode qual = null; if (matched.size() > 1) { @@ -1005,9 +968,7 @@ public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, } } - for (EvalNode matchedEval: matched) { - transformed.remove(matchedEval); - } + matched.forEach(transformed::remove); context.setToOrigin(transformed); context.addFiltersTobePushed(notMatched); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java index fa0fb4ff14..08079ba868 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java @@ -42,6 +42,7 @@ import org.apache.tajo.util.TUtil; import java.util.*; +import java.util.stream.Collectors; /** * ProjectionPushDownRule deploys expressions in a selection list to proper @@ -366,11 +367,7 @@ class FilteredTargetIterator implements Iterator { Iterator iterator; public FilteredTargetIterator(Set required) { - for (String name : nameToIdBiMap.keySet()) { - if (required.contains(name)) { - filtered.add(getTarget(name)); - } - } + filtered.addAll(nameToIdBiMap.keySet().stream().filter(name -> required.contains(name)).map(TargetListManager.this::getTarget).collect(Collectors.toList())); iterator = filtered.iterator(); } @@ -432,9 +429,7 @@ public String addExpr(EvalNode evalNode) throws DuplicateColumnException { } private void addNecessaryReferences(EvalNode evalNode) { - for (Column column : EvalTreeUtil.findUniqueColumns(evalNode)) { - requiredSet.add(column.getQualifiedName()); - } + requiredSet.addAll(EvalTreeUtil.findUniqueColumns(evalNode).stream().map(Column::getQualifiedName).collect(Collectors.toList())); } @Override @@ -968,12 +963,7 @@ static class FilteredStringsIterator implements Iterator { Iterator iterator; FilteredStringsIterator(Collection targetNames, Collection required) { - List filtered = new ArrayList<>(); - for (String name : targetNames) { - if (required.contains(name)) { - filtered.add(name); - } - } + List filtered = targetNames.stream().filter(name -> required.contains(name)).collect(Collectors.toList()); iterator = filtered.iterator(); } @@ -1016,11 +1006,7 @@ static class FilteredIterator implements Iterator { } } - for (String name : requiredReferences) { - if (targetSet.containsKey(name)) { - filtered.add(targetSet.get(name)); - } - } + filtered.addAll(requiredReferences.stream().filter(name -> targetSet.containsKey(name)).map(targetSet::get).collect(Collectors.toList())); iterator = filtered.iterator(); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 2b197ff0ec..39d5900397 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -86,16 +86,13 @@ public Expr visitProjection(Context context, Stack stack, Projection expr) Set names = new HashSet<>(); - for (NamedExpr namedExpr : expr.getNamedExprs()) { - - if (namedExpr.hasAlias()) { - if (names.contains(namedExpr.getAlias())) { - context.state.addVerification(SyntaxErrorUtil.makeDuplicateAlias(namedExpr.getAlias())); - } else { - names.add(namedExpr.getAlias()); - } + expr.getNamedExprs().stream().filter(namedExpr -> namedExpr.hasAlias()).forEach(namedExpr -> { + if (names.contains(namedExpr.getAlias())) { + context.state.addVerification(SyntaxErrorUtil.makeDuplicateAlias(namedExpr.getAlias())); + } else { + names.add(namedExpr.getAlias()); } - } + }); return expr; } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java index 14db7c0bc3..96b9c9d5ff 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java @@ -95,9 +95,7 @@ protected synchronized static void clearCache() { */ public static void shutdown() throws IOException { synchronized(storageManagers) { - for (Tablespace eachTablespace : storageManagers.values()) { - eachTablespace.close(); - } + storageManagers.values().forEach(Tablespace::close); } clearCache(); } diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index e3f7c25e4b..ec23cdd913 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -372,9 +372,7 @@ public static HTableDescriptor parseHTableDescriptor(TableMeta tableMeta, Schema Collection columnFamilies = columnMapping.getColumnFamilyNames(); //If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column. if (columnFamilies.isEmpty()) { - for (Column eachColumn: schema.getRootColumns()) { - columnFamilies.add(eachColumn.getSimpleName()); - } + columnFamilies.addAll(schema.getRootColumns().stream().map(Column::getSimpleName).collect(Collectors.toList())); } for (String eachColumnFamily: columnFamilies) { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderWrapper.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderWrapper.java index ccd528b1f7..756ba6ec05 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderWrapper.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderWrapper.java @@ -167,9 +167,7 @@ public List> getPages() { public List>> getMergedTupleIndexes() { List>> merged = new ArrayList<>(); - for (List>> eachFailureIndex: taskTupleIndexes.values()) { - merged.addAll(eachFailureIndex); - } + taskTupleIndexes.values().forEach(merged::addAll); return merged; } From fa01f22d4f6e8f3fd9717f3142cb721e21e784eb Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Fri, 15 Jan 2016 16:52:50 +0900 Subject: [PATCH 02/17] rollback to fix error --- .../apache/tajo/querymaster/DefaultTaskScheduler.java | 6 +++++- .../main/java/org/apache/tajo/querymaster/Stage.java | 4 +++- .../main/java/org/apache/tajo/querymaster/Task.java | 11 ++++++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index 07fe50b884..9bf7a73db7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -290,7 +290,11 @@ private Set getWorkerIds(Collection hosts){ Set workerIds = Sets.newHashSet(); if(hosts.isEmpty()) return workerIds; - workerIds.addAll(stage.getContext().getWorkerMap().values().stream().filter(worker -> hosts.contains(worker.getHost())).map((Function) WorkerConnectionInfo::getId).collect(Collectors.toList())); + for (WorkerConnectionInfo worker : stage.getContext().getWorkerMap().values()) { + if(hosts.contains(worker.getHost())){ + workerIds.add(worker.getId()); + } + } return workerIds; } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index e308524ce6..5b7532d719 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -495,7 +495,9 @@ private StageHistory makeStageHistory() { } Set partitions = Sets.newHashSet(); - partitions.addAll(getHashShuffleIntermediateEntries().stream().map((Function) IntermediateEntry::getPartId).collect(Collectors.toList())); + for (IntermediateEntry entry : getHashShuffleIntermediateEntries()) { + partitions.add(entry.getPartId()); + } stageHistory.setTotalInputBytes(totalInputBytes); stageHistory.setTotalReadBytes(totalReadBytes); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java index 62d7cdde0d..e6cd5503dc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java @@ -782,11 +782,16 @@ public IntermediateEntry(IntermediateEntryProto proto) { this.volume = proto.getVolume(); failureRowNums = new ArrayList<>(); - failureRowNums.addAll(proto.getFailuresList().stream().map(eachFailure -> new Pair(eachFailure.getPagePos(), - new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum()))).collect(Collectors.toList())); + for (FailureIntermediateProto eachFailure: proto.getFailuresList()) { + + failureRowNums.add(new Pair(eachFailure.getPagePos(), + new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum()))); + } pages = new ArrayList<>(); - pages.addAll(proto.getPagesList().stream().map(eachPage -> new Pair(eachPage.getPos(), eachPage.getLength())).collect(Collectors.toList())); + for (IntermediateEntryProto.PageProto eachPage: proto.getPagesList()) { + pages.add(new Pair(eachPage.getPos(), eachPage.getLength())); + } } public IntermediateEntry(int taskId, int attemptId, int partId, PullHost host) { From 2960042113c61dcccac68fb66083d60ba18e4c77 Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Fri, 15 Jan 2016 17:47:26 +0900 Subject: [PATCH 03/17] change code's line to keep code convention --- .../tajo/catalog/store/HiveCatalogStore.java | 12 ++++++++---- .../org/apache/tajo/catalog/CatalogServer.java | 6 ++++-- .../catalog/store/XMLCatalogSchemaManager.java | 3 ++- .../cli/tsql/commands/DescFunctionCommand.java | 3 ++- .../java/org/apache/tajo/QueryTestCaseBase.java | 3 ++- .../org/apache/tajo/client/TestTajoClient.java | 3 ++- .../planner/global/ExecutionBlockCursor.java | 3 ++- .../tajo/engine/planner/global/MasterPlan.java | 3 ++- .../global/builder/DistinctGroupbyBuilder.java | 15 ++++++++++----- .../org/apache/tajo/engine/utils/TableCache.java | 3 ++- .../exec/NonForwardQueryResultSystemScanner.java | 6 ++++-- .../org/apache/tajo/parser/sql/SQLAnalyzer.java | 7 +++++-- .../tajo/querymaster/DefaultTaskScheduler.java | 4 +++- .../java/org/apache/tajo/querymaster/Query.java | 4 +++- .../apache/tajo/querymaster/Repartitioner.java | 3 ++- .../apache/tajo/util/history/HistoryWriter.java | 3 ++- .../apache/tajo/util/history/QueryHistory.java | 3 ++- .../java/org/apache/tajo/worker/TaskImpl.java | 3 ++- .../java/org/apache/tajo/worker/TaskManager.java | 3 ++- .../tajo/ws/rs/resources/QueryResource.java | 3 ++- .../org/apache/tajo/plan/LogicalOptimizer.java | 3 ++- .../java/org/apache/tajo/plan/LogicalPlan.java | 6 ++++-- .../org/apache/tajo/plan/NamedExprsManager.java | 4 +++- .../tajo/plan/joinorder/JoinGraphContext.java | 6 ++++-- .../tajo/plan/logical/DistinctGroupbyNode.java | 3 ++- .../tajo/plan/nameresolver/NameResolver.java | 6 ++++-- .../rewrite/rules/CommonConditionReduceRule.java | 3 ++- .../plan/rewrite/rules/FilterPushDownRule.java | 15 ++++++++++----- .../rewrite/rules/ProjectionPushDownRule.java | 9 ++++++--- .../thirdparty/orc/ByteBufferPoolAdapter.java | 2 +- 30 files changed, 101 insertions(+), 49 deletions(-) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index f7dcb94ae2..15e78ce8d5 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -690,7 +690,8 @@ private void renameColumn(String databaseName, String tableName, CatalogProtos.A Table table = client.getHiveClient().getTable(databaseName, tableName); List columns = table.getSd().getCols(); - columns.stream().filter(currentColumn -> currentColumn.getName().equalsIgnoreCase(alterColumnProto.getOldColumnName())).forEach(currentColumn -> { + columns.stream().filter(currentColumn -> currentColumn.getName() + .equalsIgnoreCase(alterColumnProto.getOldColumnName())).forEach(currentColumn -> { currentColumn.setName(alterColumnProto.getNewColumnName()); }); client.getHiveClient().alter_table(databaseName, tableName, table); @@ -744,7 +745,8 @@ private void addPartition(String databaseName, String tableName, CatalogProtos.P partition.setParameters(params); List values = Lists.newArrayList(); - values.addAll(partitionDescProto.getPartitionKeysList().stream().map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); + values.addAll(partitionDescProto.getPartitionKeysList().stream() + .map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); partition.setValues(values); Table table = client.getHiveClient().getTable(databaseName, tableName); @@ -770,7 +772,8 @@ private void dropPartition(String databaseName, String tableName, CatalogProtos. client = clientPool.getClient(); List values = Lists.newArrayList(); - values.addAll(partitionDescProto.getPartitionKeysList().stream().map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); + values.addAll(partitionDescProto.getPartitionKeysList().stream() + .map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); client.getHiveClient().dropPartition(databaseName, tableName, values, true); } catch (Exception e) { throw new TajoInternalError(e); @@ -1245,7 +1248,8 @@ public void addPartitions(String databaseName, String tableName, List values = Lists.newArrayList(); - values.addAll(partitionDescProto.getPartitionKeysList().stream().map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); + values.addAll(partitionDescProto.getPartitionKeysList().stream() + .map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); partition.setValues(values); Table table = client.getHiveClient().getTable(databaseName, tableName); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 50d31bcf51..a514701bfd 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -1427,7 +1427,8 @@ private FunctionDescProto findFunction(String signature, List func.getSignature().getParameterTypesList() != null && + candidates.addAll(functions.get(signature).stream() + .filter(func -> func.getSignature().getParameterTypesList() != null && CatalogUtil.isMatchedFunction(func.getSignature().getParameterTypesList(), params)).collect(Collectors.toList())); // if there are more than one function candidates, we choose the nearest matched function. @@ -1450,7 +1451,8 @@ private FunctionDescProto findFunction(String signature, FunctionType type, List candidates.addAll(functions.get(signature).stream().filter(func -> func.getSignature().getType() == type && func.getSignature().getParameterTypesList().equals(params)).collect(Collectors.toList())); } else { - candidates.addAll(functions.get(signature).stream().filter(func -> func.getSignature().getParameterTypesList() != null && + candidates.addAll(functions.get(signature).stream() + .filter(func -> func.getSignature().getParameterTypesList() != null && CatalogUtil.isMatchedFunction(func.getSignature().getParameterTypesList(), params)).collect(Collectors.toList())); } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index 12814b2805..147b057571 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -306,7 +306,8 @@ public void upgradeBaseSchema(Connection conn, int currentVersion) { final List candidatePatches = new ArrayList<>(); Statement stmt; - candidatePatches.addAll(this.catalogStore.getPatches().stream().filter(patch -> currentVersion >= patch.getPriorVersion()).collect(Collectors.toList())); + candidatePatches.addAll(this.catalogStore.getPatches().stream() + .filter(patch -> currentVersion >= patch.getPriorVersion()).collect(Collectors.toList())); Collections.sort(candidatePatches); try { diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java index c62999f439..c20cb95e2e 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java @@ -92,7 +92,8 @@ public void invoke(String[] cmd) throws Exception { Map functionMap = new HashMap<>(); - functions.stream().filter(eachFunction -> !functionMap.containsKey(eachFunction.getSupplement().getShortDescription())).forEach(eachFunction -> { + functions.stream().filter(eachFunction -> !functionMap.containsKey(eachFunction.getSupplement() + .getShortDescription())).forEach(eachFunction -> { functionMap.put(eachFunction.getSupplement().getShortDescription(), eachFunction); }); diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 23e61073b3..0f84a45034 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -519,7 +519,8 @@ protected Collection getBatchQueries(Collection paths) throws IOEx List queries = Lists.newArrayList(); for (Path p : paths) { - queries.addAll(SimpleParser.parseScript(FileUtil.readTextFile(new File(p.toUri()))).stream().map(ParsedResult::getStatement).collect(Collectors.toList())); + queries.addAll(SimpleParser.parseScript( + FileUtil.readTextFile(new File(p.toUri()))).stream().map(ParsedResult::getStatement).collect(Collectors.toList())); } return queries; diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java index d5f6e136d1..b07a05fc06 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -176,7 +176,8 @@ public final void testSessionVariables() throws IOException, TajoException, Inte String prefixName = "key_"; String prefixValue = "val_"; - List unsetList = client.getAllSessionVariables().entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList()); + List unsetList = client.getAllSessionVariables().entrySet().stream() + .map(Map.Entry::getKey).collect(Collectors.toList()); client.unsetSessionVariables(unsetList); for (int i = 0; i < 10; i++) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java index cff6cf4bb7..1d2330fedf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java @@ -135,7 +135,8 @@ In the case of the upper plan, buildDepthFirstOrder() makes the following order orderRequiredChildCountMap.get(eachItem.parentEB.getId()).decrementAndGet(); } else { if (eachItem.allSiblingsOrdered()) { - orderedBlocks.addAll(notOrderedSiblingBlocks.stream().map(eachSiblingItem -> eachSiblingItem.eb).collect(Collectors.toList())); + orderedBlocks.addAll(notOrderedSiblingBlocks.stream().map(eachSiblingItem -> eachSiblingItem.eb) + .collect(Collectors.toList())); orderedBlocks.add(eachItem.eb); notOrderedSiblingBlocks.clear(); } else { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java index 670042b082..bd71fc60b6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java @@ -237,7 +237,8 @@ public List getChilds(ExecutionBlock execBlock) { } public List getChilds(ExecutionBlockId id) { - List childBlocks = execBlockGraph.getChilds(id).stream().map(cid -> execBlockMap.get(cid)).collect(Collectors.toList()); + List childBlocks = execBlockGraph.getChilds(id).stream() + .map(cid -> execBlockMap.get(cid)).collect(Collectors.toList()); return childBlocks; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index 4baf58d25f..c1206d52ae 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -227,7 +227,8 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou // Grouping columns are GROUP BY clause's column + Distinct column. List groupingColumns = new ArrayList<>(); - groupbyUniqColumns.stream().filter(eachGroupingColumn -> !groupingColumns.contains(eachGroupingColumn)).forEach(groupingColumns::add); + groupbyUniqColumns.stream().filter(eachGroupingColumn -> + !groupingColumns.contains(eachGroupingColumn)).forEach(groupingColumns::add); distinctGroupbyNode.setGroupingColumns(groupingColumns.toArray(new Column[groupingColumns.size()])); } buildInfo.addAggFunction(aggFunction); @@ -240,7 +241,8 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou List baseGroupByTargets = new ArrayList<>(); baseGroupByTargets.add(new Target(new FieldEval(new Column("?distinctseq", Type.INT2)))); - baseGroupByTargets.addAll(originalGroupingColumns.stream().map(column -> new Target(new FieldEval(column))).collect(Collectors.toList())); + baseGroupByTargets.addAll(originalGroupingColumns.stream().map(column -> + new Target(new FieldEval(column))).collect(Collectors.toList())); //Add child groupby node for each Distinct clause for (DistinctGroupbyNodeBuildInfo buildInfo: distinctNodeBuildInfos.values()) { @@ -386,7 +388,8 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou // Grouping columns are GROUP BY clause's column + Distinct column. List groupingColumns = new ArrayList<>(originalGroupingColumns); - groupbyUniqColumns.stream().filter(eachGroupingColumn -> !groupingColumns.contains(eachGroupingColumn)).forEach(groupingColumns::add); + groupbyUniqColumns.stream().filter(eachGroupingColumn -> + !groupingColumns.contains(eachGroupingColumn)).forEach(groupingColumns::add); distinctGroupbyNode.setGroupingColumns(groupingColumns.toArray(new Column[groupingColumns.size()])); } buildInfo.addAggFunction(aggFunction); @@ -422,7 +425,8 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou // finally this aggregation output tuple's order is GROUP_BY_COL1, COL2, .... + AGG_VALUE, SUM_VALUE, ... GroupbyNode otherGroupbyNode = new GroupbyNode(context.getPlan().getLogicalPlan().newPID()); - List targets = originalGroupingColumns.stream().map(column -> new Target(new FieldEval(column))).collect(Collectors.toList()); + List targets = originalGroupingColumns.stream().map(column -> + new Target(new FieldEval(column))).collect(Collectors.toList()); targets.addAll(otherAggregationFunctionTargets); otherGroupbyNode.setTargets(targets); @@ -520,7 +524,8 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou LinkedHashSet distinctColumns = EvalTreeUtil.findUniqueColumns(secondStageGroupbyNode.getAggFunctions().get(0)); List uniqueDistinctColumn = new ArrayList<>(); // remove origin group by column from distinctColumns - uniqueDistinctColumn.addAll(distinctColumns.stream().filter(eachColumn -> !originGroupColumns.contains(eachColumn)).collect(Collectors.toList())); + uniqueDistinctColumn.addAll(distinctColumns.stream().filter(eachColumn -> + !originGroupColumns.contains(eachColumn)).collect(Collectors.toList())); for (int i = 0; i < originGroupColumns.size(); i++) { secondGroupbyTargets.add(oldTargets.get(i)); if (grpIdx > 0) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java index ba7827e6d7..81316da4f5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java @@ -62,7 +62,8 @@ public synchronized void releaseCache(ExecutionBlockId ebId) { public synchronized List getCacheKeyByExecutionBlockId(ExecutionBlockId ebId) { List keys = Lists.newArrayList(); - keys.addAll(cacheMap.keySet().stream().filter(eachKey -> eachKey.ebId.equals(ebId.toString())).collect(Collectors.toList())); + keys.addAll(cacheMap.keySet().stream().filter(eachKey -> + eachKey.ebId.equals(ebId.toString())).collect(Collectors.toList())); return keys; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 50736be75e..67a469aed6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -528,9 +528,11 @@ private List getClusterInfo(Schema outSchema) { } tuples = new ArrayList<>(queryMasterList.size() + nodeStatusList.size()); - tuples.addAll(queryMasterList.stream().map(queryMaster -> getQueryMasterTuple(outSchema, queryMaster)).collect(Collectors.toList())); + tuples.addAll(queryMasterList.stream().map(queryMaster -> getQueryMasterTuple(outSchema, queryMaster)) + .collect(Collectors.toList())); - tuples.addAll(nodeStatusList.stream().map(nodeStatus -> getWorkerTuple(outSchema, nodeStatus)).collect(Collectors.toList())); + tuples.addAll(nodeStatusList.stream().map(nodeStatus -> getWorkerTuple(outSchema, nodeStatus)) + .collect(Collectors.toList())); return tuples; } diff --git a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java index 1a20028c68..cc57d63154 100644 --- a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java @@ -1351,7 +1351,9 @@ public Expr visitCreate_table_statement(Create_table_statementContext ctx) { @Override public Expr visitTruncate_table_statement(@NotNull Truncate_table_statementContext ctx) { List tableNameContexts = ctx.table_name(); - List tableNames = tableNameContexts.stream().map(eachTableNameContext -> buildIdentifierChain(eachTableNameContext.identifier())).collect(Collectors.toList()); + List tableNames = tableNameContexts.stream() + .map(eachTableNameContext -> buildIdentifierChain(eachTableNameContext.identifier())) + .collect(Collectors.toList()); return new TruncateTable(tableNames); } @@ -1400,7 +1402,8 @@ public PartitionMethodDescExpr parseTablePartitioningClause(Table_partitioning_c } else { // ( PARTITION part_name , ...) List specifiers = Lists.newArrayList(); - specifiers.addAll(hashPartitions.individual_hash_partitions().individual_hash_partition().stream().map(partition -> new PartitionSpecifier(partition.partition_name().getText())).collect(Collectors.toList())); + specifiers.addAll(hashPartitions.individual_hash_partitions().individual_hash_partition().stream() + .map(partition -> new PartitionSpecifier(partition.partition_name().getText())).collect(Collectors.toList())); return new HashPartition(buildColumnReferenceList(hashPartitions.column_reference_list()), specifiers); } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index 9bf7a73db7..8da6c89a54 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -330,7 +330,9 @@ protected LinkedList createTaskRequest(final int incompleteTas masterClientService.reserveNodeResources(callBack.getController(), request.build(), callBack); NodeResourceResponse response = callBack.get(); - taskRequestEvents.addAll(response.getResourceList().stream().map(resource -> new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId())).collect(Collectors.toList())); + taskRequestEvents.addAll(response.getResourceList().stream() + .map(resource -> new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId())) + .collect(Collectors.toList())); return taskRequestEvents; } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 0a34b69d61..4f8f128197 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -320,7 +320,9 @@ private QueryHistory makeQueryHistory() { queryHistory.setDistributedPlan(plan.toString()); List sessionVariables = new ArrayList<>(); - plan.getContext().getAllKeyValus().entrySet().stream().filter(entry -> SessionVars.exists(entry.getKey()) && SessionVars.isPublic(SessionVars.get(entry.getKey()))).forEach(entry -> { + plan.getContext().getAllKeyValus().entrySet().stream() + .filter(entry -> SessionVars.exists(entry.getKey()) && SessionVars.isPublic(SessionVars.get(entry.getKey()))) + .forEach(entry -> { sessionVariables.add(new String[]{entry.getKey(), entry.getValue()}); }); queryHistory.setSessionVariables(sessionVariables); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 28b8fab302..c75d2d1eff 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -514,7 +514,8 @@ private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext sch private static void addJoinShuffle(Stage stage, int partitionId, Map> grouppedPartitions) { Map> fetches = new HashMap<>(); - stage.getMasterPlan().getChilds(stage.getId()).stream().filter(execBlock -> grouppedPartitions.containsKey(execBlock.getId())).forEach(execBlock -> { + stage.getMasterPlan().getChilds(stage.getId()).stream() + .filter(execBlock -> grouppedPartitions.containsKey(execBlock.getId())).forEach(execBlock -> { String name = execBlock.getId().toString(); List requests = mergeShuffleRequest(name, partitionId, HASH_SHUFFLE, grouppedPartitions.get(execBlock.getId())); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java index 91387451ef..bfb893b350 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java @@ -269,7 +269,8 @@ public void run() { Calendar cal = Calendar.getInstance(); cal.add(Calendar.HOUR_OF_DAY, -2); String closeTargetTime = df.format(cal.getTime()); - List closingTargets = taskWriters.keySet().stream().filter(eachWriterTime -> eachWriterTime.compareTo(closeTargetTime) <= 0).collect(Collectors.toList()); + List closingTargets = taskWriters.keySet().stream() + .filter(eachWriterTime -> eachWriterTime.compareTo(closeTargetTime) <= 0).collect(Collectors.toList()); for (String eachWriterTime : closingTargets) { WriterHolder writerHolder; diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java index 4c18d1dc23..4cbbfcdc8a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java @@ -141,7 +141,8 @@ public QueryHistoryProto getProto() { List stageHistoryProtos = new ArrayList<>(); if (stageHistories != null) { - stageHistoryProtos.addAll(stageHistories.stream().map(eachStage -> (eachStage.getProto())).collect(Collectors.toList())); + stageHistoryProtos.addAll(stageHistories.stream() + .map(eachStage -> (eachStage.getProto())).collect(Collectors.toList())); } builder.addAllStageHistories(stageHistoryProtos); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java index 87c13a8059..02d6a6ed44 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java @@ -383,7 +383,8 @@ private void waitForFetch() throws InterruptedException, IOException { Set broadcastTableNames = new HashSet<>(); List broadcasts = context.getEnforcer().getEnforceProperties(EnforceType.BROADCAST); if (broadcasts != null) { - broadcastTableNames.addAll(broadcasts.stream().map(eachBroadcast -> eachBroadcast.getBroadcast().getTableName()).collect(Collectors.toList())); + broadcastTableNames.addAll(broadcasts.stream() + .map(eachBroadcast -> eachBroadcast.getBroadcast().getTableName()).collect(Collectors.toList())); } // localize the fetched data and skip the broadcast table diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java index 6d677c5689..237ea71097 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java @@ -214,7 +214,8 @@ public void handle(TaskManagerEvent event) { QueryStopEvent queryStopEvent = TUtil.checkTypeAndGet(event, QueryStopEvent.class); //cleanup failure ExecutionBlock - executionBlockContextMap.keySet().stream().filter(ebId -> ebId.getQueryId().equals(queryStopEvent.getQueryId())).forEach(ebId -> { + executionBlockContextMap.keySet().stream() + .filter(ebId -> ebId.getQueryId().equals(queryStopEvent.getQueryId())).forEach(ebId -> { try { executionBlockContextMap.remove(ebId).stop(); } catch (Exception e) { diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java index 2e42d5fcbe..65bbd55b24 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java @@ -175,7 +175,8 @@ public Response run(JerseyResourceDelegateContext context) { private List selectQueriesInfoByState(List queriesInfo, TajoProtos.QueryState state) { List resultQueriesInfo = new ArrayList<>(queriesInfo.size() / 2); - resultQueriesInfo.addAll(queriesInfo.stream().filter(queryInfo -> state.equals(queryInfo.getQueryState())).collect(Collectors.toList())); + resultQueriesInfo.addAll(queriesInfo.stream() + .filter(queryInfo -> state.equals(queryInfo.getQueryState())).collect(Collectors.toList())); return resultQueriesInfo; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index c3b38c01fd..995bb4f036 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -162,7 +162,8 @@ private static LogicalNode handleRemainingFiltersIfNecessary(JoinGraphContext jo markAsEvaluated.addAll(joinGraphContext.getEvaluatedJoinFilters()); Set remainingQuals = new HashSet<>(joinGraphContext.getCandidateJoinFilters()); for (JoinEdge eachEdge : joinEdges) { - remainingQuals.addAll(eachEdge.getJoinQual().stream().filter(eachQual -> !markAsEvaluated.contains(eachQual)).collect(Collectors.toList())); + remainingQuals.addAll(eachEdge.getJoinQual().stream() + .filter(eachQual -> !markAsEvaluated.contains(eachQual)).collect(Collectors.toList())); } if (!remainingQuals.isEmpty()) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index b6e961a1bc..38be276317 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -267,7 +267,8 @@ public QueryBlock getBlock(LogicalNode node) { public void removeBlock(QueryBlock block) { queryBlocks.remove(block.getName()); - List tobeRemoved = queryBlockByPID.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList()); + List tobeRemoved = queryBlockByPID.entrySet().stream() + .map(Map.Entry::getKey).collect(Collectors.toList()); for (Integer rn : tobeRemoved) { queryBlockByPID.remove(rn); } @@ -286,7 +287,8 @@ public QueryBlock getParentBlock(QueryBlock block) { } public List getChildBlocks(QueryBlock block) { - List childBlocks = queryBlockGraph.getChilds(block.getName()).stream().map(blockName -> queryBlocks.get(blockName)).collect(Collectors.toList()); + List childBlocks = queryBlockGraph.getChilds(block.getName()).stream() + .map(blockName -> queryBlocks.get(blockName)).collect(Collectors.toList()); return childBlocks; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java index 049e647aa6..4f2756371b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java @@ -224,7 +224,9 @@ public String addNamedExpr(NamedExpr namedExpr) { public Collection getAllNamedExprs() { List namedExprList = Lists.newArrayList(); - namedExprList.addAll(idToExprBiMap.entrySet().stream().map(entry -> new NamedExpr(entry.getValue(), idToNamesMap.get(entry.getKey()).get(0))).collect(Collectors.toList())); + namedExprList.addAll(idToExprBiMap.entrySet().stream() + .map(entry -> new NamedExpr(entry.getValue(), idToNamesMap.get(entry.getKey()).get(0))) + .collect(Collectors.toList())); return namedExprList; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java index bfbe28e8ca..fe5362b51f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java @@ -48,11 +48,13 @@ public JoinGraph getJoinGraph() { } public void addCandidateJoinConditions(Collection candidates) { - candidateJoinConditions.addAll(candidates.stream().filter(eachCandidate -> !evaluatedJoinConditions.contains(eachCandidate)).collect(Collectors.toList())); + candidateJoinConditions.addAll(candidates.stream() + .filter(eachCandidate -> !evaluatedJoinConditions.contains(eachCandidate)).collect(Collectors.toList())); } public void addCandidateJoinFilters(Collection candidates) { - candidateJoinFilters.addAll(candidates.stream().filter(eachCandidate -> !evaluatedJoinFilters.contains(eachCandidate)).collect(Collectors.toList())); + candidateJoinFilters.addAll(candidates.stream() + .filter(eachCandidate -> !evaluatedJoinFilters.contains(eachCandidate)).collect(Collectors.toList())); } public void removeCandidateJoinConditions(Collection willBeRemoved) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java index ee691dba1a..c5c44b6701 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java @@ -251,7 +251,8 @@ public Column[] getFirstStageShuffleKeyColumns() { } } } - subGroupbyPlan.stream().filter(eachGroupbyNode -> eachGroupbyNode.getGroupingColumns() != null && eachGroupbyNode.getGroupingColumns().length > 0).forEach(eachGroupbyNode -> { + subGroupbyPlan.stream().filter(eachGroupbyNode -> eachGroupbyNode.getGroupingColumns() != null + && eachGroupbyNode.getGroupingColumns().length > 0).forEach(eachGroupbyNode -> { for (Column eachColumn : eachGroupbyNode.getGroupingColumns()) { if (!shuffleKeyColumns.contains(eachColumn)) { shuffleKeyColumns.add(eachColumn); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java index 5d23cf9fa3..0471ca4715 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java @@ -126,7 +126,8 @@ public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tabl */ public static Collection lookupTableByColumns(LogicalPlan.QueryBlock block, String columnName) { - Set found = block.getRelations().stream().filter(rel -> rel.getLogicalSchema().contains(columnName)).collect(Collectors.toSet()); + Set found = block.getRelations().stream() + .filter(rel -> rel.getLogicalSchema().contains(columnName)).collect(Collectors.toSet()); return found; } @@ -252,7 +253,8 @@ static Column lookupColumnFromAllRelsInBlock(LogicalPlan.QueryBlock block, return ensureUniqueColumn(candidates); } else { if (includeSelfDescTable) { - List candidateRels = block.getRelations().stream().filter(rel -> describeSchemaByItself(rel)).collect(Collectors.toList()); + List candidateRels = block.getRelations().stream(). + filter(rel -> describeSchemaByItself(rel)).collect(Collectors.toList()); if (candidateRels.size() == 1) { return guessColumn(IdentifierUtil.buildFQName(candidateRels.get(0).getCanonicalName(), columnName)); } else if (candidateRels.size() > 1) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java index c0c4285566..d4de9a3db8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java @@ -182,7 +182,8 @@ private EvalNode rewrite(BinaryEval evalNode) { new HashSet<>(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(rightChild))) : new HashSet<>(Arrays.asList(AlgebraicUtil.toDisjunctiveNormalFormArray(rightChild))); - commonQuals.addAll(leftChildSplits.stream().filter(eachLeftChildSplit -> rightChildSplits.contains(eachLeftChildSplit)).collect(Collectors.toList())); + commonQuals.addAll(leftChildSplits.stream() + .filter(eachLeftChildSplit -> rightChildSplits.contains(eachLeftChildSplit)).collect(Collectors.toList())); if (leftChildSplits.size() == rightChildSplits.size() && commonQuals.size() == leftChildSplits.size()) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index e31a2654c1..0241ba9ddd 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -151,7 +151,8 @@ public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, } else { // if there remain search conditions // check if it can be evaluated here - Set matched = context.pushingDownFilters.stream().filter(eachEval -> LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)).collect(Collectors.toSet()); + Set matched = context.pushingDownFilters.stream() + .filter(eachEval -> LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)).collect(Collectors.toSet()); // if there are search conditions which can be evaluated here, // push down them and remove them from context.pushingDownFilters. @@ -213,7 +214,8 @@ public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, Lo boolean isTopMostJoin = stack.isEmpty() ? true : stack.peek().getType() != NodeType.JOIN; // TODO: currently, non-equi theta join is not supported yet. - matched.addAll(context.pushingDownFilters.stream().filter(evalNode -> LogicalPlanner.isEvaluatableJoinQual(block, evalNode, joinNode, onPredicates.contains(evalNode), + matched.addAll(context.pushingDownFilters.stream() + .filter(evalNode -> LogicalPlanner.isEvaluatableJoinQual(block, evalNode, joinNode, onPredicates.contains(evalNode), isTopMostJoin)).collect(Collectors.toList())); EvalNode qual = null; @@ -398,7 +400,8 @@ private static boolean isNonEquiThetaJoinQual(final LogicalPlan.QueryBlock block private static List extractNonEquiThetaJoinQuals(final Set predicates, final LogicalPlan.QueryBlock block, final JoinNode joinNode) { - List nonEquiThetaJoinQuals = predicates.stream().filter(eachEval -> isNonEquiThetaJoinQual(block, joinNode, eachEval)).collect(Collectors.toList()); + List nonEquiThetaJoinQuals = predicates.stream() + .filter(eachEval -> isNonEquiThetaJoinQual(block, joinNode, eachEval)).collect(Collectors.toList()); return nonEquiThetaJoinQuals; } @@ -757,7 +760,8 @@ private List addHavingNode(FilterPushDownContext context, LogicalPlan GroupbyNode groupByNode) throws TajoException { // find aggregation column Set groupingColumns = new HashSet<>(Arrays.asList(groupByNode.getGroupingColumns())); - Set aggrFunctionOutColumns = groupByNode.getOutSchema().getRootColumns().stream().filter(column -> !groupingColumns.contains(column)).map(Column::getQualifiedName).collect(Collectors.toSet()); + Set aggrFunctionOutColumns = groupByNode.getOutSchema().getRootColumns().stream() + .filter(column -> !groupingColumns.contains(column)).map(Column::getQualifiedName).collect(Collectors.toSet()); List aggrEvalOrigins = new ArrayList<>(); List aggrEvals = new ArrayList<>(); @@ -916,7 +920,8 @@ public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, Map transformed = findCanPushdownAndTransform(context, block, scanNode, null, notMatched, partitionColumns, 0); - matched.addAll(transformed.keySet().stream().filter(eval -> LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, scanNode)).collect(Collectors.toList())); + matched.addAll(transformed.keySet().stream() + .filter(eval -> LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, scanNode)).collect(Collectors.toList())); EvalNode qual = null; if (matched.size() > 1) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java index 08079ba868..d8a37c3b15 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java @@ -367,7 +367,8 @@ class FilteredTargetIterator implements Iterator { Iterator iterator; public FilteredTargetIterator(Set required) { - filtered.addAll(nameToIdBiMap.keySet().stream().filter(name -> required.contains(name)).map(TargetListManager.this::getTarget).collect(Collectors.toList())); + filtered.addAll(nameToIdBiMap.keySet().stream() + .filter(name -> required.contains(name)).map(TargetListManager.this::getTarget).collect(Collectors.toList())); iterator = filtered.iterator(); } @@ -429,7 +430,8 @@ public String addExpr(EvalNode evalNode) throws DuplicateColumnException { } private void addNecessaryReferences(EvalNode evalNode) { - requiredSet.addAll(EvalTreeUtil.findUniqueColumns(evalNode).stream().map(Column::getQualifiedName).collect(Collectors.toList())); + requiredSet.addAll(EvalTreeUtil.findUniqueColumns(evalNode).stream() + .map(Column::getQualifiedName).collect(Collectors.toList())); } @Override @@ -1006,7 +1008,8 @@ static class FilteredIterator implements Iterator { } } - filtered.addAll(requiredReferences.stream().filter(name -> targetSet.containsKey(name)).map(targetSet::get).collect(Collectors.toList())); + filtered.addAll(requiredReferences.stream().filter(name -> targetSet.containsKey(name)) + .map(targetSet::get).collect(Collectors.toList())); iterator = filtered.iterator(); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/ByteBufferPoolAdapter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/ByteBufferPoolAdapter.java index 2e9aec11ad..e5b6561d27 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/ByteBufferPoolAdapter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/ByteBufferPoolAdapter.java @@ -33,7 +33,7 @@ public ByteBufferPoolAdapter(ByteBufferAllocatorPool pool) { public final ByteBuffer getBuffer(boolean direct, int length) { return this.pool.getBuffer(direct, length); } - + @Override public final void putBuffer(ByteBuffer buffer) { this.pool.putBuffer(buffer); From da05ffa38301c7a460aaa92cf00298f2d7daca35 Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Mon, 18 Jan 2016 12:39:08 +0900 Subject: [PATCH 04/17] rewrite more --- .../store/XMLCatalogSchemaManager.java | 20 +++++-------------- .../querymaster/DefaultTaskScheduler.java | 11 +++------- .../org/apache/tajo/querymaster/Stage.java | 12 +++-------- .../org/apache/tajo/querymaster/Task.java | 11 +++------- .../org/apache/tajo/plan/LogicalPlanner.java | 17 +++++----------- .../rewrite/SelfDescSchemaBuildPhase.java | 4 +--- .../storage/HashShuffleAppenderManager.java | 4 +--- .../thirdparty/orc/ByteBufferPoolAdapter.java | 2 +- .../storage/thirdparty/orc/WriterImpl.java | 5 ++--- 9 files changed, 24 insertions(+), 62 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index 147b057571..d2e897ef61 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -600,18 +600,10 @@ protected List mergeDatabaseObjects(List objects unorderedObjects.add(object); } } - - for (DatabaseObject object: orderedObjects) { - if (object != null) { - mergedObjects.add(object); - } - } - - for (DatabaseObject object: unorderedObjects) { - if (object != null) { - mergedObjects.add(object); - } - } + + mergedObjects.addAll(orderedObjects.stream().filter(object -> object != null).collect(Collectors.toList())); + + mergedObjects.addAll(unorderedObjects.stream().filter(object -> object != null).collect(Collectors.toList())); return mergedObjects; } @@ -692,9 +684,7 @@ public StoreObject merge() { boolean alreadySetDatabaseObject = false; // first pass - for (StoreObject store : this.storeObjects) { - copySchemaInfo(store); - } + this.storeObjects.forEach(this::copySchemaInfo); // second pass for (StoreObject store: this.storeObjects) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index 8da6c89a54..d36762b893 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -290,11 +290,8 @@ private Set getWorkerIds(Collection hosts){ Set workerIds = Sets.newHashSet(); if(hosts.isEmpty()) return workerIds; - for (WorkerConnectionInfo worker : stage.getContext().getWorkerMap().values()) { - if(hosts.contains(worker.getHost())){ - workerIds.add(worker.getId()); - } - } + stage.getContext().getWorkerMap().values().stream().filter(worker -> hosts.contains(worker.getHost())) + .forEach(worker -> workerIds.add(worker.getId())); return workerIds; } @@ -1014,9 +1011,7 @@ public void assignToNonLeafTasks(LinkedList taskRequests) thro for(Map.Entry> entry: task.getFetchMap().entrySet()) { Collection fetches = entry.getValue(); if (fetches != null) { - for (FetchProto fetch : fetches) { - taskAssign.addFetch(fetch); - } + fetches.forEach(taskAssign::addFetch); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 5b7532d719..6168e6a953 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -495,9 +495,7 @@ private StageHistory makeStageHistory() { } Set partitions = Sets.newHashSet(); - for (IntermediateEntry entry : getHashShuffleIntermediateEntries()) { - partitions.add(entry.getPartId()); - } + getHashShuffleIntermediateEntries().stream().forEach(entry -> partitions.add(entry.getPartId())); stageHistory.setTotalInputBytes(totalInputBytes); stageHistory.setTotalReadBytes(totalReadBytes); @@ -763,9 +761,7 @@ protected void stopExecutionBlock() { if (!getContext().getQueryContext().getBool(SessionVars.DEBUG_ENABLED)) { List childs = getMasterPlan().getChilds(getId()); - for (ExecutionBlock executionBlock : childs) { - ebIds.add(executionBlock.getId().getProto()); - } + ebIds.addAll(childs.stream().map(executionBlock -> executionBlock.getId().getProto()).collect(Collectors.toList())); } StopExecutionBlockRequest.Builder stopRequest = StopExecutionBlockRequest.newBuilder(); @@ -1328,9 +1324,7 @@ private void finalizeShuffleReport(StageShuffleReportEvent event, ShuffleType ty completedShuffleTasks.addAndGet(report.getSucceededTasks()); if (report.getIntermediateEntriesCount() > 0) { - for (IntermediateEntryProto eachInterm : report.getIntermediateEntriesList()) { - hashShuffleIntermediateEntries.add(new IntermediateEntry(eachInterm)); - } + hashShuffleIntermediateEntries.addAll(report.getIntermediateEntriesList().stream().map(IntermediateEntry::new).collect(Collectors.toList())); } if (completedShuffleTasks.get() >= succeededObjectCount) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java index e6cd5503dc..d0ed4af1e1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java @@ -782,16 +782,11 @@ public IntermediateEntry(IntermediateEntryProto proto) { this.volume = proto.getVolume(); failureRowNums = new ArrayList<>(); - for (FailureIntermediateProto eachFailure: proto.getFailuresList()) { - - failureRowNums.add(new Pair(eachFailure.getPagePos(), - new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum()))); - } + proto.getFailuresList().stream().map(eachFailure -> new Pair(eachFailure.getPagePos(), + new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum()))).forEach(eachFailure -> failureRowNums.add(eachFailure)); pages = new ArrayList<>(); - for (IntermediateEntryProto.PageProto eachPage: proto.getPagesList()) { - pages.add(new Pair(eachPage.getPos(), eachPage.getLength())); - } + proto.getPagesList().stream().map(eachPage -> new Pair(eachPage.getPos(), eachPage.getLength())).forEach(eachPage -> pages.add(eachPage)); } public IntermediateEntry(int taskId, int attemptId, int partId, PullHost host) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index aaf64cd4a7..35506af31f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -60,6 +60,7 @@ import javax.annotation.Nullable; import java.net.URI; import java.util.*; +import java.util.stream.Collectors; import static org.apache.tajo.algebra.CreateTable.PartitionType; import static org.apache.tajo.plan.ExprNormalizer.ExprNormalizedResult; @@ -735,9 +736,7 @@ private LogicalNode insertWindowAggNode(PlanContext context, LogicalNode child, } } } - for (String winFuncRef : winFuncRefs) { - targets.add(block.namedExprsMgr.getTarget(winFuncRef)); - } + targets.addAll(winFuncRefs.stream().map(block.namedExprsMgr::getTarget).collect(Collectors.toList())); windowAggNode.setTargets(targets); verifyProjectedFields(block, windowAggNode); @@ -1189,9 +1188,7 @@ public LogicalNode visitJoin(PlanContext context, Stack stack, Join join) List newlyEvaluatedExprs = getNewlyEvaluatedExprsForJoin(context, joinNode, isTopMostJoin); List targets = new ArrayList<>(PlannerUtil.schemaToTargets(merged)); - for (String newAddedExpr : newlyEvaluatedExprs) { - targets.add(block.namedExprsMgr.getTarget(newAddedExpr, true)); - } + targets.addAll(newlyEvaluatedExprs.stream().map(newAddedExpr -> block.namedExprsMgr.getTarget(newAddedExpr, true)).collect(Collectors.toList())); joinNode.setTargets(targets); // Determine join conditions @@ -1285,9 +1282,7 @@ private LogicalNode createCartesianProduct(PlanContext context, LogicalNode left } List targets = new ArrayList<>(PlannerUtil.schemaToTargets(merged)); - for (String newAddedExpr : newlyEvaluatedExprs) { - targets.add(block.namedExprsMgr.getTarget(newAddedExpr, true)); - } + targets.addAll(newlyEvaluatedExprs.stream().map(newAddedExpr -> block.namedExprsMgr.getTarget(newAddedExpr, true)).collect(Collectors.toList())); join.setTargets(targets); return join; } @@ -1425,9 +1420,7 @@ private void setTargetOfTableSubQuery (PlanContext context, QueryBlock block, Ta // Assume that each unique expr is evaluated once. LinkedHashSet targets = createFieldTargetsFromRelation(block, subQueryNode, newlyEvaluatedExprs); - for (String newAddedExpr : newlyEvaluatedExprs) { - targets.add(block.namedExprsMgr.getTarget(newAddedExpr, true)); - } + targets.addAll(newlyEvaluatedExprs.stream().map(newAddedExpr -> block.namedExprsMgr.getTarget(newAddedExpr, true)).collect(Collectors.toList())); subQueryNode.setTargets(new ArrayList<>(targets)); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java index a49c42389f..c46779ec00 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java @@ -457,9 +457,7 @@ public int compare(Column c1, Column c2) { } // Add simple columns - for (Column eachColumn : simpleColumns) { - schema.add(eachColumn); - } + simpleColumns.forEach(schema::add); return schema.build(); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index 79203a2dc4..575c1e53d0 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -220,9 +220,7 @@ public MemoryRowBlock call() throws Exception { } public void shutdown() { - for (ExecutorService service : executors.values()) { - service.shutdownNow(); - } + executors.values().forEach(ExecutorService::shutdownNow); } public static class HashShuffleIntermediate { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/ByteBufferPoolAdapter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/ByteBufferPoolAdapter.java index e5b6561d27..2e9aec11ad 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/ByteBufferPoolAdapter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/ByteBufferPoolAdapter.java @@ -33,7 +33,7 @@ public ByteBufferPoolAdapter(ByteBufferAllocatorPool pool) { public final ByteBuffer getBuffer(boolean direct, int length) { return this.pool.getBuffer(direct, length); } - + @Override public final void putBuffer(ByteBuffer buffer) { this.pool.putBuffer(buffer); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java index 5dedc2eae8..41df1dd34d 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java @@ -52,6 +52,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; +import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -1989,9 +1990,7 @@ private int writeFooter(long bodyLength) throws IOException { // serialize the types writeTypes(builder, schema); // add the stripe information - for(OrcProto.StripeInformation stripe: stripes) { - builder.addStripes(stripe); - } + stripes.forEach(builder::addStripes); // add the column statistics writeFileStatistics(builder, treeWriter); // add all of the user metadata From 4c68271b640a4d709547da218a0403b2e2e48eff Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Fri, 22 Jan 2016 16:02:10 +0900 Subject: [PATCH 05/17] rewrite --- .../src/main/java/org/apache/tajo/algebra/CreateTable.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java index 4681250d5a..fa43b19ca7 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java @@ -419,8 +419,7 @@ public Object clone() throws CloneNotSupportedException { } hash.quantity = quantity; if (specifiers != null) { - hash.specifiers = new ArrayList<>(); - hash.specifiers.addAll(specifiers.stream().collect(Collectors.toList())); + hash.specifiers = new ArrayList<>(specifiers); } return hash; } @@ -470,8 +469,7 @@ public Object clone() throws CloneNotSupportedException { } } if (specifiers != null) { - listPartition.specifiers = new ArrayList<>(); - listPartition.specifiers.addAll(specifiers.stream().collect(Collectors.toList())); + listPartition.specifiers = new ArrayList<>(specifiers); } return listPartition; } From dce114b97a5a1ddb27b5cfae93718b84e4f4642c Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Thu, 28 Jan 2016 16:39:04 +0900 Subject: [PATCH 06/17] rewrite overall --- .../org/apache/tajo/algebra/CreateTable.java | 4 +-- .../tajo/catalog/store/HiveCatalogStore.java | 15 +++++------ .../apache/tajo/catalog/CatalogServer.java | 19 ++++++-------- .../store/XMLCatalogSchemaManager.java | 11 +++----- .../org/apache/tajo/QueryTestCaseBase.java | 5 ++-- .../planner/global/ExecutionBlockCursor.java | 4 +-- .../engine/planner/global/GlobalPlanner.java | 5 ++-- .../builder/DistinctGroupbyBuilder.java | 6 ++--- .../tajo/engine/query/TaskRequestImpl.java | 9 +++---- .../apache/tajo/engine/utils/TableCache.java | 5 ++-- .../NonForwardQueryResultSystemScanner.java | 9 +++---- .../apache/tajo/parser/sql/SQLAnalyzer.java | 4 +-- .../querymaster/DefaultTaskScheduler.java | 6 ++--- .../org/apache/tajo/querymaster/Stage.java | 5 ++-- .../tajo/util/history/QueryHistory.java | 4 +-- .../util/metrics/RegexpMetricsFilter.java | 3 +-- .../org/apache/tajo/worker/FetchImpl.java | 3 --- .../java/org/apache/tajo/worker/TaskImpl.java | 4 +-- .../ws/rs/resources/FunctionsResource.java | 3 +-- .../tajo/ws/rs/resources/QueryResource.java | 4 +-- .../apache/tajo/plan/LogicalOptimizer.java | 9 +++---- .../org/apache/tajo/plan/LogicalPlan.java | 7 ++---- .../org/apache/tajo/plan/LogicalPlanner.java | 9 +++---- .../apache/tajo/plan/NamedExprsManager.java | 5 ++-- .../tajo/plan/joinorder/JoinGraphContext.java | 9 +++---- .../rules/CommonConditionReduceRule.java | 5 ++-- .../rewrite/rules/FilterPushDownRule.java | 25 ++++++++++--------- .../rewrite/rules/ProjectionPushDownRule.java | 10 +++----- .../tajo/storage/hbase/HBaseTablespace.java | 2 +- .../storage/thirdparty/orc/WriterImpl.java | 1 - 30 files changed, 83 insertions(+), 127 deletions(-) diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java index fa43b19ca7..706e2298c4 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java @@ -419,7 +419,7 @@ public Object clone() throws CloneNotSupportedException { } hash.quantity = quantity; if (specifiers != null) { - hash.specifiers = new ArrayList<>(specifiers); + hash.specifiers = specifiers.stream().collect(Collectors.toList()); } return hash; } @@ -469,7 +469,7 @@ public Object clone() throws CloneNotSupportedException { } } if (specifiers != null) { - listPartition.specifiers = new ArrayList<>(specifiers); + listPartition.specifiers = specifiers.stream().collect(Collectors.toList()); } return listPartition; } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 15e78ce8d5..7198a9f175 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -744,9 +744,8 @@ private void addPartition(String databaseName, String tableName, CatalogProtos.P params.put(StatsSetupConst.TOTAL_SIZE, Long.toString(partitionDescProto.getNumBytes())); partition.setParameters(params); - List values = Lists.newArrayList(); - values.addAll(partitionDescProto.getPartitionKeysList().stream() - .map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); + List values = partitionDescProto.getPartitionKeysList().stream() + .map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList()); partition.setValues(values); Table table = client.getHiveClient().getTable(databaseName, tableName); @@ -771,9 +770,8 @@ private void dropPartition(String databaseName, String tableName, CatalogProtos. client = clientPool.getClient(); - List values = Lists.newArrayList(); - values.addAll(partitionDescProto.getPartitionKeysList().stream() - .map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); + List values = partitionDescProto.getPartitionKeysList().stream() + .map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList()); client.getHiveClient().dropPartition(databaseName, tableName, values, true); } catch (Exception e) { throw new TajoInternalError(e); @@ -1247,9 +1245,8 @@ public void addPartitions(String databaseName, String tableName, List values = Lists.newArrayList(); - values.addAll(partitionDescProto.getPartitionKeysList().stream() - .map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList())); + List values = partitionDescProto.getPartitionKeysList().stream() + .map(PartitionKeyProto::getPartitionValue).collect(Collectors.toList()); partition.setValues(values); Table table = client.getHiveClient().getTable(databaseName, tableName); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index a514701bfd..854dd0db97 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -63,7 +63,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; import static org.apache.tajo.exception.ExceptionUtil.printStackTraceIfError; @@ -1413,8 +1412,8 @@ private FunctionDescProto findFunction(String signature, List candidates = Lists.newArrayList(); if (functions.containsKey(signature)) { - candidates.addAll(functions.get(signature).stream().filter(func -> func.getSignature().getParameterTypesList() != null && - func.getSignature().getParameterTypesList().equals(params)).collect(Collectors.toList())); + functions.get(signature).stream().filter(func -> func.getSignature().getParameterTypesList() != null && + func.getSignature().getParameterTypesList().equals(params)).forEach(candidates::add); } /* @@ -1427,9 +1426,8 @@ private FunctionDescProto findFunction(String signature, List func.getSignature().getParameterTypesList() != null && - CatalogUtil.isMatchedFunction(func.getSignature().getParameterTypesList(), params)).collect(Collectors.toList())); + functions.get(signature).stream().filter(func -> func.getSignature().getParameterTypesList() != null && + CatalogUtil.isMatchedFunction(func.getSignature().getParameterTypesList(), params)).forEach(candidates::add); // if there are more than one function candidates, we choose the nearest matched function. if (candidates.size() > 0) { @@ -1448,12 +1446,11 @@ private FunctionDescProto findFunction(String signature, FunctionType type, List if (functions.containsKey(signature)) { if (strictTypeCheck) { - candidates.addAll(functions.get(signature).stream().filter(func -> func.getSignature().getType() == type && - func.getSignature().getParameterTypesList().equals(params)).collect(Collectors.toList())); + functions.get(signature).stream().filter(func -> func.getSignature().getType() == type && + func.getSignature().getParameterTypesList().equals(params)).forEach(candidates::add); } else { - candidates.addAll(functions.get(signature).stream() - .filter(func -> func.getSignature().getParameterTypesList() != null && - CatalogUtil.isMatchedFunction(func.getSignature().getParameterTypesList(), params)).collect(Collectors.toList())); + functions.get(signature).stream().filter(func -> func.getSignature().getParameterTypesList() != null && + CatalogUtil.isMatchedFunction(func.getSignature().getParameterTypesList(), params)).forEach(candidates::add); } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index d2e897ef61..d8feb52908 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -303,13 +303,10 @@ public void upgradeBaseSchema(Connection conn, int currentVersion) { throw new TajoInternalError("Database schema files are not loaded."); } - final List candidatePatches = new ArrayList<>(); + final List candidatePatches = this.catalogStore.getPatches().stream() + .filter(patch -> currentVersion >= patch.getPriorVersion()).sorted().collect(Collectors.toList()); Statement stmt; - candidatePatches.addAll(this.catalogStore.getPatches().stream() - .filter(patch -> currentVersion >= patch.getPriorVersion()).collect(Collectors.toList())); - - Collections.sort(candidatePatches); try { stmt = conn.createStatement(); } catch (SQLException e) { @@ -601,9 +598,9 @@ protected List mergeDatabaseObjects(List objects } } - mergedObjects.addAll(orderedObjects.stream().filter(object -> object != null).collect(Collectors.toList())); + orderedObjects.stream().filter(object -> object != null).forEach(mergedObjects::add); - mergedObjects.addAll(unorderedObjects.stream().filter(object -> object != null).collect(Collectors.toList())); + unorderedObjects.stream().filter(object -> object != null).forEach(mergedObjects::add); return mergedObjects; } diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 0f84a45034..70c99276cd 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -73,7 +73,6 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.*; -import java.util.stream.Collectors; import static org.junit.Assert.*; @@ -519,8 +518,8 @@ protected Collection getBatchQueries(Collection paths) throws IOEx List queries = Lists.newArrayList(); for (Path p : paths) { - queries.addAll(SimpleParser.parseScript( - FileUtil.readTextFile(new File(p.toUri()))).stream().map(ParsedResult::getStatement).collect(Collectors.toList())); + SimpleParser.parseScript(FileUtil.readTextFile(new File(p.toUri()))).stream() + .map(ParsedResult::getStatement).forEach(queries::add); } return queries; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java index 1d2330fedf..0b45ba532f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java @@ -19,7 +19,6 @@ import java.util.*; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; /** * A distributed execution plan (DEP) is a direct acyclic graph (DAG) of ExecutionBlocks. @@ -135,8 +134,7 @@ In the case of the upper plan, buildDepthFirstOrder() makes the following order orderRequiredChildCountMap.get(eachItem.parentEB.getId()).decrementAndGet(); } else { if (eachItem.allSiblingsOrdered()) { - orderedBlocks.addAll(notOrderedSiblingBlocks.stream().map(eachSiblingItem -> eachSiblingItem.eb) - .collect(Collectors.toList())); + notOrderedSiblingBlocks.stream().map(eachSiblingItem -> eachSiblingItem.eb).forEach(orderedBlocks::add); orderedBlocks.add(eachItem.eb); notOrderedSiblingBlocks.clear(); } else { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 1ed89c089d..7d1ca79787 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -548,8 +548,9 @@ private ExecutionBlock buildGroupByIncludingDistinctFunctionsMultiStage(GlobalPl } } - List firstStageTargets = firstStageGroupingColumns.stream().map(column -> new Target(new FieldEval(column))).collect(Collectors.toList()); - firstStageTargets.addAll(firstPhaseEvalNodeTargets.stream().collect(Collectors.toList())); + List firstStageTargets = firstStageGroupingColumns.stream() + .map(column -> new Target(new FieldEval(column))).collect(Collectors.toList()); + firstPhaseEvalNodeTargets.stream().collect(Collectors.toList()).forEach(firstStageTargets::add); // Create the groupby node for the first stage and set all necessary descriptions GroupbyNode firstStageGroupby = new GroupbyNode(context.plan.getLogicalPlan().newPID()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index c1206d52ae..122600e572 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -241,8 +241,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou List baseGroupByTargets = new ArrayList<>(); baseGroupByTargets.add(new Target(new FieldEval(new Column("?distinctseq", Type.INT2)))); - baseGroupByTargets.addAll(originalGroupingColumns.stream().map(column -> - new Target(new FieldEval(column))).collect(Collectors.toList())); + originalGroupingColumns.stream().map(column -> new Target(new FieldEval(column))).forEach(baseGroupByTargets::add); //Add child groupby node for each Distinct clause for (DistinctGroupbyNodeBuildInfo buildInfo: distinctNodeBuildInfos.values()) { @@ -524,8 +523,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou LinkedHashSet distinctColumns = EvalTreeUtil.findUniqueColumns(secondStageGroupbyNode.getAggFunctions().get(0)); List uniqueDistinctColumn = new ArrayList<>(); // remove origin group by column from distinctColumns - uniqueDistinctColumn.addAll(distinctColumns.stream().filter(eachColumn -> - !originGroupColumns.contains(eachColumn)).collect(Collectors.toList())); + distinctColumns.stream().filter(eachColumn -> !originGroupColumns.contains(eachColumn)).forEach(uniqueDistinctColumn::add); for (int i = 0; i < originGroupColumns.size(); i++) { secondGroupbyTargets.add(oldTargets.get(i)); if (grpIdx > 0) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java index 5f6f6d0bfd..00db037e19 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; @@ -217,14 +216,14 @@ public List getFetches() { return this.fetches; } - private void initFetches() { - if (this.fetches != null) { + private void initFetches() { + if (this.fetches != null) { return; } TaskRequestProtoOrBuilder p = viaProto ? proto : builder; this.fetches = new ArrayList<>(); - fetches.addAll(p.getFetchesList().stream().collect(Collectors.toList())); - } + p.getFetchesList().forEach(fetch -> fetches.add(fetch)); + } private void maybeInitBuilder() { if (viaProto || builder == null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java index 81316da4f5..e461a07db8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TableCache.java @@ -61,9 +61,8 @@ public synchronized void releaseCache(ExecutionBlockId ebId) { } public synchronized List getCacheKeyByExecutionBlockId(ExecutionBlockId ebId) { - List keys = Lists.newArrayList(); - keys.addAll(cacheMap.keySet().stream().filter(eachKey -> - eachKey.ebId.equals(ebId.toString())).collect(Collectors.toList())); + List keys = cacheMap.keySet().stream().filter(eachKey -> eachKey.ebId.equals(ebId.toString())) + .collect(Collectors.toList()); return keys; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 67a469aed6..d9e8b9305d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -70,7 +70,6 @@ import java.util.List; import java.util.Map; import java.util.Stack; -import java.util.stream.Collectors; public class NonForwardQueryResultSystemScanner implements NonForwardQueryResultScanner { @@ -528,12 +527,10 @@ private List getClusterInfo(Schema outSchema) { } tuples = new ArrayList<>(queryMasterList.size() + nodeStatusList.size()); - tuples.addAll(queryMasterList.stream().map(queryMaster -> getQueryMasterTuple(outSchema, queryMaster)) - .collect(Collectors.toList())); + queryMasterList.stream().map(queryMaster -> getQueryMasterTuple(outSchema, queryMaster)).forEach(tuples::add); + + nodeStatusList.stream().map(nodeStatus -> getWorkerTuple(outSchema, nodeStatus)).forEach(tuples::add); - tuples.addAll(nodeStatusList.stream().map(nodeStatus -> getWorkerTuple(outSchema, nodeStatus)) - .collect(Collectors.toList())); - return tuples; } diff --git a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java index cc57d63154..a15dc613df 100644 --- a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java @@ -1402,8 +1402,8 @@ public PartitionMethodDescExpr parseTablePartitioningClause(Table_partitioning_c } else { // ( PARTITION part_name , ...) List specifiers = Lists.newArrayList(); - specifiers.addAll(hashPartitions.individual_hash_partitions().individual_hash_partition().stream() - .map(partition -> new PartitionSpecifier(partition.partition_name().getText())).collect(Collectors.toList())); + hashPartitions.individual_hash_partitions().individual_hash_partition().stream() + .map(partition -> new PartitionSpecifier(partition.partition_name().getText())).forEach(specifiers::add); return new HashPartition(buildColumnReferenceList(hashPartitions.column_reference_list()), specifiers); } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index d36762b893..0c9cb8972f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -63,8 +63,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; -import java.util.stream.Collectors; import static org.apache.tajo.ResourceProtos.*; @@ -327,9 +325,9 @@ protected LinkedList createTaskRequest(final int incompleteTas masterClientService.reserveNodeResources(callBack.getController(), request.build(), callBack); NodeResourceResponse response = callBack.get(); - taskRequestEvents.addAll(response.getResourceList().stream() + response.getResourceList().stream() .map(resource -> new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId())) - .collect(Collectors.toList())); + .forEach(taskRequestEvents::add); return taskRequestEvents; } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 6168e6a953..909f4ea4d2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -79,7 +79,6 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; -import java.util.stream.Collectors; import static org.apache.tajo.ResourceProtos.*; import static org.apache.tajo.conf.TajoConf.ConfVars; @@ -761,7 +760,7 @@ protected void stopExecutionBlock() { if (!getContext().getQueryContext().getBool(SessionVars.DEBUG_ENABLED)) { List childs = getMasterPlan().getChilds(getId()); - ebIds.addAll(childs.stream().map(executionBlock -> executionBlock.getId().getProto()).collect(Collectors.toList())); + childs.stream().map(executionBlock -> executionBlock.getId().getProto()).forEach(ebIds::add); } StopExecutionBlockRequest.Builder stopRequest = StopExecutionBlockRequest.newBuilder(); @@ -1324,7 +1323,7 @@ private void finalizeShuffleReport(StageShuffleReportEvent event, ShuffleType ty completedShuffleTasks.addAndGet(report.getSucceededTasks()); if (report.getIntermediateEntriesCount() > 0) { - hashShuffleIntermediateEntries.addAll(report.getIntermediateEntriesList().stream().map(IntermediateEntry::new).collect(Collectors.toList())); + report.getIntermediateEntriesList().stream().map(IntermediateEntry::new).forEach(hashShuffleIntermediateEntries::add); } if (completedShuffleTasks.get() >= succeededObjectCount) { diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java index 4cbbfcdc8a..f77cefccc9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; public class QueryHistory implements GsonObject, History { @Expose @@ -141,8 +140,7 @@ public QueryHistoryProto getProto() { List stageHistoryProtos = new ArrayList<>(); if (stageHistories != null) { - stageHistoryProtos.addAll(stageHistories.stream() - .map(eachStage -> (eachStage.getProto())).collect(Collectors.toList())); + stageHistories.stream().map(eachStage -> (eachStage.getProto())).forEach(stageHistoryProtos::add); } builder.addAllStageHistories(stageHistoryProtos); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java index 3804347893..7a6aa54f0a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java @@ -25,13 +25,12 @@ import java.util.Collection; import java.util.List; import java.util.regex.Pattern; -import java.util.stream.Collectors; public class RegexpMetricsFilter implements MetricFilter { List filterPatterns = new ArrayList<>(); public RegexpMetricsFilter(Collection filterExpressions) { - filterPatterns.addAll(filterExpressions.stream().map(Pattern::compile).collect(Collectors.toList())); + filterExpressions.stream().map(Pattern::compile).forEach(filterPatterns::add); } @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java index b49d449fb6..5f190f3605 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java @@ -28,14 +28,11 @@ import org.apache.tajo.querymaster.Task; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; import org.apache.tajo.storage.TupleRange; -import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java index 02d6a6ed44..66b8536b0e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java @@ -68,7 +68,6 @@ import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ExecutorService; -import java.util.stream.Collectors; import static org.apache.tajo.ResourceProtos.*; @@ -383,8 +382,7 @@ private void waitForFetch() throws InterruptedException, IOException { Set broadcastTableNames = new HashSet<>(); List broadcasts = context.getEnforcer().getEnforceProperties(EnforceType.BROADCAST); if (broadcasts != null) { - broadcastTableNames.addAll(broadcasts.stream() - .map(eachBroadcast -> eachBroadcast.getBroadcast().getTableName()).collect(Collectors.toList())); + broadcasts.stream().map(eachBroadcast -> eachBroadcast.getBroadcast().getTableName()).forEach(broadcastTableNames::add); } // localize the fetched data and skip the broadcast table diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java index 3a67134d81..c444133c37 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java @@ -33,7 +33,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; @Path("/functions") public class FunctionsResource { @@ -92,7 +91,7 @@ public Response run(JerseyResourceDelegateContext context) { if (functionDescriptors.size() > 0) { List functionSignature = new ArrayList<>(functionDescriptors.size()); - functionSignature.addAll(functionDescriptors.stream().map(FunctionDesc::getSignature).collect(Collectors.toList())); + functionDescriptors.stream().map(FunctionDesc::getSignature).forEach(functionSignature::add); return Response.ok(functionSignature).build(); } else { diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java index 65bbd55b24..e96e1c864c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java @@ -44,7 +44,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @Path("/queries") public class QueryResource { @@ -175,8 +174,7 @@ public Response run(JerseyResourceDelegateContext context) { private List selectQueriesInfoByState(List queriesInfo, TajoProtos.QueryState state) { List resultQueriesInfo = new ArrayList<>(queriesInfo.size() / 2); - resultQueriesInfo.addAll(queriesInfo.stream() - .filter(queryInfo -> state.equals(queryInfo.getQueryState())).collect(Collectors.toList())); + queriesInfo.stream().filter(queryInfo -> state.equals(queryInfo.getQueryState())).forEach(resultQueriesInfo::add); return resultQueriesInfo; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index 995bb4f036..f453eb9a51 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -43,7 +43,6 @@ import org.apache.tajo.util.graph.DirectedGraphCursor; import java.util.*; -import java.util.stream.Collectors; import static org.apache.tajo.plan.LogicalPlan.BlockEdge; import static org.apache.tajo.plan.joinorder.GreedyHeuristicJoinOrderAlgorithm.getCost; @@ -161,10 +160,8 @@ private static LogicalNode handleRemainingFiltersIfNecessary(JoinGraphContext jo Collection markAsEvaluated = new HashSet<>(joinGraphContext.getEvaluatedJoinConditions()); markAsEvaluated.addAll(joinGraphContext.getEvaluatedJoinFilters()); Set remainingQuals = new HashSet<>(joinGraphContext.getCandidateJoinFilters()); - for (JoinEdge eachEdge : joinEdges) { - remainingQuals.addAll(eachEdge.getJoinQual().stream() - .filter(eachQual -> !markAsEvaluated.contains(eachQual)).collect(Collectors.toList())); - } + joinEdges.forEach(eachEdge -> + eachEdge.getJoinQual().stream().filter(eachQual -> !markAsEvaluated.contains(eachQual)).forEach(remainingQuals::add)); if (!remainingQuals.isEmpty()) { LogicalNode topParent = PlannerUtil.findTopParentNode(block.getRoot(), NodeType.JOIN); @@ -197,7 +194,7 @@ public LogicalNode visitJoin(Set ctx, LogicalPlan plan, LogicalPlan.Quer super.visitJoin(ctx, plan, block, node, stack); if (node.hasTargets()) { - ctx.addAll(node.getTargets().stream().collect(Collectors.toList())); + node.getTargets().forEach(ctx::add); } return node; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index 38be276317..5cab345b15 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -267,11 +267,8 @@ public QueryBlock getBlock(LogicalNode node) { public void removeBlock(QueryBlock block) { queryBlocks.remove(block.getName()); - List tobeRemoved = queryBlockByPID.entrySet().stream() - .map(Map.Entry::getKey).collect(Collectors.toList()); - for (Integer rn : tobeRemoved) { - queryBlockByPID.remove(rn); - } + List tobeRemoved = queryBlockByPID.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList()); + tobeRemoved.forEach(rn -> queryBlockByPID.remove(rn)); } public void disconnectBlocks(QueryBlock srcBlock, QueryBlock targetBlock) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 35506af31f..0decd8cc26 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -60,7 +60,6 @@ import javax.annotation.Nullable; import java.net.URI; import java.util.*; -import java.util.stream.Collectors; import static org.apache.tajo.algebra.CreateTable.PartitionType; import static org.apache.tajo.plan.ExprNormalizer.ExprNormalizedResult; @@ -736,7 +735,7 @@ private LogicalNode insertWindowAggNode(PlanContext context, LogicalNode child, } } } - targets.addAll(winFuncRefs.stream().map(block.namedExprsMgr::getTarget).collect(Collectors.toList())); + winFuncRefs.stream().map(block.namedExprsMgr::getTarget).forEach(targets::add); windowAggNode.setTargets(targets); verifyProjectedFields(block, windowAggNode); @@ -1188,7 +1187,7 @@ public LogicalNode visitJoin(PlanContext context, Stack stack, Join join) List newlyEvaluatedExprs = getNewlyEvaluatedExprsForJoin(context, joinNode, isTopMostJoin); List targets = new ArrayList<>(PlannerUtil.schemaToTargets(merged)); - targets.addAll(newlyEvaluatedExprs.stream().map(newAddedExpr -> block.namedExprsMgr.getTarget(newAddedExpr, true)).collect(Collectors.toList())); + newlyEvaluatedExprs.stream().map(newAddedExpr -> block.namedExprsMgr.getTarget(newAddedExpr, true)).forEach(targets::add); joinNode.setTargets(targets); // Determine join conditions @@ -1282,7 +1281,7 @@ private LogicalNode createCartesianProduct(PlanContext context, LogicalNode left } List targets = new ArrayList<>(PlannerUtil.schemaToTargets(merged)); - targets.addAll(newlyEvaluatedExprs.stream().map(newAddedExpr -> block.namedExprsMgr.getTarget(newAddedExpr, true)).collect(Collectors.toList())); + newlyEvaluatedExprs.stream().map(newAddedExpr -> block.namedExprsMgr.getTarget(newAddedExpr, true)).forEach(targets::add); join.setTargets(targets); return join; } @@ -1420,7 +1419,7 @@ private void setTargetOfTableSubQuery (PlanContext context, QueryBlock block, Ta // Assume that each unique expr is evaluated once. LinkedHashSet targets = createFieldTargetsFromRelation(block, subQueryNode, newlyEvaluatedExprs); - targets.addAll(newlyEvaluatedExprs.stream().map(newAddedExpr -> block.namedExprsMgr.getTarget(newAddedExpr, true)).collect(Collectors.toList())); + newlyEvaluatedExprs.stream().map(newAddedExpr -> block.namedExprsMgr.getTarget(newAddedExpr, true)).forEach(targets::add); subQueryNode.setTargets(new ArrayList<>(targets)); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java index 4f2756371b..b2dda5b96e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java @@ -33,7 +33,6 @@ import org.apache.tajo.util.TUtil; import java.util.*; -import java.util.stream.Collectors; import static org.apache.tajo.catalog.TypeConverter.convert; @@ -224,9 +223,9 @@ public String addNamedExpr(NamedExpr namedExpr) { public Collection getAllNamedExprs() { List namedExprList = Lists.newArrayList(); - namedExprList.addAll(idToExprBiMap.entrySet().stream() + idToExprBiMap.entrySet().stream() .map(entry -> new NamedExpr(entry.getValue(), idToNamesMap.get(entry.getKey()).get(0))) - .collect(Collectors.toList())); + .forEach(namedExprList::add); return namedExprList; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java index fe5362b51f..26ed7c3392 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java @@ -26,7 +26,6 @@ import java.util.Collection; import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; public class JoinGraphContext { private Set rootVertexes = new HashSet<>(); // most left vertex in the join plan @@ -48,13 +47,13 @@ public JoinGraph getJoinGraph() { } public void addCandidateJoinConditions(Collection candidates) { - candidateJoinConditions.addAll(candidates.stream() - .filter(eachCandidate -> !evaluatedJoinConditions.contains(eachCandidate)).collect(Collectors.toList())); + candidates.stream().filter(eachCandidate -> !evaluatedJoinConditions.contains(eachCandidate)) + .forEach(candidateJoinConditions::add); } public void addCandidateJoinFilters(Collection candidates) { - candidateJoinFilters.addAll(candidates.stream() - .filter(eachCandidate -> !evaluatedJoinFilters.contains(eachCandidate)).collect(Collectors.toList())); + candidates.stream().filter(eachCandidate -> !evaluatedJoinFilters.contains(eachCandidate)) + .forEach(candidateJoinFilters::add); } public void removeCandidateJoinConditions(Collection willBeRemoved) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java index d4de9a3db8..4955799b71 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java @@ -33,7 +33,6 @@ import java.util.HashSet; import java.util.Set; import java.util.Stack; -import java.util.stream.Collectors; /** * Condition reduce rule reduces the query predicate based on distributivity. @@ -182,8 +181,8 @@ private EvalNode rewrite(BinaryEval evalNode) { new HashSet<>(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(rightChild))) : new HashSet<>(Arrays.asList(AlgebraicUtil.toDisjunctiveNormalFormArray(rightChild))); - commonQuals.addAll(leftChildSplits.stream() - .filter(eachLeftChildSplit -> rightChildSplits.contains(eachLeftChildSplit)).collect(Collectors.toList())); + leftChildSplits.stream().filter(eachLeftChildSplit -> rightChildSplits.contains(eachLeftChildSplit)) + .forEach(commonQuals::add); if (leftChildSplits.size() == rightChildSplits.size() && commonQuals.size() == leftChildSplits.size()) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index 0241ba9ddd..3a464fbf43 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -214,9 +214,10 @@ public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, Lo boolean isTopMostJoin = stack.isEmpty() ? true : stack.peek().getType() != NodeType.JOIN; // TODO: currently, non-equi theta join is not supported yet. - matched.addAll(context.pushingDownFilters.stream() - .filter(evalNode -> LogicalPlanner.isEvaluatableJoinQual(block, evalNode, joinNode, onPredicates.contains(evalNode), - isTopMostJoin)).collect(Collectors.toList())); + context.pushingDownFilters.stream() + .filter(evalNode -> + LogicalPlanner.isEvaluatableJoinQual(block, evalNode, joinNode, onPredicates.contains(evalNode), isTopMostJoin)) + .forEach(matched::add); EvalNode qual = null; if (matched.size() > 1) { @@ -357,13 +358,13 @@ private static Set extractNonPushableOuterJoinQuals(final LogicalPlan } Set nonPushableQuals = new HashSet<>(); - for (EvalNode eachQual : onPredicates) { - nonPushableQuals.addAll(preservedTableNameSet.stream().filter(relName -> isEvalNeedRelation(eachQual, relName)).map(relName -> eachQual).collect(Collectors.toList())); - } + onPredicates.forEach(eachQual -> + preservedTableNameSet.stream().filter(relName -> isEvalNeedRelation(eachQual, relName)).map(relName -> eachQual) + .forEach(nonPushableQuals::add)); - for (EvalNode eachQual : wherePredicates) { - nonPushableQuals.addAll(nullSupplyingTableNameSet.stream().filter(relName -> isEvalNeedRelation(eachQual, relName)).map(relName -> eachQual).collect(Collectors.toList())); - } + wherePredicates.forEach(eachQual -> + nullSupplyingTableNameSet.stream().filter(relName -> isEvalNeedRelation(eachQual, relName)).map(relName -> eachQual) + .forEach(nonPushableQuals::add)); return nonPushableQuals; } @@ -588,7 +589,7 @@ public LogicalNode visitProjection(FilterPushDownContext context, stack.pop(); // find not matched after visiting child - notMatched.addAll(context.pushingDownFilters.stream().map(transformedMap::get).collect(Collectors.toList())); + context.pushingDownFilters.stream().map(transformedMap::get).forEach(notMatched::add); EvalNode qual = null; if (notMatched.size() > 1) { @@ -920,8 +921,8 @@ public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, Map transformed = findCanPushdownAndTransform(context, block, scanNode, null, notMatched, partitionColumns, 0); - matched.addAll(transformed.keySet().stream() - .filter(eval -> LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, scanNode)).collect(Collectors.toList())); + transformed.keySet().stream() + .filter(eval -> LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, scanNode)).forEach(matched::add); EvalNode qual = null; if (matched.size() > 1) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java index d8a37c3b15..4ea5a8a59d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java @@ -367,8 +367,8 @@ class FilteredTargetIterator implements Iterator { Iterator iterator; public FilteredTargetIterator(Set required) { - filtered.addAll(nameToIdBiMap.keySet().stream() - .filter(name -> required.contains(name)).map(TargetListManager.this::getTarget).collect(Collectors.toList())); + nameToIdBiMap.keySet().stream() + .filter(name -> required.contains(name)).map(TargetListManager.this::getTarget).forEach(filtered::add); iterator = filtered.iterator(); } @@ -430,8 +430,7 @@ public String addExpr(EvalNode evalNode) throws DuplicateColumnException { } private void addNecessaryReferences(EvalNode evalNode) { - requiredSet.addAll(EvalTreeUtil.findUniqueColumns(evalNode).stream() - .map(Column::getQualifiedName).collect(Collectors.toList())); + EvalTreeUtil.findUniqueColumns(evalNode).stream().map(Column::getQualifiedName).forEach(requiredSet::add); } @Override @@ -1008,8 +1007,7 @@ static class FilteredIterator implements Iterator { } } - filtered.addAll(requiredReferences.stream().filter(name -> targetSet.containsKey(name)) - .map(targetSet::get).collect(Collectors.toList())); + requiredReferences.stream().filter(name -> targetSet.containsKey(name)).map(targetSet::get).forEach(filtered::add); iterator = filtered.iterator(); } diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index ec23cdd913..4ac47289e1 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -372,7 +372,7 @@ public static HTableDescriptor parseHTableDescriptor(TableMeta tableMeta, Schema Collection columnFamilies = columnMapping.getColumnFamilyNames(); //If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column. if (columnFamilies.isEmpty()) { - columnFamilies.addAll(schema.getRootColumns().stream().map(Column::getSimpleName).collect(Collectors.toList())); + schema.getRootColumns().stream().map(Column::getSimpleName).forEach(columnFamilies::add); } for (String eachColumnFamily: columnFamilies) { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java index 41df1dd34d..d5e2aa4713 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java @@ -52,7 +52,6 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; -import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; From 572f17c5e6a099305c410005ce82e669263217f7 Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Sat, 30 Jan 2016 02:11:28 +0900 Subject: [PATCH 07/17] using parallelStream --- .../tajo/catalog/store/HiveCatalogStore.java | 2 +- .../store/XMLCatalogSchemaManager.java | 42 ++++++++----------- .../tajo/cli/tsql/CliClientParamsFactory.java | 2 +- .../org/apache/tajo/TajoTestingCluster.java | 2 +- .../engine/planner/TestLogicalPlanner.java | 10 ++--- .../planner/physical/TestPhysicalPlanner.java | 4 +- .../planner/global/ExecutionBlockCursor.java | 2 +- .../engine/planner/global/GlobalPlanner.java | 2 +- .../builder/DistinctGroupbyBuilder.java | 4 +- .../rewriter/rules/BroadcastJoinRule.java | 4 +- .../rewriter/rules/GlobalPlanRewriteUtil.java | 2 +- .../planner/physical/BSTIndexScanExec.java | 4 +- .../DistinctGroupbyFirstAggregationExec.java | 2 +- .../DistinctGroupbyHashAggregationExec.java | 4 +- .../physical/HashShuffleFileWriteExec.java | 2 +- .../org/apache/tajo/master/QueryManager.java | 2 +- .../ExplainGlobalPlanPreprocessorForTest.java | 2 +- .../tajo/master/exec/QueryExecutor.java | 2 +- .../apache/tajo/parser/sql/SQLAnalyzer.java | 5 +-- .../querymaster/DefaultTaskScheduler.java | 2 +- .../org/apache/tajo/querymaster/Query.java | 2 +- .../apache/tajo/querymaster/QueryMaster.java | 2 +- .../org/apache/tajo/querymaster/Task.java | 8 ++-- .../tajo/util/metrics/TajoSystemMetrics.java | 2 +- .../tajo/webapp/QueryExecutorServlet.java | 4 +- .../tajo/worker/ExecutionBlockContext.java | 3 +- .../org/apache/tajo/worker/TaskExecutor.java | 2 +- .../java/org/apache/tajo/worker/TaskImpl.java | 2 +- .../org/apache/tajo/worker/TaskManager.java | 4 +- .../org/apache/tajo/plan/LogicalPlan.java | 2 +- .../tajo/plan/joinorder/JoinGraphContext.java | 4 +- .../tajo/plan/joinorder/JoinOrderingUtil.java | 5 ++- .../tajo/plan/nameresolver/NameResolver.java | 2 +- .../rules/CommonConditionReduceRule.java | 2 +- .../rewrite/rules/FilterPushDownRule.java | 22 +++++----- .../rewrite/rules/ProjectionPushDownRule.java | 2 +- .../plan/verifier/PreLogicalPlanVerifier.java | 2 +- .../tajo/storage/OldStorageManager.java | 2 +- .../tajo/storage/hbase/HBaseTablespace.java | 2 +- .../storage/HashShuffleAppenderManager.java | 2 +- .../storage/thirdparty/orc/WriterImpl.java | 2 +- 41 files changed, 84 insertions(+), 95 deletions(-) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 7198a9f175..5f5f2ff4b4 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -690,7 +690,7 @@ private void renameColumn(String databaseName, String tableName, CatalogProtos.A Table table = client.getHiveClient().getTable(databaseName, tableName); List columns = table.getSd().getCols(); - columns.stream().filter(currentColumn -> currentColumn.getName() + columns.parallelStream().filter(currentColumn -> currentColumn.getName() .equalsIgnoreCase(alterColumnProto.getOldColumnName())).forEach(currentColumn -> { currentColumn.setName(alterColumnProto.getNewColumnName()); }); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index d8feb52908..2e9b610b42 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -631,30 +631,22 @@ protected void validatePatch(List patches, SchemaPatch testPatch) { protected void mergePatches(List patches) { final List objects = new ArrayList<>(); - - Collections.sort(patches); - - for (SchemaPatch patch: patches) { + + patches.stream().forEachOrdered(patch -> { validatePatch(patches, patch); - + objects.clear(); List tempObjects = new ArrayList<>(); tempObjects.addAll(patch.getObjects()); patch.clearObjects(); - patch.addObjects(mergeDatabaseObjects(tempObjects)); - + patch.addObjects(mergeDatabaseObjects(tempObjects)); + targetStore.addPatch(patch); - } + }); } protected void validateSQLObject(List queries, SQLObject testQuery) { - int occurredCount = 0; - - for (SQLObject query: queries) { - if (query.getType() == testQuery.getType()) { - occurredCount++; - } - } + int occurredCount = (int) queries.parallelStream().filter(query -> query.getType() == testQuery.getType()).count(); if (occurredCount > 1) { throw new TajoInternalError("Duplicate Query type (" + testQuery.getType() + ") has found."); @@ -662,38 +654,38 @@ protected void validateSQLObject(List queries, SQLObject testQuery) { } protected void mergeExistQueries(List queries) { - for (SQLObject query: queries) { + queries.forEach(query -> { validateSQLObject(queries, query); - + targetStore.addExistQuery(query); - } + }); } protected void mergeDropStatements(List queries) { - for (SQLObject query: queries) { + queries.forEach(query -> { validateSQLObject(queries, query); - + targetStore.addDropStatement(query); - } + }); } public StoreObject merge() { boolean alreadySetDatabaseObject = false; // first pass - this.storeObjects.forEach(this::copySchemaInfo); + this.storeObjects.parallelStream().forEach(this::copySchemaInfo); // second pass for (StoreObject store: this.storeObjects) { - if (store.getSchema().getVersion() == targetStore.getSchema().getVersion() && + if (store.getSchema().getVersion() == targetStore.getSchema().getVersion() && !alreadySetDatabaseObject) { BaseSchema targetSchema = targetStore.getSchema(); targetSchema.clearObjects(); targetSchema.addObjects(mergeDatabaseObjects(store.getSchema().getObjects())); - + alreadySetDatabaseObject = true; } - + mergePatches(store.getPatches()); mergeExistQueries(store.getExistQueries()); mergeDropStatements(store.getDropStatements()); diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java index c6b997fc0c..c8a4b5ae89 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java @@ -41,7 +41,7 @@ class CliClientParamsFactory { public static Properties get(@Nullable Properties connParam) { Properties copy = connParam == null ? new Properties() : (Properties) connParam.clone(); - DEFAULT_PARAMS.entrySet().stream().filter(entry -> !copy.contains(entry.getKey())).forEach(entry -> { + DEFAULT_PARAMS.entrySet().parallelStream().filter(entry -> !copy.contains(entry.getKey())).forEach(entry -> { copy.setProperty(entry.getKey(), entry.getValue()); }); return copy; diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index d43c84470c..54b8b51f26 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -471,7 +471,7 @@ public void shutdownMiniTajoCluster() { if(this.tajoMaster != null) { this.tajoMaster.stop(); } - tajoWorkers.forEach(TajoWorker::stopWorkerForce); + tajoWorkers.parallelStream().forEach(TajoWorker::stopWorkerForce); tajoWorkers.clear(); this.tajoMaster= null; } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index d8b5a8af59..18c32c6389 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -526,7 +526,7 @@ public final void testJoinWithMultipleJoinQual1() throws IOException, TajoExcept } } - qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + qualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); @@ -564,7 +564,7 @@ public final void testJoinWithMultipleJoinQual2() throws IOException, TajoExcept } } - qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + qualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "SelectionQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); @@ -607,7 +607,7 @@ public final void testJoinWithMultipleJoinQual3() throws IOException, TajoExcept } } - qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + qualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); @@ -674,12 +674,12 @@ public final void testJoinWithMultipleJoinQual4() throws IOException, TajoExcept } - joinQualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + joinQualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); - scanMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + scanMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index c4fb292ecd..7cfa0b2d09 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -844,7 +844,7 @@ public final void testAggregationFunction() throws IOException, TajoException { // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); - groupbyNode.getAggFunctions().forEach(AggregationFunctionCallEval::setFirstPhase); + groupbyNode.getAggFunctions().parallelStream().forEach(AggregationFunctionCallEval::setFirstPhase); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); @@ -873,7 +873,7 @@ public final void testCountFunction() throws IOException, TajoException { // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); - groupbyNode.getAggFunctions().forEach(AggregationFunctionCallEval::setFirstPhase); + groupbyNode.getAggFunctions().parallelStream().forEach(AggregationFunctionCallEval::setFirstPhase); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java index 0b45ba532f..ba17fa6338 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java @@ -103,7 +103,7 @@ public String toString() { // Add all execution blocks in a depth first and postfix order private void buildDepthFirstOrder(ExecutionBlock current) { if (!masterPlan.isLeaf(current.getId())) { - masterPlan.getChilds(current).forEach(this::buildDepthFirstOrder); + masterPlan.getChilds(current).parallelStream().forEach(this::buildDepthFirstOrder); } orderedBlocks.add(current); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 7d1ca79787..acffd96ea2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -550,7 +550,7 @@ private ExecutionBlock buildGroupByIncludingDistinctFunctionsMultiStage(GlobalPl List firstStageTargets = firstStageGroupingColumns.stream() .map(column -> new Target(new FieldEval(column))).collect(Collectors.toList()); - firstPhaseEvalNodeTargets.stream().collect(Collectors.toList()).forEach(firstStageTargets::add); + firstPhaseEvalNodeTargets.forEach(firstStageTargets::add); // Create the groupby node for the first stage and set all necessary descriptions GroupbyNode firstStageGroupby = new GroupbyNode(context.plan.getLogicalPlan().newPID()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index 122600e572..6665fdc750 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -226,9 +226,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou distinctNodeBuildInfos.put(groupbyMapKey, buildInfo); // Grouping columns are GROUP BY clause's column + Distinct column. - List groupingColumns = new ArrayList<>(); - groupbyUniqColumns.stream().filter(eachGroupingColumn -> - !groupingColumns.contains(eachGroupingColumn)).forEach(groupingColumns::add); + List groupingColumns = groupbyUniqColumns.stream().distinct().collect(Collectors.toList()); distinctGroupbyNode.setGroupingColumns(groupingColumns.toArray(new Column[groupingColumns.size()])); } buildInfo.addAggFunction(aggFunction); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java index 4da6701699..020896c7d6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java @@ -224,7 +224,7 @@ private void visitNonLeafNode(Context context, ExecutionBlock current) throws Ta } JoinType joinType = ((JoinNode)found).getJoinType(); - childs.stream().filter(child -> !child.isPreservedRow()).forEach(child -> { + childs.parallelStream().filter(child -> !child.isPreservedRow()).forEach(child -> { updateBroadcastableRelForChildEb(context, child, joinType); updateInputBasedOnChildEb(child, current); }); @@ -243,7 +243,7 @@ private void visitNonLeafNode(Context context, ExecutionBlock current) throws Ta } } else { List relations = new ArrayList<>(current.getBroadcastRelations()); - relations.forEach(current::removeBroadcastRelation); + relations.parallelStream().forEach(current::removeBroadcastRelation); } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java index 566d417f64..f0da19cea2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java @@ -36,7 +36,7 @@ public class GlobalPlanRewriteUtil { * @return */ public static ExecutionBlock mergeExecutionBlocks(MasterPlan plan, ExecutionBlock child, ExecutionBlock parent) { - child.getBroadcastRelations().forEach(parent::addBroadcastRelation); + child.getBroadcastRelations().parallelStream().forEach(parent::addBroadcastRelation); // connect parent and grand children List grandChilds = plan.getChilds(child); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index bb9da9695f..29fbb32bb5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@ -94,9 +94,7 @@ public BSTIndexScanExec(TaskAttemptContext context, IndexScanNode plan, private static Schema mergeSubSchemas(Schema originalSchema, Schema subSchema, List targets, EvalNode qual) { Set qualAndTargets = new HashSet<>(); qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(qual)); - for (Target target : targets) { - qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(target.getEvalTree())); - } + targets.forEach(target -> qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(target.getEvalTree()))); SchemaBuilder mergedSchema = SchemaBuilder.builder(); originalSchema.getRootColumns().stream() diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java index a0bf345c9c..c3e9059d68 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java @@ -363,7 +363,7 @@ public void rescan() { } public void close() throws IOException { - distinctAggrDatas.values().forEach(TupleSet::clear); + distinctAggrDatas.values().parallelStream().forEach(TupleSet::clear); distinctAggrDatas.clear(); distinctAggrDatas = null; currentGroupingTuples = null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java index 05b58a8476..6b872c2b0f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java @@ -261,7 +261,7 @@ public Tuple next() throws IOException { listIndex++; } - tupleSlots.forEach(TupleList::clear); + tupleSlots.parallelStream().forEach(TupleList::clear); tupleSlots.clear(); currentAggregatedTupleIndex = 0; @@ -438,7 +438,7 @@ public TupleList aggregate(Map groupTuples) { } public void close() throws IOException { - hashTable.values().forEach(TupleMap::clear); + hashTable.values().parallelStream().forEach(TupleMap::clear); hashTable.clear(); hashTable = null; iterator = null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java index 3cd9671ed9..549b2809e2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java @@ -234,7 +234,7 @@ public void rescan() throws IOException { @Override public void close() throws IOException{ if (partitionMemoryMap.size() > 0) { - partitionMemoryMap.values().forEach(RowBlock::release); + partitionMemoryMap.values().parallelStream().forEach(RowBlock::release); partitionMemoryMap.clear(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java index 8f80b825dd..87ff8cadd5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java @@ -95,7 +95,7 @@ public void serviceInit(Configuration conf) throws Exception { @Override public void serviceStop() throws Exception { - runningQueries.values().forEach(QueryInProgress::stopProgress); + runningQueries.values().parallelStream().forEach(QueryInProgress::stopProgress); super.serviceStop(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java index 1ec99f5a1c..e8f271634a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java @@ -47,7 +47,7 @@ public void prepareTest(MasterPlan plan) { for (ExecutionBlock block : cursor) { List outgoingChannels = plan.getOutgoingChannels(block.getId()); if (outgoingChannels != null) { - outgoingChannels.stream().filter(channel -> channel.hasShuffleKeys()).forEach(channel -> { + outgoingChannels.parallelStream().filter(channel -> channel.hasShuffleKeys()).forEach(channel -> { Column[] shuffleKeys = channel.getShuffleKeys(); Arrays.sort(shuffleKeys, columnComparator); channel.setShuffleKeys(shuffleKeys); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index d7a08aec27..5d93b1f856 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -402,7 +402,7 @@ public static void startScriptExecutors(QueryContext queryContext, EvalContext e } public static void stopScriptExecutors(EvalContext evalContext) { - evalContext.getAllScriptEngines().forEach(TajoScriptEngine::shutdown); + evalContext.getAllScriptEngines().parallelStream().forEach(TajoScriptEngine::shutdown); } /** diff --git a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java index a15dc613df..6a9fdcc958 100644 --- a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java @@ -1401,9 +1401,8 @@ public PartitionMethodDescExpr parseTablePartitioningClause(Table_partitioning_c visitNumeric_value_expression(hashPartitions.hash_partitions_by_quantity().quantity)); } else { // ( PARTITION part_name , ...) - List specifiers = Lists.newArrayList(); - hashPartitions.individual_hash_partitions().individual_hash_partition().stream() - .map(partition -> new PartitionSpecifier(partition.partition_name().getText())).forEach(specifiers::add); + List specifiers = hashPartitions.individual_hash_partitions().individual_hash_partition() + .stream().map(partition -> new PartitionSpecifier(partition.partition_name().getText())).collect(Collectors.toList()); return new HashPartition(buildColumnReferenceList(hashPartitions.column_reference_list()), specifiers); } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index 0c9cb8972f..0c8df14256 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -288,7 +288,7 @@ private Set getWorkerIds(Collection hosts){ Set workerIds = Sets.newHashSet(); if(hosts.isEmpty()) return workerIds; - stage.getContext().getWorkerMap().values().stream().filter(worker -> hosts.contains(worker.getHost())) + stage.getContext().getWorkerMap().values().parallelStream().filter(worker -> hosts.contains(worker.getHost())) .forEach(worker -> workerIds.add(worker.getId())); return workerIds; } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 4f8f128197..dccb577f7b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -339,7 +339,7 @@ public List getPartitions() { } public void clearPartitions() { - getStages().forEach(Stage::clearPartitions); + getStages().parallelStream().forEach(Stage::clearPartitions); } public SerializedException getFailureReason() { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index ddefc19ab9..f7921366ff 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -447,7 +447,7 @@ public void run() { List tempTasks = new ArrayList<>(); tempTasks.addAll(queryMasterTasks.values()); - tempTasks.stream().filter(eachTask -> !eachTask.isStopped()).forEach(eachTask -> { + tempTasks.parallelStream().filter(eachTask -> !eachTask.isStopped()).forEach(eachTask -> { try { long lastHeartbeat = eachTask.getLastClientHeartbeat(); long time = System.currentTimeMillis() - lastHeartbeat; diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java index d0ed4af1e1..6e499ab775 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java @@ -782,11 +782,13 @@ public IntermediateEntry(IntermediateEntryProto proto) { this.volume = proto.getVolume(); failureRowNums = new ArrayList<>(); - proto.getFailuresList().stream().map(eachFailure -> new Pair(eachFailure.getPagePos(), - new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum()))).forEach(eachFailure -> failureRowNums.add(eachFailure)); + proto.getFailuresList().stream().map(eachFailure -> + new Pair(eachFailure.getPagePos(), new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum()))) + .forEach(eachFailure -> failureRowNums.add(eachFailure)); pages = new ArrayList<>(); - proto.getPagesList().stream().map(eachPage -> new Pair(eachPage.getPos(), eachPage.getLength())).forEach(eachPage -> pages.add(eachPage)); + proto.getPagesList().stream().map(eachPage -> new Pair(eachPage.getPos(), eachPage.getLength())) + .forEach(eachPage -> pages.add(eachPage)); } public IntermediateEntry(int taskId, int attemptId, int partId, PullHost host) { diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java index c255f9a176..b5a3384609 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java @@ -104,7 +104,7 @@ public void stop() { protected void stopAndClearReporter() { synchronized(metricsReporters) { - metricsReporters.forEach(TajoMetricsScheduledReporter::close); + metricsReporters.parallelStream().forEach(TajoMetricsScheduledReporter::close); metricsReporters.clear(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java index 2863a1d2e3..8e0fcebdf5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java @@ -210,7 +210,7 @@ public void service(HttpServletRequest request, } } else if("clearAllQueryRunner".equals(action)) { synchronized(queryRunners) { - queryRunners.values().forEach(QueryRunner::setStop); + queryRunners.values().parallelStream().forEach(QueryRunner::setStop); queryRunners.clear(); } } else if("killQuery".equals(action)) { @@ -265,7 +265,7 @@ public void run() { List queryRunnerList; synchronized(queryRunners) { queryRunnerList = new ArrayList<>(queryRunners.values()); - queryRunnerList.stream().filter(eachQueryRunner -> !eachQueryRunner.running.get() && + queryRunnerList.parallelStream().filter(eachQueryRunner -> !eachQueryRunner.running.get() && (System.currentTimeMillis() - eachQueryRunner.finishTime > 180 * 1000)).forEach(eachQueryRunner -> { queryRunners.remove(eachQueryRunner.queryRunnerId); }); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java index 3a6f6e1dcf..4e0b5476c0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java @@ -187,9 +187,8 @@ public void stop(){ } // If ExecutionBlock is stopped, all running or pending tasks will be marked as failed. - tasks.values().stream().filter(task -> task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_PENDING || + tasks.values().parallelStream().filter(task -> task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_PENDING || task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_RUNNING).forEach(task -> { - try { task.abort(); } catch (Throwable e) { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java index ff748f5542..6fed6b42be 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java @@ -102,7 +102,7 @@ protected void serviceStop() throws Exception { isStopped = true; threadPool.shutdown(); - fetcherThreadPoolList.forEach(ExecutorService::shutdown); + fetcherThreadPoolList.parallelStream().forEach(ExecutorService::shutdown); super.serviceStop(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java index 66b8536b0e..e7f4899016 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java @@ -190,7 +190,7 @@ private void startScriptExecutors() throws IOException { } private void stopScriptExecutors() { - context.getEvalContext().getAllScriptEngines().forEach(TajoScriptEngine::shutdown); + context.getEvalContext().getAllScriptEngines().parallelStream().forEach(TajoScriptEngine::shutdown); } @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java index 237ea71097..7e91604e40 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java @@ -91,7 +91,7 @@ protected void serviceInit(Configuration conf) throws Exception { @Override protected void serviceStop() throws Exception { - executionBlockContextMap.values().forEach(ExecutionBlockContext::stop); + executionBlockContextMap.values().parallelStream().forEach(ExecutionBlockContext::stop); super.serviceStop(); } @@ -214,7 +214,7 @@ public void handle(TaskManagerEvent event) { QueryStopEvent queryStopEvent = TUtil.checkTypeAndGet(event, QueryStopEvent.class); //cleanup failure ExecutionBlock - executionBlockContextMap.keySet().stream() + executionBlockContextMap.keySet().parallelStream() .filter(ebId -> ebId.getQueryId().equals(queryStopEvent.getQueryId())).forEach(ebId -> { try { executionBlockContextMap.remove(ebId).stop(); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index 5cab345b15..f2a068f6f1 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -268,7 +268,7 @@ public QueryBlock getBlock(LogicalNode node) { public void removeBlock(QueryBlock block) { queryBlocks.remove(block.getName()); List tobeRemoved = queryBlockByPID.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList()); - tobeRemoved.forEach(rn -> queryBlockByPID.remove(rn)); + tobeRemoved.parallelStream().forEach(rn -> queryBlockByPID.remove(rn)); } public void disconnectBlocks(QueryBlock srcBlock, QueryBlock targetBlock) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java index 26ed7c3392..1e92bf050d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java @@ -47,12 +47,12 @@ public JoinGraph getJoinGraph() { } public void addCandidateJoinConditions(Collection candidates) { - candidates.stream().filter(eachCandidate -> !evaluatedJoinConditions.contains(eachCandidate)) + candidates.parallelStream().filter(eachCandidate -> !evaluatedJoinConditions.contains(eachCandidate)) .forEach(candidateJoinConditions::add); } public void addCandidateJoinFilters(Collection candidates) { - candidates.stream().filter(eachCandidate -> !evaluatedJoinFilters.contains(eachCandidate)) + candidates.parallelStream().filter(eachCandidate -> !evaluatedJoinFilters.contains(eachCandidate)) .forEach(candidateJoinFilters::add); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java index 45f38dcbd7..e117c0d9e5 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java @@ -43,8 +43,9 @@ public class JoinOrderingUtil { */ public static Set findJoinConditionForJoinVertex(Set candidates, JoinEdge edge, boolean isOnPredicates) { - Set conditionsForThisJoin = candidates.stream().filter(predicate -> EvalTreeUtil.isJoinQual(predicate, false) - && checkIfEvaluatedAtEdge(predicate, edge, isOnPredicates)).collect(Collectors.toSet()); + Set conditionsForThisJoin = candidates.stream().filter(predicate -> + EvalTreeUtil.isJoinQual(predicate, false) && checkIfEvaluatedAtEdge(predicate, edge, isOnPredicates)) + .collect(Collectors.toSet()); return conditionsForThisJoin; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java index 0471ca4715..efd89bcfd9 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java @@ -126,7 +126,7 @@ public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tabl */ public static Collection lookupTableByColumns(LogicalPlan.QueryBlock block, String columnName) { - Set found = block.getRelations().stream() + Set found = block.getRelations().parallelStream() .filter(rel -> rel.getLogicalSchema().contains(columnName)).collect(Collectors.toSet()); return found; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java index 4955799b71..cd3dadc88d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java @@ -181,7 +181,7 @@ private EvalNode rewrite(BinaryEval evalNode) { new HashSet<>(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(rightChild))) : new HashSet<>(Arrays.asList(AlgebraicUtil.toDisjunctiveNormalFormArray(rightChild))); - leftChildSplits.stream().filter(eachLeftChildSplit -> rightChildSplits.contains(eachLeftChildSplit)) + leftChildSplits.parallelStream().filter(eachLeftChildSplit -> rightChildSplits.contains(eachLeftChildSplit)) .forEach(commonQuals::add); if (leftChildSplits.size() == rightChildSplits.size() && diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index 3a464fbf43..f3f92f0a00 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -151,7 +151,7 @@ public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, } else { // if there remain search conditions // check if it can be evaluated here - Set matched = context.pushingDownFilters.stream() + Set matched = context.pushingDownFilters.parallelStream() .filter(eachEval -> LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)).collect(Collectors.toSet()); // if there are search conditions which can be evaluated here, @@ -358,13 +358,13 @@ private static Set extractNonPushableOuterJoinQuals(final LogicalPlan } Set nonPushableQuals = new HashSet<>(); - onPredicates.forEach(eachQual -> - preservedTableNameSet.stream().filter(relName -> isEvalNeedRelation(eachQual, relName)).map(relName -> eachQual) - .forEach(nonPushableQuals::add)); + onPredicates.parallelStream().forEach(eachQual -> + preservedTableNameSet.parallelStream().filter(relName -> isEvalNeedRelation(eachQual, relName)) + .map(relName -> eachQual).forEach(nonPushableQuals::add)); - wherePredicates.forEach(eachQual -> - nullSupplyingTableNameSet.stream().filter(relName -> isEvalNeedRelation(eachQual, relName)).map(relName -> eachQual) - .forEach(nonPushableQuals::add)); + wherePredicates.parallelStream().forEach(eachQual -> + nullSupplyingTableNameSet.parallelStream().filter(relName -> isEvalNeedRelation(eachQual, relName)) + .map(relName -> eachQual).forEach(nonPushableQuals::add)); return nonPushableQuals; } @@ -456,7 +456,7 @@ private Map transformEvalsWidthByPassNode( } Set columns = EvalTreeUtil.findUniqueColumns(copy); - columns.stream().filter(c -> c.hasQualifier()).forEach(c -> { + columns.parallelStream().filter(c -> c.hasQualifier()).forEach(c -> { EvalTreeUtil.changeColumnRef(copy, c.getQualifiedName(), c.getSimpleName()); }); @@ -626,7 +626,7 @@ public LogicalNode visitProjection(FilterPushDownContext context, } private Collection reverseTransform(BiMap map, Set remainFilters) { - Set reversed = remainFilters.stream().map(map::get).collect(Collectors.toSet()); + Set reversed = remainFilters.parallelStream().map(map::get).collect(Collectors.toSet()); return reversed; } @@ -761,7 +761,7 @@ private List addHavingNode(FilterPushDownContext context, LogicalPlan GroupbyNode groupByNode) throws TajoException { // find aggregation column Set groupingColumns = new HashSet<>(Arrays.asList(groupByNode.getGroupingColumns())); - Set aggrFunctionOutColumns = groupByNode.getOutSchema().getRootColumns().stream() + Set aggrFunctionOutColumns = groupByNode.getOutSchema().getRootColumns().parallelStream() .filter(column -> !groupingColumns.contains(column)).map(Column::getQualifiedName).collect(Collectors.toSet()); List aggrEvalOrigins = new ArrayList<>(); @@ -974,7 +974,7 @@ public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, } } - matched.forEach(transformed::remove); + matched.parallelStream().forEach(transformed::remove); context.setToOrigin(transformed); context.addFiltersTobePushed(notMatched); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java index 4ea5a8a59d..235147fda7 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java @@ -430,7 +430,7 @@ public String addExpr(EvalNode evalNode) throws DuplicateColumnException { } private void addNecessaryReferences(EvalNode evalNode) { - EvalTreeUtil.findUniqueColumns(evalNode).stream().map(Column::getQualifiedName).forEach(requiredSet::add); + EvalTreeUtil.findUniqueColumns(evalNode).parallelStream().map(Column::getQualifiedName).forEach(requiredSet::add); } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 39d5900397..40a19347b7 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -86,7 +86,7 @@ public Expr visitProjection(Context context, Stack stack, Projection expr) Set names = new HashSet<>(); - expr.getNamedExprs().stream().filter(namedExpr -> namedExpr.hasAlias()).forEach(namedExpr -> { + expr.getNamedExprs().parallelStream().filter(namedExpr -> namedExpr.hasAlias()).forEach(namedExpr -> { if (names.contains(namedExpr.getAlias())) { context.state.addVerification(SyntaxErrorUtil.makeDuplicateAlias(namedExpr.getAlias())); } else { diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java index 96b9c9d5ff..0dafc2580a 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java @@ -95,7 +95,7 @@ protected synchronized static void clearCache() { */ public static void shutdown() throws IOException { synchronized(storageManagers) { - storageManagers.values().forEach(Tablespace::close); + storageManagers.values().parallelStream().forEach(Tablespace::close); } clearCache(); } diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index 4ac47289e1..393537888f 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -372,7 +372,7 @@ public static HTableDescriptor parseHTableDescriptor(TableMeta tableMeta, Schema Collection columnFamilies = columnMapping.getColumnFamilyNames(); //If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column. if (columnFamilies.isEmpty()) { - schema.getRootColumns().stream().map(Column::getSimpleName).forEach(columnFamilies::add); + schema.getRootColumns().parallelStream().map(Column::getSimpleName).forEach(columnFamilies::add); } for (String eachColumnFamily: columnFamilies) { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index 575c1e53d0..70b3524df7 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -220,7 +220,7 @@ public MemoryRowBlock call() throws Exception { } public void shutdown() { - executors.values().forEach(ExecutorService::shutdownNow); + executors.values().parallelStream().forEach(ExecutorService::shutdownNow); } public static class HashShuffleIntermediate { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java index d5e2aa4713..f6fb3b717d 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java @@ -1989,7 +1989,7 @@ private int writeFooter(long bodyLength) throws IOException { // serialize the types writeTypes(builder, schema); // add the stripe information - stripes.forEach(builder::addStripes); + stripes.parallelStream().forEach(builder::addStripes); // add the column statistics writeFileStatistics(builder, treeWriter); // add all of the user metadata From ee94df40fe7f56ff3546b1263485ea10bef9f4f3 Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Sun, 31 Jan 2016 00:33:43 +0900 Subject: [PATCH 08/17] remove parallelstream partially --- .../org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java | 2 +- .../engine/planner/global/rewriter/rules/BroadcastJoinRule.java | 2 +- .../planner/global/rewriter/rules/GlobalPlanRewriteUtil.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index 2e9b610b42..23ddac7b34 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -673,7 +673,7 @@ public StoreObject merge() { boolean alreadySetDatabaseObject = false; // first pass - this.storeObjects.parallelStream().forEach(this::copySchemaInfo); + this.storeObjects.forEach(this::copySchemaInfo); // second pass for (StoreObject store: this.storeObjects) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java index 020896c7d6..c78bfe4b58 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java @@ -224,7 +224,7 @@ private void visitNonLeafNode(Context context, ExecutionBlock current) throws Ta } JoinType joinType = ((JoinNode)found).getJoinType(); - childs.parallelStream().filter(child -> !child.isPreservedRow()).forEach(child -> { + childs.stream().filter(child -> !child.isPreservedRow()).forEach(child -> { updateBroadcastableRelForChildEb(context, child, joinType); updateInputBasedOnChildEb(child, current); }); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java index f0da19cea2..566d417f64 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java @@ -36,7 +36,7 @@ public class GlobalPlanRewriteUtil { * @return */ public static ExecutionBlock mergeExecutionBlocks(MasterPlan plan, ExecutionBlock child, ExecutionBlock parent) { - child.getBroadcastRelations().parallelStream().forEach(parent::addBroadcastRelation); + child.getBroadcastRelations().forEach(parent::addBroadcastRelation); // connect parent and grand children List grandChilds = plan.getChilds(child); From 9fc2f7cca453876c76964effaf161fc94f722d70 Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Sun, 31 Jan 2016 00:56:31 +0900 Subject: [PATCH 09/17] add parallelstream --- .../org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index 23ddac7b34..2e9b610b42 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -673,7 +673,7 @@ public StoreObject merge() { boolean alreadySetDatabaseObject = false; // first pass - this.storeObjects.forEach(this::copySchemaInfo); + this.storeObjects.parallelStream().forEach(this::copySchemaInfo); // second pass for (StoreObject store: this.storeObjects) { From b24170a66163124b48f95e33f8f9673922a8d5cc Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Sun, 31 Jan 2016 01:26:08 +0900 Subject: [PATCH 10/17] remove parallelstream --- .../org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index 2e9b610b42..23ddac7b34 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -673,7 +673,7 @@ public StoreObject merge() { boolean alreadySetDatabaseObject = false; // first pass - this.storeObjects.parallelStream().forEach(this::copySchemaInfo); + this.storeObjects.forEach(this::copySchemaInfo); // second pass for (StoreObject store: this.storeObjects) { From 69d7daa4085796dad2772f971d24a27f8da3864b Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Wed, 3 Feb 2016 13:54:11 +0900 Subject: [PATCH 11/17] remove parallelStream more --- .../global/rewriter/rules/BroadcastJoinRule.java | 2 +- .../ExplainGlobalPlanPreprocessorForTest.java | 2 +- .../tajo/plan/joinorder/JoinGraphContext.java | 4 ++-- .../tajo/plan/nameresolver/NameResolver.java | 2 +- .../rewrite/rules/CommonConditionReduceRule.java | 2 +- .../plan/rewrite/rules/FilterPushDownRule.java | 16 ++++++++-------- .../rewrite/rules/ProjectionPushDownRule.java | 2 +- .../plan/verifier/PreLogicalPlanVerifier.java | 2 +- .../tajo/storage/hbase/HBaseTablespace.java | 2 +- .../tajo/storage/thirdparty/orc/WriterImpl.java | 2 +- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java index c78bfe4b58..4da6701699 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/BroadcastJoinRule.java @@ -243,7 +243,7 @@ private void visitNonLeafNode(Context context, ExecutionBlock current) throws Ta } } else { List relations = new ArrayList<>(current.getBroadcastRelations()); - relations.parallelStream().forEach(current::removeBroadcastRelation); + relations.forEach(current::removeBroadcastRelation); } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java index e8f271634a..1ec99f5a1c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainGlobalPlanPreprocessorForTest.java @@ -47,7 +47,7 @@ public void prepareTest(MasterPlan plan) { for (ExecutionBlock block : cursor) { List outgoingChannels = plan.getOutgoingChannels(block.getId()); if (outgoingChannels != null) { - outgoingChannels.parallelStream().filter(channel -> channel.hasShuffleKeys()).forEach(channel -> { + outgoingChannels.stream().filter(channel -> channel.hasShuffleKeys()).forEach(channel -> { Column[] shuffleKeys = channel.getShuffleKeys(); Arrays.sort(shuffleKeys, columnComparator); channel.setShuffleKeys(shuffleKeys); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java index 1e92bf050d..26ed7c3392 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java @@ -47,12 +47,12 @@ public JoinGraph getJoinGraph() { } public void addCandidateJoinConditions(Collection candidates) { - candidates.parallelStream().filter(eachCandidate -> !evaluatedJoinConditions.contains(eachCandidate)) + candidates.stream().filter(eachCandidate -> !evaluatedJoinConditions.contains(eachCandidate)) .forEach(candidateJoinConditions::add); } public void addCandidateJoinFilters(Collection candidates) { - candidates.parallelStream().filter(eachCandidate -> !evaluatedJoinFilters.contains(eachCandidate)) + candidates.stream().filter(eachCandidate -> !evaluatedJoinFilters.contains(eachCandidate)) .forEach(candidateJoinFilters::add); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java index efd89bcfd9..0471ca4715 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java @@ -126,7 +126,7 @@ public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tabl */ public static Collection lookupTableByColumns(LogicalPlan.QueryBlock block, String columnName) { - Set found = block.getRelations().parallelStream() + Set found = block.getRelations().stream() .filter(rel -> rel.getLogicalSchema().contains(columnName)).collect(Collectors.toSet()); return found; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java index cd3dadc88d..4955799b71 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java @@ -181,7 +181,7 @@ private EvalNode rewrite(BinaryEval evalNode) { new HashSet<>(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(rightChild))) : new HashSet<>(Arrays.asList(AlgebraicUtil.toDisjunctiveNormalFormArray(rightChild))); - leftChildSplits.parallelStream().filter(eachLeftChildSplit -> rightChildSplits.contains(eachLeftChildSplit)) + leftChildSplits.stream().filter(eachLeftChildSplit -> rightChildSplits.contains(eachLeftChildSplit)) .forEach(commonQuals::add); if (leftChildSplits.size() == rightChildSplits.size() && diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index f3f92f0a00..5954a6f797 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -151,7 +151,7 @@ public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, } else { // if there remain search conditions // check if it can be evaluated here - Set matched = context.pushingDownFilters.parallelStream() + Set matched = context.pushingDownFilters.stream() .filter(eachEval -> LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)).collect(Collectors.toSet()); // if there are search conditions which can be evaluated here, @@ -358,12 +358,12 @@ private static Set extractNonPushableOuterJoinQuals(final LogicalPlan } Set nonPushableQuals = new HashSet<>(); - onPredicates.parallelStream().forEach(eachQual -> - preservedTableNameSet.parallelStream().filter(relName -> isEvalNeedRelation(eachQual, relName)) + onPredicates.forEach(eachQual -> + preservedTableNameSet.stream().filter(relName -> isEvalNeedRelation(eachQual, relName)) .map(relName -> eachQual).forEach(nonPushableQuals::add)); - wherePredicates.parallelStream().forEach(eachQual -> - nullSupplyingTableNameSet.parallelStream().filter(relName -> isEvalNeedRelation(eachQual, relName)) + wherePredicates.forEach(eachQual -> + nullSupplyingTableNameSet.stream().filter(relName -> isEvalNeedRelation(eachQual, relName)) .map(relName -> eachQual).forEach(nonPushableQuals::add)); return nonPushableQuals; @@ -456,7 +456,7 @@ private Map transformEvalsWidthByPassNode( } Set columns = EvalTreeUtil.findUniqueColumns(copy); - columns.parallelStream().filter(c -> c.hasQualifier()).forEach(c -> { + columns.stream().filter(c -> c.hasQualifier()).forEach(c -> { EvalTreeUtil.changeColumnRef(copy, c.getQualifiedName(), c.getSimpleName()); }); @@ -626,7 +626,7 @@ public LogicalNode visitProjection(FilterPushDownContext context, } private Collection reverseTransform(BiMap map, Set remainFilters) { - Set reversed = remainFilters.parallelStream().map(map::get).collect(Collectors.toSet()); + Set reversed = remainFilters.stream().map(map::get).collect(Collectors.toSet()); return reversed; } @@ -761,7 +761,7 @@ private List addHavingNode(FilterPushDownContext context, LogicalPlan GroupbyNode groupByNode) throws TajoException { // find aggregation column Set groupingColumns = new HashSet<>(Arrays.asList(groupByNode.getGroupingColumns())); - Set aggrFunctionOutColumns = groupByNode.getOutSchema().getRootColumns().parallelStream() + Set aggrFunctionOutColumns = groupByNode.getOutSchema().getRootColumns().stream() .filter(column -> !groupingColumns.contains(column)).map(Column::getQualifiedName).collect(Collectors.toSet()); List aggrEvalOrigins = new ArrayList<>(); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java index 235147fda7..4ea5a8a59d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java @@ -430,7 +430,7 @@ public String addExpr(EvalNode evalNode) throws DuplicateColumnException { } private void addNecessaryReferences(EvalNode evalNode) { - EvalTreeUtil.findUniqueColumns(evalNode).parallelStream().map(Column::getQualifiedName).forEach(requiredSet::add); + EvalTreeUtil.findUniqueColumns(evalNode).stream().map(Column::getQualifiedName).forEach(requiredSet::add); } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 40a19347b7..39d5900397 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -86,7 +86,7 @@ public Expr visitProjection(Context context, Stack stack, Projection expr) Set names = new HashSet<>(); - expr.getNamedExprs().parallelStream().filter(namedExpr -> namedExpr.hasAlias()).forEach(namedExpr -> { + expr.getNamedExprs().stream().filter(namedExpr -> namedExpr.hasAlias()).forEach(namedExpr -> { if (names.contains(namedExpr.getAlias())) { context.state.addVerification(SyntaxErrorUtil.makeDuplicateAlias(namedExpr.getAlias())); } else { diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index 393537888f..4ac47289e1 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -372,7 +372,7 @@ public static HTableDescriptor parseHTableDescriptor(TableMeta tableMeta, Schema Collection columnFamilies = columnMapping.getColumnFamilyNames(); //If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column. if (columnFamilies.isEmpty()) { - schema.getRootColumns().parallelStream().map(Column::getSimpleName).forEach(columnFamilies::add); + schema.getRootColumns().stream().map(Column::getSimpleName).forEach(columnFamilies::add); } for (String eachColumnFamily: columnFamilies) { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java index f6fb3b717d..d5e2aa4713 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java @@ -1989,7 +1989,7 @@ private int writeFooter(long bodyLength) throws IOException { // serialize the types writeTypes(builder, schema); // add the stripe information - stripes.parallelStream().forEach(builder::addStripes); + stripes.forEach(builder::addStripes); // add the column statistics writeFileStatistics(builder, treeWriter); // add all of the user metadata From c140a04e17870d532d296f4704f3ca8a3b8a9eaf Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Thu, 11 Feb 2016 11:13:25 +0900 Subject: [PATCH 12/17] fix internal error --- .../apache/tajo/engine/planner/TestLogicalPlanner.java | 10 +++++----- .../physical/DistinctGroupbyHashAggregationExec.java | 2 +- .../apache/tajo/querymaster/DefaultTaskScheduler.java | 2 +- .../apache/tajo/util/metrics/TajoSystemMetrics.java | 2 +- .../org/apache/tajo/webapp/QueryExecutorServlet.java | 4 ++-- .../main/java/org/apache/tajo/plan/LogicalPlan.java | 2 +- .../tajo/plan/rewrite/rules/FilterPushDownRule.java | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 18c32c6389..d8b5a8af59 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -526,7 +526,7 @@ public final void testJoinWithMultipleJoinQual1() throws IOException, TajoExcept } } - qualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { + qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); @@ -564,7 +564,7 @@ public final void testJoinWithMultipleJoinQual2() throws IOException, TajoExcept } } - qualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { + qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "SelectionQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); @@ -607,7 +607,7 @@ public final void testJoinWithMultipleJoinQual3() throws IOException, TajoExcept } } - qualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { + qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); @@ -674,12 +674,12 @@ public final void testJoinWithMultipleJoinQual4() throws IOException, TajoExcept } - joinQualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { + joinQualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); - scanMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { + scanMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java index 6b872c2b0f..7df80ae6e2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java @@ -261,7 +261,7 @@ public Tuple next() throws IOException { listIndex++; } - tupleSlots.parallelStream().forEach(TupleList::clear); + tupleSlots.forEach(TupleList::clear); tupleSlots.clear(); currentAggregatedTupleIndex = 0; diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index 0c8df14256..0c9cb8972f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -288,7 +288,7 @@ private Set getWorkerIds(Collection hosts){ Set workerIds = Sets.newHashSet(); if(hosts.isEmpty()) return workerIds; - stage.getContext().getWorkerMap().values().parallelStream().filter(worker -> hosts.contains(worker.getHost())) + stage.getContext().getWorkerMap().values().stream().filter(worker -> hosts.contains(worker.getHost())) .forEach(worker -> workerIds.add(worker.getId())); return workerIds; } diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java index b5a3384609..c255f9a176 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java @@ -104,7 +104,7 @@ public void stop() { protected void stopAndClearReporter() { synchronized(metricsReporters) { - metricsReporters.parallelStream().forEach(TajoMetricsScheduledReporter::close); + metricsReporters.forEach(TajoMetricsScheduledReporter::close); metricsReporters.clear(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java index 8e0fcebdf5..2863a1d2e3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java @@ -210,7 +210,7 @@ public void service(HttpServletRequest request, } } else if("clearAllQueryRunner".equals(action)) { synchronized(queryRunners) { - queryRunners.values().parallelStream().forEach(QueryRunner::setStop); + queryRunners.values().forEach(QueryRunner::setStop); queryRunners.clear(); } } else if("killQuery".equals(action)) { @@ -265,7 +265,7 @@ public void run() { List queryRunnerList; synchronized(queryRunners) { queryRunnerList = new ArrayList<>(queryRunners.values()); - queryRunnerList.parallelStream().filter(eachQueryRunner -> !eachQueryRunner.running.get() && + queryRunnerList.stream().filter(eachQueryRunner -> !eachQueryRunner.running.get() && (System.currentTimeMillis() - eachQueryRunner.finishTime > 180 * 1000)).forEach(eachQueryRunner -> { queryRunners.remove(eachQueryRunner.queryRunnerId); }); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index f2a068f6f1..5cab345b15 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -268,7 +268,7 @@ public QueryBlock getBlock(LogicalNode node) { public void removeBlock(QueryBlock block) { queryBlocks.remove(block.getName()); List tobeRemoved = queryBlockByPID.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList()); - tobeRemoved.parallelStream().forEach(rn -> queryBlockByPID.remove(rn)); + tobeRemoved.forEach(rn -> queryBlockByPID.remove(rn)); } public void disconnectBlocks(QueryBlock srcBlock, QueryBlock targetBlock) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index 5954a6f797..9d948f232f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -974,7 +974,7 @@ public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, } } - matched.parallelStream().forEach(transformed::remove); + matched.forEach(transformed::remove); context.setToOrigin(transformed); context.addFiltersTobePushed(notMatched); From f49480ad10234718a34a43bf74409a3687dc3e58 Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Thu, 11 Feb 2016 12:26:03 +0900 Subject: [PATCH 13/17] replace all parallelStream() with stream() to fix internalerror --- .../java/org/apache/tajo/catalog/store/HiveCatalogStore.java | 2 +- .../apache/tajo/catalog/store/XMLCatalogSchemaManager.java | 2 +- .../java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java | 2 +- .../src/test/java/org/apache/tajo/TajoTestingCluster.java | 2 +- .../tajo/engine/planner/physical/TestPhysicalPlanner.java | 4 ++-- .../tajo/engine/planner/global/ExecutionBlockCursor.java | 2 +- .../planner/physical/DistinctGroupbyFirstAggregationExec.java | 2 +- .../planner/physical/DistinctGroupbyHashAggregationExec.java | 2 +- .../engine/planner/physical/HashShuffleFileWriteExec.java | 2 +- .../src/main/java/org/apache/tajo/master/QueryManager.java | 2 +- .../main/java/org/apache/tajo/master/exec/QueryExecutor.java | 2 +- .../src/main/java/org/apache/tajo/querymaster/Query.java | 2 +- .../main/java/org/apache/tajo/querymaster/QueryMaster.java | 2 +- .../src/main/java/org/apache/tajo/querymaster/Stage.java | 2 +- .../java/org/apache/tajo/worker/ExecutionBlockContext.java | 2 +- .../src/main/java/org/apache/tajo/worker/TaskExecutor.java | 2 +- tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java | 2 +- .../src/main/java/org/apache/tajo/worker/TaskManager.java | 4 ++-- .../main/java/org/apache/tajo/storage/OldStorageManager.java | 2 +- .../org/apache/tajo/storage/HashShuffleAppenderManager.java | 2 +- 20 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 5f5f2ff4b4..7198a9f175 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -690,7 +690,7 @@ private void renameColumn(String databaseName, String tableName, CatalogProtos.A Table table = client.getHiveClient().getTable(databaseName, tableName); List columns = table.getSd().getCols(); - columns.parallelStream().filter(currentColumn -> currentColumn.getName() + columns.stream().filter(currentColumn -> currentColumn.getName() .equalsIgnoreCase(alterColumnProto.getOldColumnName())).forEach(currentColumn -> { currentColumn.setName(alterColumnProto.getNewColumnName()); }); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index 23ddac7b34..77f2c517b6 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -646,7 +646,7 @@ protected void mergePatches(List patches) { } protected void validateSQLObject(List queries, SQLObject testQuery) { - int occurredCount = (int) queries.parallelStream().filter(query -> query.getType() == testQuery.getType()).count(); + int occurredCount = (int) queries.stream().filter(query -> query.getType() == testQuery.getType()).count(); if (occurredCount > 1) { throw new TajoInternalError("Duplicate Query type (" + testQuery.getType() + ") has found."); diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java index c8a4b5ae89..c6b997fc0c 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java @@ -41,7 +41,7 @@ class CliClientParamsFactory { public static Properties get(@Nullable Properties connParam) { Properties copy = connParam == null ? new Properties() : (Properties) connParam.clone(); - DEFAULT_PARAMS.entrySet().parallelStream().filter(entry -> !copy.contains(entry.getKey())).forEach(entry -> { + DEFAULT_PARAMS.entrySet().stream().filter(entry -> !copy.contains(entry.getKey())).forEach(entry -> { copy.setProperty(entry.getKey(), entry.getValue()); }); return copy; diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 54b8b51f26..d43c84470c 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -471,7 +471,7 @@ public void shutdownMiniTajoCluster() { if(this.tajoMaster != null) { this.tajoMaster.stop(); } - tajoWorkers.parallelStream().forEach(TajoWorker::stopWorkerForce); + tajoWorkers.forEach(TajoWorker::stopWorkerForce); tajoWorkers.clear(); this.tajoMaster= null; } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 7cfa0b2d09..c4fb292ecd 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -844,7 +844,7 @@ public final void testAggregationFunction() throws IOException, TajoException { // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); - groupbyNode.getAggFunctions().parallelStream().forEach(AggregationFunctionCallEval::setFirstPhase); + groupbyNode.getAggFunctions().forEach(AggregationFunctionCallEval::setFirstPhase); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); @@ -873,7 +873,7 @@ public final void testCountFunction() throws IOException, TajoException { // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); - groupbyNode.getAggFunctions().parallelStream().forEach(AggregationFunctionCallEval::setFirstPhase); + groupbyNode.getAggFunctions().forEach(AggregationFunctionCallEval::setFirstPhase); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java index ba17fa6338..0b45ba532f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java @@ -103,7 +103,7 @@ public String toString() { // Add all execution blocks in a depth first and postfix order private void buildDepthFirstOrder(ExecutionBlock current) { if (!masterPlan.isLeaf(current.getId())) { - masterPlan.getChilds(current).parallelStream().forEach(this::buildDepthFirstOrder); + masterPlan.getChilds(current).forEach(this::buildDepthFirstOrder); } orderedBlocks.add(current); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java index c3e9059d68..a0bf345c9c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java @@ -363,7 +363,7 @@ public void rescan() { } public void close() throws IOException { - distinctAggrDatas.values().parallelStream().forEach(TupleSet::clear); + distinctAggrDatas.values().forEach(TupleSet::clear); distinctAggrDatas.clear(); distinctAggrDatas = null; currentGroupingTuples = null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java index 7df80ae6e2..05b58a8476 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java @@ -438,7 +438,7 @@ public TupleList aggregate(Map groupTuples) { } public void close() throws IOException { - hashTable.values().parallelStream().forEach(TupleMap::clear); + hashTable.values().forEach(TupleMap::clear); hashTable.clear(); hashTable = null; iterator = null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java index 549b2809e2..3cd9671ed9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java @@ -234,7 +234,7 @@ public void rescan() throws IOException { @Override public void close() throws IOException{ if (partitionMemoryMap.size() > 0) { - partitionMemoryMap.values().parallelStream().forEach(RowBlock::release); + partitionMemoryMap.values().forEach(RowBlock::release); partitionMemoryMap.clear(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java index 87ff8cadd5..8f80b825dd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java @@ -95,7 +95,7 @@ public void serviceInit(Configuration conf) throws Exception { @Override public void serviceStop() throws Exception { - runningQueries.values().parallelStream().forEach(QueryInProgress::stopProgress); + runningQueries.values().forEach(QueryInProgress::stopProgress); super.serviceStop(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index 5d93b1f856..d7a08aec27 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -402,7 +402,7 @@ public static void startScriptExecutors(QueryContext queryContext, EvalContext e } public static void stopScriptExecutors(EvalContext evalContext) { - evalContext.getAllScriptEngines().parallelStream().forEach(TajoScriptEngine::shutdown); + evalContext.getAllScriptEngines().forEach(TajoScriptEngine::shutdown); } /** diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index dccb577f7b..4f8f128197 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -339,7 +339,7 @@ public List getPartitions() { } public void clearPartitions() { - getStages().parallelStream().forEach(Stage::clearPartitions); + getStages().forEach(Stage::clearPartitions); } public SerializedException getFailureReason() { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index f7921366ff..ddefc19ab9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -447,7 +447,7 @@ public void run() { List tempTasks = new ArrayList<>(); tempTasks.addAll(queryMasterTasks.values()); - tempTasks.parallelStream().filter(eachTask -> !eachTask.isStopped()).forEach(eachTask -> { + tempTasks.stream().filter(eachTask -> !eachTask.isStopped()).forEach(eachTask -> { try { long lastHeartbeat = eachTask.getLastClientHeartbeat(); long time = System.currentTimeMillis() - lastHeartbeat; diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 909f4ea4d2..738f132370 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -494,7 +494,7 @@ private StageHistory makeStageHistory() { } Set partitions = Sets.newHashSet(); - getHashShuffleIntermediateEntries().stream().forEach(entry -> partitions.add(entry.getPartId())); + getHashShuffleIntermediateEntries().forEach(entry -> partitions.add(entry.getPartId())); stageHistory.setTotalInputBytes(totalInputBytes); stageHistory.setTotalReadBytes(totalReadBytes); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java index 4e0b5476c0..6c001b4abf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java @@ -187,7 +187,7 @@ public void stop(){ } // If ExecutionBlock is stopped, all running or pending tasks will be marked as failed. - tasks.values().parallelStream().filter(task -> task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_PENDING || + tasks.values().stream().filter(task -> task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_PENDING || task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_RUNNING).forEach(task -> { try { task.abort(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java index 6fed6b42be..ff748f5542 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java @@ -102,7 +102,7 @@ protected void serviceStop() throws Exception { isStopped = true; threadPool.shutdown(); - fetcherThreadPoolList.parallelStream().forEach(ExecutorService::shutdown); + fetcherThreadPoolList.forEach(ExecutorService::shutdown); super.serviceStop(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java index e7f4899016..66b8536b0e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java @@ -190,7 +190,7 @@ private void startScriptExecutors() throws IOException { } private void stopScriptExecutors() { - context.getEvalContext().getAllScriptEngines().parallelStream().forEach(TajoScriptEngine::shutdown); + context.getEvalContext().getAllScriptEngines().forEach(TajoScriptEngine::shutdown); } @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java index 7e91604e40..237ea71097 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java @@ -91,7 +91,7 @@ protected void serviceInit(Configuration conf) throws Exception { @Override protected void serviceStop() throws Exception { - executionBlockContextMap.values().parallelStream().forEach(ExecutionBlockContext::stop); + executionBlockContextMap.values().forEach(ExecutionBlockContext::stop); super.serviceStop(); } @@ -214,7 +214,7 @@ public void handle(TaskManagerEvent event) { QueryStopEvent queryStopEvent = TUtil.checkTypeAndGet(event, QueryStopEvent.class); //cleanup failure ExecutionBlock - executionBlockContextMap.keySet().parallelStream() + executionBlockContextMap.keySet().stream() .filter(ebId -> ebId.getQueryId().equals(queryStopEvent.getQueryId())).forEach(ebId -> { try { executionBlockContextMap.remove(ebId).stop(); diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java index 0dafc2580a..96b9c9d5ff 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java @@ -95,7 +95,7 @@ protected synchronized static void clearCache() { */ public static void shutdown() throws IOException { synchronized(storageManagers) { - storageManagers.values().parallelStream().forEach(Tablespace::close); + storageManagers.values().forEach(Tablespace::close); } clearCache(); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index 70b3524df7..575c1e53d0 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -220,7 +220,7 @@ public MemoryRowBlock call() throws Exception { } public void shutdown() { - executors.values().parallelStream().forEach(ExecutorService::shutdownNow); + executors.values().forEach(ExecutorService::shutdownNow); } public static class HashShuffleIntermediate { From 17d95e2d549edf36c1d5f44f38014d5a0dd43e70 Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Wed, 9 Mar 2016 17:45:07 +0900 Subject: [PATCH 14/17] refine --- .../org/apache/tajo/TajoTestingCluster.java | 2 +- .../engine/planner/TestLogicalPlanner.java | 10 +-- .../planner/physical/TestPhysicalPlanner.java | 4 +- .../global/ParallelExecutionQueue.java | 5 +- .../builder/DistinctGroupbyBuilder.java | 4 +- .../tajo/engine/query/TaskRequestImpl.java | 76 +++++++++---------- .../NonForwardQueryResultSystemScanner.java | 14 ++-- .../querymaster/DefaultTaskScheduler.java | 13 ++-- .../org/apache/tajo/querymaster/Query.java | 4 +- .../ws/rs/resources/FunctionsResource.java | 6 +- .../tajo/ws/rs/resources/QueryResource.java | 7 +- .../org/apache/tajo/plan/LogicalPlan.java | 5 +- 12 files changed, 69 insertions(+), 81 deletions(-) diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index d43c84470c..54b8b51f26 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -471,7 +471,7 @@ public void shutdownMiniTajoCluster() { if(this.tajoMaster != null) { this.tajoMaster.stop(); } - tajoWorkers.forEach(TajoWorker::stopWorkerForce); + tajoWorkers.parallelStream().forEach(TajoWorker::stopWorkerForce); tajoWorkers.clear(); this.tajoMaster= null; } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index d8b5a8af59..18c32c6389 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -526,7 +526,7 @@ public final void testJoinWithMultipleJoinQual1() throws IOException, TajoExcept } } - qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + qualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); @@ -564,7 +564,7 @@ public final void testJoinWithMultipleJoinQual2() throws IOException, TajoExcept } } - qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + qualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "SelectionQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); @@ -607,7 +607,7 @@ public final void testJoinWithMultipleJoinQual3() throws IOException, TajoExcept } } - qualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + qualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); @@ -674,12 +674,12 @@ public final void testJoinWithMultipleJoinQual4() throws IOException, TajoExcept } - joinQualMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + joinQualMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); - scanMap.entrySet().stream().filter(entry -> !entry.getValue()).forEach(entry -> { + scanMap.entrySet().parallelStream().filter(entry -> !entry.getValue()).forEach(entry -> { Preconditions.checkArgument(false, "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); }); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index c4fb292ecd..7cfa0b2d09 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -844,7 +844,7 @@ public final void testAggregationFunction() throws IOException, TajoException { // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); - groupbyNode.getAggFunctions().forEach(AggregationFunctionCallEval::setFirstPhase); + groupbyNode.getAggFunctions().parallelStream().forEach(AggregationFunctionCallEval::setFirstPhase); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); @@ -873,7 +873,7 @@ public final void testCountFunction() throws IOException, TajoException { // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); - groupbyNode.getAggFunctions().forEach(AggregationFunctionCallEval::setFirstPhase); + groupbyNode.getAggFunctions().parallelStream().forEach(AggregationFunctionCallEval::setFirstPhase); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java index efb6f47f5d..ce10a17b60 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java @@ -76,9 +76,8 @@ public synchronized int size() { public synchronized ExecutionBlock[] first() { int max = Math.min(maximum, executable.size()); List result = new ArrayList<>(); - executable.stream().filter(queue -> result.size() < max && isExecutableNow(queue.peekLast())).forEach(queue -> { - result.add(queue.removeLast()); - }); + executable.stream().filter(queue -> result.size() < max && isExecutableNow(queue.peekLast())) + .forEach(queue -> result.add(queue.removeLast())); LOG.info("Initial executable blocks " + result); return result.toArray(new ExecutionBlock[result.size()]); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index 6665fdc750..94cdb74951 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -519,9 +519,9 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou List oldTargets = secondStageGroupbyNode.getTargets(); List secondGroupbyTargets = new ArrayList<>(); LinkedHashSet distinctColumns = EvalTreeUtil.findUniqueColumns(secondStageGroupbyNode.getAggFunctions().get(0)); - List uniqueDistinctColumn = new ArrayList<>(); // remove origin group by column from distinctColumns - distinctColumns.stream().filter(eachColumn -> !originGroupColumns.contains(eachColumn)).forEach(uniqueDistinctColumn::add); + List uniqueDistinctColumn = distinctColumns.stream() + .filter(eachColumn -> !originGroupColumns.contains(eachColumn)).collect(Collectors.toList()); for (int i = 0; i < originGroupColumns.size(); i++) { secondGroupbyTargets.add(oldTargets.get(i)); if (grpIdx > 0) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java index 00db037e19..4ca44d3e17 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; @@ -221,42 +222,37 @@ private void initFetches() { return; } TaskRequestProtoOrBuilder p = viaProto ? proto : builder; - this.fetches = new ArrayList<>(); - p.getFetchesList().forEach(fetch -> fetches.add(fetch)); + this.fetches = p.getFetchesList().stream().collect(Collectors.toList()); } private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = TaskRequestProto.newBuilder(proto); - } - viaProto = true; - } + if (viaProto || builder == null) { + builder = TaskRequestProto.newBuilder(proto); + } + viaProto = true; + } - private void mergeLocalToBuilder() { - if (id != null) { - builder.setId(this.id.getProto()); - } - if (fragments != null) { - for (FragmentProto fragment : fragments) { - builder.addFragments(fragment); - } - } - if (this.outputTable != null) { - builder.setOutputTable(this.outputTable); - } - if (this.isUpdated) { - builder.setClusteredOutput(this.clusteredOutput); - } - if (this.plan != null) { - builder.setPlan(this.plan); - } - if (this.interQuery != null) { - builder.setInterQuery(this.interQuery); - } + private void mergeLocalToBuilder() { + if (id != null) { + builder.setId(this.id.getProto()); + } + if (fragments != null) { + fragments.forEach(builder::addFragments); + } + if (this.outputTable != null) { + builder.setOutputTable(this.outputTable); + } + if (this.isUpdated) { + builder.setClusteredOutput(this.clusteredOutput); + } + if (this.plan != null) { + builder.setPlan(this.plan); + } + if (this.interQuery != null) { + builder.setInterQuery(this.interQuery); + } if (this.fetches != null) { - for (FetchProto fetch : fetches) { - builder.addFetches(fetch); - } + fetches.forEach(builder::addFetches); } if (this.queryMasterHostAndPort != null) { builder.setQueryMasterHostAndPort(this.queryMasterHostAndPort); @@ -270,14 +266,14 @@ private void mergeLocalToBuilder() { if (this.enforcer != null) { builder.setEnforcer(enforcer.getProto()); } - } + } - private void mergeLocalToProto() { - if(viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } + private void mergeLocalToProto() { + if(viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index d9e8b9305d..e4acab7d66 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -70,6 +70,7 @@ import java.util.List; import java.util.Map; import java.util.Stack; +import java.util.stream.Collectors; public class NonForwardQueryResultSystemScanner implements NonForwardQueryResultScanner { @@ -517,17 +518,16 @@ private Tuple getWorkerTuple(Schema outSchema, NodeStatus aNodeStatus) { private List getClusterInfo(Schema outSchema) { Map workerMap = masterContext.getResourceManager().getNodes(); - List tuples; List queryMasterList = new ArrayList<>(); List nodeStatusList = new ArrayList<>(); - - for (NodeStatus aNodeStatus : workerMap.values()) { + + workerMap.values().forEach(aNodeStatus -> { queryMasterList.add(aNodeStatus); nodeStatusList.add(aNodeStatus); - } - - tuples = new ArrayList<>(queryMasterList.size() + nodeStatusList.size()); - queryMasterList.stream().map(queryMaster -> getQueryMasterTuple(outSchema, queryMaster)).forEach(tuples::add); + }); + + List tuples = queryMasterList.stream().map(queryMaster -> getQueryMasterTuple(outSchema, queryMaster)) + .collect(Collectors.toList()); nodeStatusList.stream().map(nodeStatus -> getWorkerTuple(outSchema, nodeStatus)).forEach(tuples::add); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index 0c9cb8972f..f700c9cee6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -63,6 +63,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.apache.tajo.ResourceProtos.*; @@ -285,12 +286,10 @@ public void handle(TaskSchedulerEvent event) { } private Set getWorkerIds(Collection hosts){ - Set workerIds = Sets.newHashSet(); - if(hosts.isEmpty()) return workerIds; + if(hosts.isEmpty()) return new HashSet<>(); - stage.getContext().getWorkerMap().values().stream().filter(worker -> hosts.contains(worker.getHost())) - .forEach(worker -> workerIds.add(worker.getId())); - return workerIds; + return stage.getContext().getWorkerMap().values().stream().filter(worker -> hosts.contains(worker.getHost())) + .map(WorkerConnectionInfo::getId).collect(Collectors.toSet()); } @@ -326,8 +325,8 @@ protected LinkedList createTaskRequest(final int incompleteTas NodeResourceResponse response = callBack.get(); response.getResourceList().stream() - .map(resource -> new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId())) - .forEach(taskRequestEvents::add); + .map(resource -> new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId())) + .forEach(taskRequestEvents::add); return taskRequestEvents; } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 4f8f128197..32224175fd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -305,9 +305,7 @@ public QueryHistory getQueryHistory() { } private List makeStageHistories() { - List stageHistories = getStages().stream().map(Stage::getStageHistory).collect(Collectors.toList()); - - return stageHistories; + return getStages().stream().map(Stage::getStageHistory).collect(Collectors.toList()); } private QueryHistory makeQueryHistory() { diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java index c444133c37..a01c985082 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; @Path("/functions") public class FunctionsResource { @@ -89,9 +90,8 @@ public Response run(JerseyResourceDelegateContext context) { Collection functionDescriptors = masterContext.getCatalog().getFunctions(); if (functionDescriptors.size() > 0) { - List functionSignature = - new ArrayList<>(functionDescriptors.size()); - functionDescriptors.stream().map(FunctionDesc::getSignature).forEach(functionSignature::add); + List functionSignature = functionDescriptors.stream().map(FunctionDesc::getSignature) + .collect(Collectors.toList()); return Response.ok(functionSignature).build(); } else { diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java index e96e1c864c..7aff6f744b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java @@ -44,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Path("/queries") public class QueryResource { @@ -172,11 +173,7 @@ public Response run(JerseyResourceDelegateContext context) { } private List selectQueriesInfoByState(List queriesInfo, TajoProtos.QueryState state) { - List resultQueriesInfo = new ArrayList<>(queriesInfo.size() / 2); - - queriesInfo.stream().filter(queryInfo -> state.equals(queryInfo.getQueryState())).forEach(resultQueriesInfo::add); - - return resultQueriesInfo; + return queriesInfo.stream().filter(queryInfo -> state.equals(queryInfo.getQueryState())).collect(Collectors.toList()); } private List selectQueriesInfoByTime(List queriesInfo, long startTime, long endTime) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index 5cab345b15..7c8964beac 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -284,9 +284,8 @@ public QueryBlock getParentBlock(QueryBlock block) { } public List getChildBlocks(QueryBlock block) { - List childBlocks = queryBlockGraph.getChilds(block.getName()).stream() - .map(blockName -> queryBlocks.get(blockName)).collect(Collectors.toList()); - return childBlocks; + return queryBlockGraph.getChilds(block.getName()).stream() + .map(blockName -> queryBlocks.get(blockName)).collect(Collectors.toList()); } public void mapExprToBlock(Expr expr, String blockName) { From 387e3cfa00d043cca095cf2177b0594327b8db13 Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Thu, 10 Mar 2016 01:26:29 +0900 Subject: [PATCH 15/17] rewrite about HashSet --- .../apache/tajo/querymaster/DefaultTaskScheduler.java | 2 +- .../joinorder/GreedyHeuristicJoinOrderAlgorithm.java | 3 ++- .../org/apache/tajo/plan/joinorder/JoinOrderingUtil.java | 8 ++++---- .../org/apache/tajo/plan/nameresolver/NameResolver.java | 6 ++---- .../tajo/plan/rewrite/rules/FilterPushDownRule.java | 9 +++++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index f700c9cee6..8ed056577c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -289,7 +289,7 @@ private Set getWorkerIds(Collection hosts){ if(hosts.isEmpty()) return new HashSet<>(); return stage.getContext().getWorkerMap().values().stream().filter(worker -> hosts.contains(worker.getHost())) - .map(WorkerConnectionInfo::getId).collect(Collectors.toSet()); + .map(WorkerConnectionInfo::getId).collect(Collectors.toCollection(HashSet::new)); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java index 10a1adde67..a06a962dc8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java @@ -47,7 +47,8 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm { public FoundJoinOrder findBestOrder(LogicalPlan plan, LogicalPlan.QueryBlock block, JoinGraphContext graphContext) throws TajoException { - Set vertexes = block.getRelations().stream().map(RelationVertex::new).collect(Collectors.toSet()); + Set vertexes = block.getRelations().stream().map(RelationVertex::new) + .collect(Collectors.toCollection(HashSet::new)); // As illustrated at LogicalOptimizer.JoinGraphBuilder, the join graph initially forms a kind of tree. // This join graph can be updated by adding new join edges or removing existing join edges diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java index e117c0d9e5..4a7c553b75 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java @@ -43,10 +43,10 @@ public class JoinOrderingUtil { */ public static Set findJoinConditionForJoinVertex(Set candidates, JoinEdge edge, boolean isOnPredicates) { - Set conditionsForThisJoin = candidates.stream().filter(predicate -> - EvalTreeUtil.isJoinQual(predicate, false) && checkIfEvaluatedAtEdge(predicate, edge, isOnPredicates)) - .collect(Collectors.toSet()); - return conditionsForThisJoin; + return candidates.stream() + .filter(predicate -> EvalTreeUtil.isJoinQual(predicate, false) + && checkIfEvaluatedAtEdge(predicate, edge, isOnPredicates)) + .collect(Collectors.toCollection(HashSet::new)); } /** diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java index 0471ca4715..de8bc36b86 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java @@ -126,10 +126,8 @@ public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tabl */ public static Collection lookupTableByColumns(LogicalPlan.QueryBlock block, String columnName) { - Set found = block.getRelations().stream() - .filter(rel -> rel.getLogicalSchema().contains(columnName)).collect(Collectors.toSet()); - - return found; + return block.getRelations().stream().filter(rel -> rel.getLogicalSchema().contains(columnName)) + .collect(Collectors.toCollection(HashSet::new)); } /** diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index 9d948f232f..cedf223068 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -152,7 +152,8 @@ public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, // check if it can be evaluated here Set matched = context.pushingDownFilters.stream() - .filter(eachEval -> LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)).collect(Collectors.toSet()); + .filter(eachEval -> LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)) + .collect(Collectors.toCollection(HashSet::new)); // if there are search conditions which can be evaluated here, // push down them and remove them from context.pushingDownFilters. @@ -626,8 +627,7 @@ public LogicalNode visitProjection(FilterPushDownContext context, } private Collection reverseTransform(BiMap map, Set remainFilters) { - Set reversed = remainFilters.stream().map(map::get).collect(Collectors.toSet()); - return reversed; + return remainFilters.stream().map(map::get).collect(Collectors.toCollection(HashSet::new)); } private BiMap findCanPushdownAndTransform( @@ -762,7 +762,8 @@ private List addHavingNode(FilterPushDownContext context, LogicalPlan // find aggregation column Set groupingColumns = new HashSet<>(Arrays.asList(groupByNode.getGroupingColumns())); Set aggrFunctionOutColumns = groupByNode.getOutSchema().getRootColumns().stream() - .filter(column -> !groupingColumns.contains(column)).map(Column::getQualifiedName).collect(Collectors.toSet()); + .filter(column -> !groupingColumns.contains(column)).map(Column::getQualifiedName) + .collect(Collectors.toCollection(HashSet::new)); List aggrEvalOrigins = new ArrayList<>(); List aggrEvals = new ArrayList<>(); From 52935019bbba4d63bec713ab3e01adf2c679872e Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Thu, 10 Mar 2016 01:34:45 +0900 Subject: [PATCH 16/17] rollback unnecessarily changed --- .../org/apache/tajo/querymaster/DefaultTaskScheduler.java | 2 +- .../plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java | 3 +-- .../org/apache/tajo/plan/joinorder/JoinOrderingUtil.java | 6 +++--- .../org/apache/tajo/plan/nameresolver/NameResolver.java | 2 +- .../tajo/plan/rewrite/rules/FilterPushDownRule.java | 8 +++----- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index 8ed056577c..f700c9cee6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -289,7 +289,7 @@ private Set getWorkerIds(Collection hosts){ if(hosts.isEmpty()) return new HashSet<>(); return stage.getContext().getWorkerMap().values().stream().filter(worker -> hosts.contains(worker.getHost())) - .map(WorkerConnectionInfo::getId).collect(Collectors.toCollection(HashSet::new)); + .map(WorkerConnectionInfo::getId).collect(Collectors.toSet()); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java index a06a962dc8..10a1adde67 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java @@ -47,8 +47,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm { public FoundJoinOrder findBestOrder(LogicalPlan plan, LogicalPlan.QueryBlock block, JoinGraphContext graphContext) throws TajoException { - Set vertexes = block.getRelations().stream().map(RelationVertex::new) - .collect(Collectors.toCollection(HashSet::new)); + Set vertexes = block.getRelations().stream().map(RelationVertex::new).collect(Collectors.toSet()); // As illustrated at LogicalOptimizer.JoinGraphBuilder, the join graph initially forms a kind of tree. // This join graph can be updated by adding new join edges or removing existing join edges diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java index 4a7c553b75..77cfe8c41c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderingUtil.java @@ -44,9 +44,9 @@ public class JoinOrderingUtil { public static Set findJoinConditionForJoinVertex(Set candidates, JoinEdge edge, boolean isOnPredicates) { return candidates.stream() - .filter(predicate -> EvalTreeUtil.isJoinQual(predicate, false) - && checkIfEvaluatedAtEdge(predicate, edge, isOnPredicates)) - .collect(Collectors.toCollection(HashSet::new)); + .filter(predicate -> + EvalTreeUtil.isJoinQual(predicate, false) && checkIfEvaluatedAtEdge(predicate, edge, isOnPredicates)) + .collect(Collectors.toSet()); } /** diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java index de8bc36b86..97937d1342 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java @@ -127,7 +127,7 @@ public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tabl public static Collection lookupTableByColumns(LogicalPlan.QueryBlock block, String columnName) { return block.getRelations().stream().filter(rel -> rel.getLogicalSchema().contains(columnName)) - .collect(Collectors.toCollection(HashSet::new)); + .collect(Collectors.toSet()); } /** diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index cedf223068..e23d5d725a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -152,8 +152,7 @@ public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, // check if it can be evaluated here Set matched = context.pushingDownFilters.stream() - .filter(eachEval -> LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)) - .collect(Collectors.toCollection(HashSet::new)); + .filter(eachEval -> LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)).collect(Collectors.toSet()); // if there are search conditions which can be evaluated here, // push down them and remove them from context.pushingDownFilters. @@ -627,7 +626,7 @@ public LogicalNode visitProjection(FilterPushDownContext context, } private Collection reverseTransform(BiMap map, Set remainFilters) { - return remainFilters.stream().map(map::get).collect(Collectors.toCollection(HashSet::new)); + return remainFilters.stream().map(map::get).collect(Collectors.toSet()); } private BiMap findCanPushdownAndTransform( @@ -762,8 +761,7 @@ private List addHavingNode(FilterPushDownContext context, LogicalPlan // find aggregation column Set groupingColumns = new HashSet<>(Arrays.asList(groupByNode.getGroupingColumns())); Set aggrFunctionOutColumns = groupByNode.getOutSchema().getRootColumns().stream() - .filter(column -> !groupingColumns.contains(column)).map(Column::getQualifiedName) - .collect(Collectors.toCollection(HashSet::new)); + .filter(column -> !groupingColumns.contains(column)).map(Column::getQualifiedName).collect(Collectors.toSet()); List aggrEvalOrigins = new ArrayList<>(); List aggrEvals = new ArrayList<>(); From a119826c873476bffc92e1c82624b98fc05cb614 Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Tue, 22 Mar 2016 11:06:55 +0900 Subject: [PATCH 17/17] apply some comments --- .../org/apache/tajo/algebra/CreateTable.java | 5 ++- .../store/XMLCatalogSchemaManager.java | 35 ++++++------------- .../apache/tajo/client/TestTajoClient.java | 4 +-- .../tajo/engine/query/TaskRequestImpl.java | 3 +- .../org/apache/tajo/querymaster/Query.java | 6 +--- 5 files changed, 16 insertions(+), 37 deletions(-) diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java index 706e2298c4..0b627baaab 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java @@ -30,7 +30,6 @@ import java.lang.reflect.Type; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; public class CreateTable extends Expr { @Expose @SerializedName("IsExternal") @@ -419,7 +418,7 @@ public Object clone() throws CloneNotSupportedException { } hash.quantity = quantity; if (specifiers != null) { - hash.specifiers = specifiers.stream().collect(Collectors.toList()); + hash.specifiers = new ArrayList<>(specifiers); } return hash; } @@ -469,7 +468,7 @@ public Object clone() throws CloneNotSupportedException { } } if (specifiers != null) { - listPartition.specifiers = specifiers.stream().collect(Collectors.toList()); + listPartition.specifiers = new ArrayList<>(specifiers); } return listPartition; } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index 77f2c517b6..16d5c4f62c 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -44,6 +44,7 @@ import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.stream.Collectors; +import java.util.stream.Stream; public class XMLCatalogSchemaManager { protected final Log LOG = LogFactory.getLog(getClass()); @@ -598,9 +599,8 @@ protected List mergeDatabaseObjects(List objects } } - orderedObjects.stream().filter(object -> object != null).forEach(mergedObjects::add); - - unorderedObjects.stream().filter(object -> object != null).forEach(mergedObjects::add); + Stream.concat(orderedObjects.stream(), unorderedObjects.stream()) + .filter(object -> object != null).forEach(mergedObjects::add); return mergedObjects; } @@ -630,12 +630,9 @@ protected void validatePatch(List patches, SchemaPatch testPatch) { } protected void mergePatches(List patches) { - final List objects = new ArrayList<>(); - patches.stream().forEachOrdered(patch -> { validatePatch(patches, patch); - objects.clear(); List tempObjects = new ArrayList<>(); tempObjects.addAll(patch.getObjects()); patch.clearObjects(); @@ -653,22 +650,6 @@ protected void validateSQLObject(List queries, SQLObject testQuery) { } } - protected void mergeExistQueries(List queries) { - queries.forEach(query -> { - validateSQLObject(queries, query); - - targetStore.addExistQuery(query); - }); - } - - protected void mergeDropStatements(List queries) { - queries.forEach(query -> { - validateSQLObject(queries, query); - - targetStore.addDropStatement(query); - }); - } - public StoreObject merge() { boolean alreadySetDatabaseObject = false; @@ -687,8 +668,14 @@ public StoreObject merge() { } mergePatches(store.getPatches()); - mergeExistQueries(store.getExistQueries()); - mergeDropStatements(store.getDropStatements()); + store.getExistQueries().forEach(query -> { + validateSQLObject(store.getExistQueries(), query); + targetStore.addExistQuery(query); + }); + store.getDropStatements().forEach(query -> { + validateSQLObject(store.getDropStatements(), query); + targetStore.addDropStatement(query); + }); } return this.targetStore; diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java index b07a05fc06..2fe9f5e014 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -57,7 +57,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.*; -import java.util.stream.Collectors; import static org.junit.Assert.*; @@ -176,8 +175,7 @@ public final void testSessionVariables() throws IOException, TajoException, Inte String prefixName = "key_"; String prefixValue = "val_"; - List unsetList = client.getAllSessionVariables().entrySet().stream() - .map(Map.Entry::getKey).collect(Collectors.toList()); + List unsetList = new ArrayList<>(client.getAllSessionVariables().keySet()); client.unsetSessionVariables(unsetList); for (int i = 0; i < 10; i++) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java index 4ca44d3e17..0bdeea8f5d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; @@ -222,7 +221,7 @@ private void initFetches() { return; } TaskRequestProtoOrBuilder p = viaProto ? proto : builder; - this.fetches = p.getFetchesList().stream().collect(Collectors.toList()); + this.fetches = new ArrayList<>(p.getFetchesList()); } private void maybeInitBuilder() { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 32224175fd..9b1227ef2d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -300,14 +300,10 @@ public void setFinishTime() { public QueryHistory getQueryHistory() { QueryHistory queryHistory = makeQueryHistory(); - queryHistory.setStageHistories(makeStageHistories()); + queryHistory.setStageHistories(getStages().stream().map(Stage::getStageHistory).collect(Collectors.toList())); return queryHistory; } - private List makeStageHistories() { - return getStages().stream().map(Stage::getStageHistory).collect(Collectors.toList()); - } - private QueryHistory makeQueryHistory() { QueryHistory queryHistory = new QueryHistory();