From d196d65aa77d6a147f854ba00e5d0c09b5c056f6 Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sun, 7 May 2017 00:21:55 -0700 Subject: [PATCH 01/14] Add missing dependencies --- pom.xml | 70 ++++++++++++++++++++++++++++++++++++++++-- vxquery-cli/pom.xml | 23 +++++++++++--- vxquery-core/pom.xml | 55 +++++++++++++++++++++++++++++++++ vxquery-server/pom.xml | 10 ++++++ vxquery-xtest/pom.xml | 45 +++++++++++++++++++++++++++ 5 files changed, 197 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 5f03a49bc..6fef7c6ba 100644 --- a/pom.xml +++ b/pom.xml @@ -222,7 +222,37 @@ args4j args4j - 2.0.9 + 2.33 + + + + log4j + log4j + 1.2.17 + + + + javax.xml.bind + jaxb-api + 2.2.2 + + + + xml-apis + xml-apis + 1.3.04 + + + + stax + stax-api + 1.0.1 + + + + javax.servlet + servlet-api + 2.5 @@ -231,6 +261,12 @@ ${hyracks.version} + + org.apache.hyracks + hyracks-util + ${hyracks.version} + + org.apache.hyracks hyracks-control-common @@ -328,12 +364,24 @@ jar + + org.apache.hadoop + hadoop-common + 2.7.0 + + org.apache.hadoop hadoop-hdfs 2.7.0 + + org.apache.hadoop + hadoop-mapreduce-client-core + 2.7.0 + + ant ant-trax @@ -383,6 +431,12 @@ 2.8.4 + + com.fasterxml.jackson.core + jackson-databind + 2.8.4 + + commons-io commons-io @@ -395,6 +449,18 @@ 1.4 + + commons-lang + commons-lang + 2.6 + + + + com.google.guava + guava + 11.0.2 + + junit junit @@ -670,7 +736,7 @@ UTF-8 UTF-8 - 0.3.0 + 0.3.2-SNAPSHOT 0.11 diff --git a/vxquery-cli/pom.xml b/vxquery-cli/pom.xml index 52c32ad9d..48bd63fa2 100644 --- a/vxquery-cli/pom.xml +++ b/vxquery-cli/pom.xml @@ -87,6 +87,16 @@ compile + + args4j + args4j + + + + commons-io + commons-io + + org.apache.hyracks hyracks-api @@ -122,19 +132,24 @@ hyracks-control-common + + org.apache.hyracks + hyracks-dataflow-common + + org.apache.hyracks hyracks-dataflow-std - org.apache.hyracks - hyracks-hdfs-core + org.apache.hyracks + hyracks-hdfs-core - org.apache.hyracks - hyracks-hdfs-2.x + org.apache.hyracks + hyracks-hdfs-2.x diff --git a/vxquery-core/pom.xml b/vxquery-core/pom.xml index 38908d7e9..67d45e36e 100644 --- a/vxquery-core/pom.xml +++ b/vxquery-core/pom.xml @@ -175,6 +175,21 @@ commons-lang3 + + org.apache.hyracks + algebricks-common + + + + org.apache.hyracks + algebricks-core + + + + org.apache.hyracks + algebricks-runtime + + org.apache.hyracks algebricks-compiler @@ -195,6 +210,11 @@ hyracks-api + + org.apache.hyracks + hyracks-util + + org.apache.hyracks hyracks-data-std @@ -261,6 +281,11 @@ jackson-core + + com.fasterxml.jackson.core + jackson-databind + + commons-io commons-io @@ -271,12 +296,42 @@ commons-codec + + commons-lang + commons-lang + + + + com.google.guava + guava + + junit junit test + + javax.xml.bind + jaxb-api + + + + xml-apis + xml-apis + + + + stax + stax-api + + + + log4j + log4j + + lucene-core org.apache.lucene diff --git a/vxquery-server/pom.xml b/vxquery-server/pom.xml index e572c3605..5cc9b5d84 100644 --- a/vxquery-server/pom.xml +++ b/vxquery-server/pom.xml @@ -143,6 +143,11 @@ 0.7-SNAPSHOT + + org.apache.hyracks + hyracks-api + + org.apache.hyracks hyracks-control-cc @@ -152,6 +157,11 @@ org.apache.hyracks hyracks-control-nc + + + args4j + args4j + diff --git a/vxquery-xtest/pom.xml b/vxquery-xtest/pom.xml index a00bec201..b8b7a8603 100644 --- a/vxquery-xtest/pom.xml +++ b/vxquery-xtest/pom.xml @@ -143,6 +143,26 @@ 0.7-SNAPSHOT + + args4j + args4j + + + + xml-apis + xml-apis + + + + stax + stax-api + + + + javax.servlet + servlet-api + + org.apache.hyracks hyracks-api @@ -153,6 +173,11 @@ hyracks-client + + org.apache.hyracks + hyracks-control-common + + org.apache.hyracks hyracks-control-cc @@ -168,6 +193,11 @@ hyracks-dataflow-std + + org.apache.hyracks + hyracks-dataflow-common + + org.apache.hyracks hyracks-hdfs-2.x @@ -178,11 +208,26 @@ commons-io + + org.apache.commons + commons-lang3 + + + + org.apache.hadoop + hadoop-common + + org.apache.hadoop hadoop-hdfs + + org.apache.hadoop + hadoop-mapreduce-client-core + + org.mortbay.jetty jetty From 1f03bf766406d5b5e61164f8897b0b074dce8135 Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sun, 7 May 2017 01:07:08 -0700 Subject: [PATCH 02/14] Fixes for current Hyracks 0.3.2-SNAPSHOT mvn package works --- .../java/org/apache/vxquery/cli/VXQuery.java | 33 +++++++++--------- .../VXQueryCollectionOperatorDescriptor.java | 6 ++-- .../VXQueryIndexingOperatorDescriptor.java | 2 +- .../FnDocAvailableScalarEvaluatorFactory.java | 2 +- .../node/FnDocScalarEvaluatorFactory.java | 2 +- .../apache/vxquery/xtest/TestClusterUtil.java | 34 +++++++++---------- .../org/apache/vxquery/xtest/TestRunner.java | 2 +- 7 files changed, 40 insertions(+), 41 deletions(-) diff --git a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java index e0e38433e..9395792ce 100644 --- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java +++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java @@ -20,7 +20,7 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.io.StringReader; -import java.net.InetAddress; +import java.net.Inet4Address; import java.nio.file.Files; import java.util.ArrayList; import java.util.Date; @@ -285,33 +285,32 @@ protected ResultSetId createResultSetId() { * @throws Exception */ public void startLocalHyracks() throws Exception { - String localAddress = InetAddress.getLocalHost().getHostAddress(); + String localAddress = Inet4Address.getLoopbackAddress().getHostAddress(); CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = localAddress; - ccConfig.clientNetPort = 39000; - ccConfig.clusterNetIpAddress = localAddress; - ccConfig.clusterNetPort = 39001; - ccConfig.httpPort = 39002; - ccConfig.profileDumpPeriod = 10000; + ccConfig.setClientListenAddress(localAddress); + ccConfig.setClientListenPort(39000); + ccConfig.setClusterListenAddress(localAddress); + ccConfig.setClusterListenPort(39001); + ccConfig.setConsoleListenPort(39002); + ccConfig.setProfileDumpPeriod(10000); cc = new ClusterControllerService(ccConfig); cc.start(); ncs = new NodeControllerService[opts.localNodeControllers]; for (int i = 0; i < ncs.length; i++) { - NCConfig ncConfig = new NCConfig(); - ncConfig.ccHost = "localhost"; - ncConfig.ccPort = 39001; - ncConfig.clusterNetIPAddress = localAddress; - ncConfig.dataIPAddress = localAddress; - ncConfig.resultIPAddress = localAddress; - ncConfig.nodeId = "nc" + (i + 1); + NCConfig ncConfig = new NCConfig("nc" + (i + 1)); + ncConfig.setClusterAddress("localhost"); + ncConfig.setClusterPort(39001); + ncConfig.setClusterListenAddress(localAddress); + ncConfig.setDataListenAddress(localAddress); + ncConfig.setResultListenAddress(localAddress); //TODO: enable index folder as a cli option for on-the-fly indexing queries - ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString(); + ncConfig.setIODevices(new String[] { Files.createTempDirectory(ncConfig.getNodeId()).toString() }); ncs[i] = new NodeControllerService(ncConfig); ncs[i].start(); } - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + hcc = new HyracksConnection(ccConfig.getClientListenAddress(), ccConfig.getClientListenPort()); } /** diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index 623b48cf5..f5edc4a1d 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -23,7 +23,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.net.InetAddress; +import java.net.Inet4Address; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -109,7 +109,7 @@ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final IFrameFieldAppender appender = new FrameFixedFieldTupleAppender(fieldOutputCount); final short partitionId = (short) ctx.getTaskAttemptId().getTaskId().getPartition(); final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources); - final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); + final String nodeId = ctx.getJobletContext().getServiceContext().getNodeId(); final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final ArrayBackedValueStorage jsonAbvs = new ArrayBackedValueStorage(); final String collectionName = collectionPartitions[partition % collectionPartitions.length]; @@ -182,7 +182,7 @@ public void nextFrame(ByteBuffer buffer) throws HyracksDataException { try { hdfs.scheduleSplits(); ArrayList schedule = hdfs - .getScheduleForNode(InetAddress.getLocalHost().getHostAddress()); + .getScheduleForNode(Inet4Address.getLoopbackAddress().getHostAddress()); List splits = hdfs.getSplits(); List fileSplits = new ArrayList<>(); for (int i : schedule) { diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java index ac92a0eed..a6ced9d32 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java @@ -85,7 +85,7 @@ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final IFrameFieldAppender appender = new FrameFixedFieldTupleAppender(fieldOutputCount); final short partitionId = (short) ctx.getTaskAttemptId().getTaskId().getPartition(); final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources); - final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); + final String nodeId = ctx.getJobletContext().getServiceContext().getNodeId(); final String collectionName = collectionPartitions[partition % collectionPartitions.length]; String collectionModifiedName = collectionName.replace("${nodeId}", nodeId); IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil( diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java index db908f638..8dc49020c 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java @@ -57,7 +57,7 @@ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvalu final DataInputStream di = new DataInputStream(bbis); final int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); - final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); + final String nodeId = ctx.getJobletContext().getServiceContext().getNodeId(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java index 5f08a8eef..b372444c4 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java @@ -57,7 +57,7 @@ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvalu final DataInputStream di = new DataInputStream(bbis); final int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); - final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); + final String nodeId = ctx.getJobletContext().getServiceContext().getNodeId(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java index 0e5b48105..148dfa540 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.IOException; +import java.net.Inet4Address; import java.net.InetAddress; import java.net.UnknownHostException; @@ -45,26 +46,25 @@ private TestClusterUtil() { } public static CCConfig createCCConfig() throws UnknownHostException { - String publicAddress = InetAddress.getLocalHost().getHostAddress(); + String publicAddress = Inet4Address.getLoopbackAddress().getHostAddress(); CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = publicAddress; - ccConfig.clientNetPort = CLIENT_NET_PORT; - ccConfig.clusterNetIpAddress = publicAddress; - ccConfig.clusterNetPort = CLUSTER_NET_PORT; - ccConfig.profileDumpPeriod = PROFILE_DUMP_PERIOD; + ccConfig.setClientListenAddress(publicAddress); + ccConfig.setClientListenPort(CLIENT_NET_PORT); + ccConfig.setClusterListenAddress(publicAddress); + ccConfig.setClusterListenPort(CLUSTER_NET_PORT); + ccConfig.setProfileDumpPeriod(PROFILE_DUMP_PERIOD); return ccConfig; } public static NCConfig createNCConfig() throws UnknownHostException { - String publicAddress = InetAddress.getLocalHost().getHostAddress(); - NCConfig ncConfig1 = new NCConfig(); - ncConfig1.ccHost = CC_HOST; - ncConfig1.ccPort = CLUSTER_NET_PORT; - ncConfig1.clusterNetIPAddress = publicAddress; - ncConfig1.dataIPAddress = publicAddress; - ncConfig1.resultIPAddress = publicAddress; - ncConfig1.nodeId = NODE_ID; - ncConfig1.ioDevices = IO_DEVICES; + String publicAddress = Inet4Address.getLoopbackAddress().getHostAddress(); + NCConfig ncConfig1 = new NCConfig(NODE_ID); + ncConfig1.setClusterAddress(CC_HOST); + ncConfig1.setClusterPort(CLUSTER_NET_PORT); + ncConfig1.setClusterListenAddress(publicAddress); + ncConfig1.setDataPublicAddress(publicAddress); + ncConfig1.setResultPublicAddress(publicAddress); + ncConfig1.setIODevices(new String[] { IO_DEVICES }); return ncConfig1; } @@ -75,11 +75,11 @@ public static ClusterControllerService startCC(XTestOptions opts) throws IOExcep File ccRoot = File.createTempFile(TestRunner.class.getName(), ".data", outDir); ccRoot.delete(); ccRoot.mkdir(); - ccConfig.ccRoot = ccRoot.getAbsolutePath(); + ccConfig.setRootDir(ccRoot.getAbsolutePath()); try { ClusterControllerService cc = new ClusterControllerService(ccConfig); cc.start(); - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + hcc = new HyracksConnection(ccConfig.getClientListenAddress(), ccConfig.getClientListenPort()); hds = new HyracksDataset(hcc, opts.frameSize, opts.threads); return cc; } catch (Exception e) { diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java index e4ba6eb92..2a78853b0 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java @@ -112,7 +112,7 @@ public TestCaseResult run(final TestCase testCase) { IFrame frame = new VSizeFrame(resultDisplayFrameMgr); IHyracksDatasetReader reader = hds.createReader(jobId, ccb.getResultSetId()); // TODO(tillw) remove this loop once the IHyracksDatasetReader reliably returns the correct exception - while (reader.getResultStatus() == DatasetJobRecord.Status.RUNNING) { + while (reader.getResultStatus().getState() == DatasetJobRecord.State.RUNNING) { Thread.sleep(1); } IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor(); From b4493821ca04e822e19e6455f72a0a7f15b0905b Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sat, 13 May 2017 16:28:42 -0700 Subject: [PATCH 03/14] More fixes for current Hyracks 0.3.2-SNAPSHOT --- .../algebricks/VXQueryComparatorFactoryProvider.java | 6 ++++++ .../rewriter/rules/IntroduceTwoStepAggregateRule.java | 9 +++++---- .../metadata/VXQueryCollectionOperatorDescriptor.java | 2 +- .../metadata/VXQueryIndexingOperatorDescriptor.java | 4 ++-- .../functions/index/ShowIndexScalarEvaluatorFactory.java | 2 +- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryComparatorFactoryProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryComparatorFactoryProvider.java index b7196cfc1..be76a22e9 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryComparatorFactoryProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryComparatorFactoryProvider.java @@ -30,6 +30,12 @@ public IBinaryComparatorFactory getBinaryComparatorFactory(Object type, boolean return new BinaryComparatorFactory(type, ascending); } + @Override + public IBinaryComparatorFactory getBinaryComparatorFactory(Object type, boolean ascending, boolean ignoreCase) + throws AlgebricksException { + throw new UnsupportedOperationException(); + } + private static class BinaryComparatorFactory implements IBinaryComparatorFactory { private static final long serialVersionUID = 1L; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceTwoStepAggregateRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceTwoStepAggregateRule.java index 806b53292..df76805a4 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceTwoStepAggregateRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceTwoStepAggregateRule.java @@ -20,9 +20,6 @@ import java.util.Map; import org.apache.commons.lang3.mutable.Mutable; -import org.apache.vxquery.functions.BuiltinFunctions; -import org.apache.vxquery.functions.BuiltinOperators; - import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -37,6 +34,8 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator; import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; +import org.apache.vxquery.functions.BuiltinFunctions; +import org.apache.vxquery.functions.BuiltinOperators; /** * The rule searches for aggregate operators with an aggregate function @@ -107,9 +106,11 @@ public boolean rewritePre(Mutable opRef, IOptimizationContext if (aggregateFunctionCall.isTwoStep()) { return false; } - aggregateFunctionCall.setTwoStep(true); + aggregateFunctionCall = new AggregateFunctionCallExpression(aggregateFunctionCall.getFunctionInfo(), true, + aggregateFunctionCall.getArguments()); aggregateFunctionCall.setStepOneAggregate(AGGREGATE_MAP.get(functionCall.getFunctionIdentifier()).first); aggregateFunctionCall.setStepTwoAggregate(AGGREGATE_MAP.get(functionCall.getFunctionIdentifier()).second); + aggregate.getExpressions().get(0).setValue(aggregateFunctionCall); return true; } return false; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index f5edc4a1d..9a0fa1a4e 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -93,7 +93,7 @@ public VXQueryCollectionOperatorDescriptor(IOperatorDescriptorRegistry spec, Abs totalDataSources = (short) ds.getTotalDataSources(); childSeq = ds.getChildSeq(); valueSeq = ds.getValueSeq(); - recordDescriptors[0] = rDesc; + outRecDescs[0] = rDesc; this.tag = ds.getTag(); this.hdfsConf = hdfsConf; this.nodeControllerInfos = nodeControllerInfos; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java index a6ced9d32..01fc42aef 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java @@ -71,7 +71,7 @@ public VXQueryIndexingOperatorDescriptor(IOperatorDescriptorRegistry spec, VXQue collectionPartitions = ds.getPartitions(); dataSourceId = (short) ds.getDataSourceId(); totalDataSources = (short) ds.getTotalDataSources(); - recordDescriptors[0] = rDesc; + outRecDescs[0] = rDesc; this.elementPath = ds.getElementPath(); } @@ -89,7 +89,7 @@ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final String collectionName = collectionPartitions[partition % collectionPartitions.length]; String collectionModifiedName = collectionName.replace("${nodeId}", nodeId); IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil( - ctx.getIOManager().getIODevices().get(0).getMount()); + ctx.getIoManager().getIODevices().get(0).getMount()); indexCentralizerUtil.readIndexDirectory(); return new AbstractUnaryInputUnaryOutputOperatorNodePushable() { diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java index 6677bd90a..f47488096 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java @@ -52,7 +52,7 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S abvs.reset(); sb.reset(abvs); IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil( - ctx.getIOManager().getIODevices().get(0).getMount()); + ctx.getIoManager().getIODevices().get(0).getMount()); indexCentralizerUtil.readIndexDirectory(); indexCentralizerUtil.getAllCollections(sb); sb.finish(); From 8b6fd08c12a40b92ee312352b8916ea85811db97 Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sun, 14 May 2017 22:56:05 -0700 Subject: [PATCH 04/14] Enable verbose logging for VXQueryIT --- .../src/test/java/org/apache/vxquery/xtest/VXQueryIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java index c6f54f636..e8aee89cb 100644 --- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java +++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java @@ -45,6 +45,7 @@ public static XTestOptions getOptions() { XTestOptions options = getDefaultTestOptions(); options.catalog = XQTS_CATALOG; options.previousTestResults = StringUtils.join(new String[] { "results", "xqts.txt" }, File.separator); + options.verbose = true; return options; } From 061b2f04e3f5a2324961b05e4df60abd1a571193 Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sat, 27 May 2017 20:15:21 -0700 Subject: [PATCH 05/14] Add option to set implicit timezone for tests - The implicit timezone for VXQueryIT is set to "PST". --- .../apache/vxquery/context/DynamicContextImpl.java | 9 +++++++++ .../vxquery/context/DynamicContextImplFactory.java | 11 +++++------ .../accessors/atomic/XSDateTimePointable.java | 5 ++++- .../java/org/apache/vxquery/xtest/TestRunner.java | 13 +++++++++++++ .../java/org/apache/vxquery/xtest/XTestOptions.java | 3 +++ .../java/org/apache/vxquery/xtest/VXQueryIT.java | 2 +- 6 files changed, 35 insertions(+), 8 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImpl.java b/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImpl.java index 6a2a93059..7805c7213 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImpl.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImpl.java @@ -24,6 +24,7 @@ import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.data.std.api.IValueReference; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.vxquery.datamodel.accessors.atomic.XSDateTimePointable; public class DynamicContextImpl implements DynamicContext { private StaticContext sCtx; @@ -56,6 +57,14 @@ public void setCurrentDateTime(IValueReference value) { @Override public void getCurrentDateTime(IPointable value) { + if (currentDateTime == null) { + // if not set, get it from the JVM + final int dtLen = XSDateTimePointable.TYPE_TRAITS.getFixedLength(); + currentDateTime = new byte[dtLen]; + XSDateTimePointable datetimep = new XSDateTimePointable(); + datetimep.set(currentDateTime, 0, dtLen); + datetimep.setCurrentDateTime(); + } value.set(currentDateTime, 0, currentDateTime.length); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImplFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImplFactory.java index a30c5adaa..c00ffc15c 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImplFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImplFactory.java @@ -69,12 +69,11 @@ public DynamicContext createDynamicContext(IHyracksJobletContext ctx) { static IDynamicContextFactory createInstance(DynamicContextImpl dCtx) { IStaticContextFactory scFactory = dCtx.getStaticContext().createFactory(); - final int dtLen = XSDateTimePointable.TYPE_TRAITS.getFixedLength(); - byte[] currentDateTime = new byte[dtLen]; - XSDateTimePointable datetimep = new XSDateTimePointable(); - datetimep.set(currentDateTime, 0, dtLen); - datetimep.setCurrentDateTime(); - + VoidPointable vp = new VoidPointable(); + dCtx.getCurrentDateTime(vp); + byte[] currentDateTime = new byte[vp.getLength()]; + vp.copyInto(currentDateTime); + Map vMap = dCtx.getVariableMap(); int nVars = vMap.size(); QName[] variableNames = new QName[nVars]; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java index 0f3a44a2e..40cf55b66 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java @@ -76,7 +76,10 @@ public ITypeTraits getTypeTraits() { }; public void setCurrentDateTime() { - Calendar cal = Calendar.getInstance(); + setCurrentDateTime(Calendar.getInstance()); + } + + public void setCurrentDateTime(Calendar cal) { TimeZone tz = cal.getTimeZone(); int year = cal.get(Calendar.YEAR); int month = cal.get(Calendar.MONTH); diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java index 2a78853b0..e890e1629 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java @@ -18,8 +18,10 @@ import java.io.InputStreamReader; import java.io.Reader; import java.net.UnknownHostException; +import java.util.Calendar; import java.util.EnumSet; import java.util.Map; +import java.util.TimeZone; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -45,6 +47,7 @@ import org.apache.vxquery.context.DynamicContextImpl; import org.apache.vxquery.context.RootStaticContextImpl; import org.apache.vxquery.context.StaticContextImpl; +import org.apache.vxquery.datamodel.accessors.atomic.XSDateTimePointable; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.result.ResultUtils; @@ -103,6 +106,16 @@ public TestCaseResult run(final TestCase testCase) { in.close(); DynamicContext dCtx = new DynamicContextImpl(compiler.getModule().getModuleContext()); + + if (opts.timezone != null) { + final int dtLen = XSDateTimePointable.TYPE_TRAITS.getFixedLength(); + byte[] currentDateTime = new byte[dtLen]; + XSDateTimePointable datetimep = new XSDateTimePointable(); + datetimep.set(currentDateTime, 0, dtLen); + datetimep.setCurrentDateTime(Calendar.getInstance(TimeZone.getTimeZone(opts.timezone))); + dCtx.setCurrentDateTime(datetimep); + } + spec.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory())); spec.setMaxReattempts(0); diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java index 496b74a8f..d0f2760ad 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java @@ -29,6 +29,9 @@ public class XTestOptions { @Option(name = "-catalog", required = true, usage = "Test catalog XML.") String catalog; + @Option(name = "-tz", required = true, usage = "Implicit timezone for test execution (default: JVM timezone).") + String timezone; + @Option(name = "-threads", required = false, usage = "Number of threads. (default: 1)") int threads; diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java index e8aee89cb..e6dfa22d5 100644 --- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java +++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java @@ -45,7 +45,7 @@ public static XTestOptions getOptions() { XTestOptions options = getDefaultTestOptions(); options.catalog = XQTS_CATALOG; options.previousTestResults = StringUtils.join(new String[] { "results", "xqts.txt" }, File.separator); - options.verbose = true; + options.timezone = "PST"; return options; } From 029f3bc75595ea6c4fa0023154fd5dd3b61d3d49 Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sat, 27 May 2017 20:19:18 -0700 Subject: [PATCH 06/14] The implicit timezone for tests is not required. --- .../src/main/java/org/apache/vxquery/xtest/XTestOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java index d0f2760ad..34cd5b1ee 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java @@ -29,7 +29,7 @@ public class XTestOptions { @Option(name = "-catalog", required = true, usage = "Test catalog XML.") String catalog; - @Option(name = "-tz", required = true, usage = "Implicit timezone for test execution (default: JVM timezone).") + @Option(name = "-tz", required = false, usage = "Implicit timezone for test execution (default: JVM timezone).") String timezone; @Option(name = "-threads", required = false, usage = "Number of threads. (default: 1)") From 37642bee292ba7613cb91d45b2593c3cd6c23665 Mon Sep 17 00:00:00 2001 From: Preston Carman Date: Sun, 4 Jun 2017 12:48:53 -0700 Subject: [PATCH 07/14] Preparing the site for the next release. - Updating the index page to highlight JSONiq extension to XQuery. - Fixing issue with JavaDocs. - Adding contribution details with git commands. --- src/site/apt/development_contribution.apt | 78 ++++++++++++++++-- src/site/apt/index.apt | 5 +- src/site/resources/images/vxquery_stack.png | Bin 13264 -> 14590 bytes .../step/ChildPathStepUnnesting.java | 3 +- .../DescendantOrSelfPathStepUnnesting.java | 3 +- .../functions/util/FunctionHelper.java | 7 +- 6 files changed, 82 insertions(+), 14 deletions(-) diff --git a/src/site/apt/development_contribution.apt b/src/site/apt/development_contribution.apt index f767cfda6..c64bad202 100644 --- a/src/site/apt/development_contribution.apt +++ b/src/site/apt/development_contribution.apt @@ -17,7 +17,7 @@ Contributing Code The following steps outline how to submit code to the VXQuery community for inclusion. Please read the Developer {{{http://vxquery.apache.org/developer_get_started.html}Get Started}} Guide - to answer question about getting start as a developer. + to answer questions about getting start as a developer. VXQuery community supports two methods for contributing code to the project. [[1]] <> @@ -55,13 +55,19 @@ Contributing Code VXQuery uses the following convention when creating a branch: authors_username/topic_or_issue (examples: prestonc/vxquery_142 or tillw/group_by_clause). The following branch name helps keep branches separated and keeps it easy to determine the author and topic. + The authors_username is very important when reviewing a developers code on your own machine. +--- +git checkout master +git pull +git branch prestonc/vxquery_142 +git checkout prestonc/vxquery_142 +--- * Make the change. :-) - * Add new tests. (optional) If the change is not covered in the XQTS, please create a new test in the VXQuery test suite @@ -82,12 +88,19 @@ Contributing Code Remove an extra debug code and verify the patch only includes code for the change. + * Commit and push code. + + Commit changes to the branch and push to github. + * Create a github Pull Request. Once the work has been tested, a pull request can be created for the change branch. Please use the Apache VXQuery master as branch to compare the change branch. - The branch should be up-to-date with the lastest Apache VXQuery master branch. + The branch should be up-to-date with the latest Apache VXQuery master branch. + + Git rebase is a nice option for keeping your code up-to-date with master without messing up the Pull Request. + (Merge will show changes in master as your changes on your branch.) * Post your Pull Request. @@ -96,16 +109,41 @@ Contributing Code At least one other member of the community should review the change. If there is any feedback, address this and repeat the posting process. + * Update your Pull Request. + + Update your change to address any comments from reviewers. + + * Prepare your change for merge. + + Squash your changes into a single commit with a nice commit message. + The commit message's first line should be less than 50 character and any additional comments + are included below a blank line. + +--- +VXQUERY-142: fn:doc support for source files + +The fn:doc function now supports reading files defined in the test suite XML source tag. +--- + + Git rebase has a option of merging commits into a single commit that works nicely for squashing your changes. + ({{{http://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html}git ready}} has a nice example.) + Although, this will not work if you happened use merge when updated to the latest master branch. + + ** Code Reviewer * Review the Pull Request. - Post inline or global comments for the developer. + Post in-line or global comments for the developer. Be polite in your suggestions. Guide the developer to bring the code up to VXQuery's code standards. * Double check the VXQuery and XQTS tests. + Each Pull Request automatically triggers a {{{https://asterix-jenkins.ics.uci.edu/job/vxquery-pr/}VXQuery Jenkins}} + job that runs all the tests. + + ** VXQuery Committer (author or sponsor of the change) @@ -114,9 +152,39 @@ Contributing Code they are pushing on to the repository. Often the committer will also be the reviewer for non-committer changes. + * Add ASF as a git remote (first time committers). + + Create a git remote for ASF repository. {{{https://git-wip-us.apache.org/repos/asf/vxquery.git}}} + * Double check the VXQuery and XQTS tests. + A {{{https://asterix-jenkins.ics.uci.edu/job/vxquery-pr/}VXQuery Jenkins}} instance has been set up to + check the last ten Pull Requests. + The Pull Request being reviewed should pass all tests. + Each commit to a Pull Request will trigger a new test run. + Confirm the last test run passes all the tests. + + * Double check the change. + + Confirm the change has a single commit and includes a nice commit message. + * Merge the change with ASF master. - When merging the change, do not rebase. + When merging the change, do not <> (we do not want to change the Apache commit history). Instead do a single merge commit into Apache VXQuery master. + Since the Pull Request now has a single commit, an alternative would be to cherry pick that commit + from the given branch into master. + +--- +git checkout master +git merge prestonc/vxquery_142 +git log +--- + + Review the log to confirm the history is correct. + + + * Push change to ASF remote. + + Confirm the log is correct on your local master. + Push master to the ASF remote. diff --git a/src/site/apt/index.apt b/src/site/apt/index.apt index 3986f814a..919bf1f92 100644 --- a/src/site/apt/index.apt +++ b/src/site/apt/index.apt @@ -16,8 +16,9 @@ Apache VXQuery Apache VXQuery\x99 will be a standards compliant XML Query processor implemented in Java. - The focus is on the evaluation of queries on large amounts of XML data. - Specifically the goal is to evaluate queries on large collections of relatively small XML documents. + The XQuery processor supports the {{{http://www.jsoniq.org/}JSONiq}} extension to XQuery. + The focus is on the evaluation of queries on large amounts of JSON and XML data. + Specifically the goal is to evaluate queries on large collections of relatively small JSON or XML documents. To achieve this queries will be evaluated on a cluster of shared nothing machines. There are lots of large collections of relatively small documents like e.g. the {{{http://www.sec.gov/info/edgar/ednews/dissemin.htm}EDGAR dataset}} or the {{{http://wiki.openstreetmap.org/wiki/Download}OpenStreetMap dataset}}. diff --git a/src/site/resources/images/vxquery_stack.png b/src/site/resources/images/vxquery_stack.png index 01549a564daf1564f56e154b78f94bd1937d3e5d..5a0efe57a13e1cd92008d4e719da966f9b023f95 100644 GIT binary patch literal 14590 zcmZX5cQo5y_`j_-tx*(Ftx;9fjv!XWu23y%wDt&U#vV1Qw%W6ZRYmO;qg2$cShZJ4 z%^>#trJwIP-}5`?`%iAp9q)Vd+~?lBUa#kQM?Ke3qoHJ@Bp@K5QHQ8N2?z+$`0v%X z$nc-8-XJdm0_h%g6-5L1%uWtnvgwNetZq_P--C=Wii{=t_c0INe*BJ7`{qOMKNdmB zBmlCcV3u$&E>K~6JPsHQ6`b^76$0riQniL+)so& zxc&bZZr0YS*sp&h*zA3kGTCo0dcO2}aFsuK$})f2mw)Zn@1{&wm`T+B@t`7G2CEJ* zwcX#Bqx*Y@NsjgN*4m_NwtM``L)X&*%r%)=oFNnNmKqxfGyME!p|tj$$UOzt0}!)( z+IxnP0Y?BrhIyuOvHnDYqWg`Hb1C7v8$obiqD69Xfs;4POS~>zZj9U=sOP9`jLyD= z`E9#9HBZsKz|x*ZofbLgr&o@f*BiaKbF!&&>MsqJ*&R=v2uX~wj=!`LnKT{s_Ku8v zr%x*qyqEd6wY*E&jJEAK>e1vXhY)1wl@kpI3wB?gp}qA{zpa||wTcx$D}E`%TXCerPh?@LJXEE05X=x7cUGAkM~ z(;-;5#*+_ou1K6jZ)?w$4=wvBFnAu5Li^GOY)(FBzWyYg8cv?Uvh+Nb#an$ZQ^zVc z{F#X=Pzm$a^bQ#KynEGz6RofvSK&p!Q~GWp>K7`9;lYsXrr<4f*`rn^ldpqk*VTJh zx5>-9Wky-)>e^K>b_jqFWoRxi@q=NqiE_qNSlmf|wgTHrzW6o4LZAYg-hlq5@t`&INLdSam={x2(UgEYl+gm3BXx9h0?2SK$R*S(=v2m=r zBnj*FxXe9&Pc@DKQ_lm_j6EAtwEmjK-@~&L8`%B%kUiQ7zun_!C;EVt7?V=c!fCG0 z5(zoL-GpW{AHVNuenE#aM;5)S9_87+p0>f^A%V6;ugR4-FXtJQ6!~iPl_$0+y@*mt~~)# zU=W(Kw;*&R=|4~~Z%NQSOmB@7^2;ByFR@W(Tl+*g!QG_D$pXMme?(W9B=_3>URPDz zF!pCeSIZHC5zW~jg79QJXCvA8{_EKS@!#PH-lQX~0!)Cuqdq(9SEyWD7ejykE?}x32~=G$#KZOu z*w=#uQ2(ym;1*1Ed`sN5A|72&PLGWbmQ0wS8#2PeUwi^@#&$iyOsAOgeU^17EU2{` zh&rCn*^7{3A+fJ$SQMpDbjS%4;(qJeHJ^6P?>2f1;;lCFTrc5Rb>x|b-+VxMS_2dI zg`XX!w{Vrcc3D|5Hngr+HIM|_*A%5q4`f4TBSn~nzgLriio0n-5822+v_nqRxIY^& z)tFUjGf%&!Psmb|jZtw(;iEO0U)sdI8n+>*dR;DdV5XlCql`QPJ8+a~M{U8<5rEh{ zBh)i2ZB8X2%aw?*yALt=H5veEoe!w-^Yr%kI^S01<)kesj+-bEs<&Wwan0$G&Dr(P z3xUR^8EH;RSG~sjR2HFVVW28Jn7m7Q{zWfj-0vHg13lunV26%Op~!?P*wJ&mI{*pz zERs$qNr4!so#B>T+_m`(b5m{mmaT|wX9KAEe`w@{RP`V9&QxYt0Zj99*o<}SXDk0M zEIeN_-HZoXvhlzMx7z!Cs`#j$bv{P6sp98N+jea;|5Rb*dlv2EHG7A1r@@I|%YM1y zh!i}7jXB$7+tAPxZp|f!iv-&n83t8k(#cMUrOq4q-t+V`$gMB}9%ch=wLNs&YUJdTnO%%)q5)boBWGa^8oR3c^)B)>lm%Ypzr$JG#8xAAu6Yx?{n*cn^J0-nLeV z{lVGCW_|)P<2W{61KpFe9jMK>el<{dAx6L1#wY9PWR5pshDRj%p;hX_NqsLX*$ut$ zNJK+354Ep0+*QiDo6qTo4<`*>N5u=%G#c2zWEuhKXJaOG=$F`=Gv0?*;OG`W+ZF|0^ z#0CW!`b{D|Exc4lcEL^NvZ3BHTlXI~#M-D4Ap#_3+W0;}GJgv$8t@F!(9#!`vLI)D zWQzIT?<;U}?h5T9C8qZRY{Bk@NIz zYOQ_is8+kgDo*qfN^rqsRK508P}#JSawB^1%eDvRYo#9DtV@0QGqas2aZQK10GB44 zK2Kv_OX2nbpVT&^{VUkZ@Qal^>o^yx&!!u>*j%<5IZ~(CwsO_UhMAZ@P@Uwyg~*iY zJ=XSPP0+PbRnbm{%2P!|K(@IVsL8a2rZBaS8r^q1S3dvnc3&y!Na+3@rHZiW)G~Dv z=s(EB$TE9BVQ&T~p{5`9&J3lWRw{tUQdSO-l1%Xb!BxDq--yQ&f#(UC>gXRB^^xQS z?I^2*pOoSlD+onrt(qE@@ZcN`{K%iT@XaM{-}+lRZ4G z7maHku#2?)E!gf=1<%aOdFZGHK?%Fs%N9~Gxmg@In9qLCG`P*nf7iYC<`82rb5z#Q zpoJ|~absJ{v>z+J>d2Ax=KM-N0=oJYDSw!=F;;Tp{;*2IK}TJ6awse9hzn@Il`Amc zDD3BZyuImy(K*AEp=qaH0C{y|>x8)c9+xAtGXB|(+KNUbpjUR`9fUq5hEI!hbp5*l zeF_9+mip4-JYH%6zS>Bt*9Y%3poej4k?KWRT_ReEP#taOo8=vXA2d+6i$m>j7LPt? zJXjUxD1vU3r*C<)LkT7d%JDS80E{Vrd?hw5#mNjBf>DugcLa!b}bJy0?K%=UkL=s}X_^imb%vn&;2 zaRu#SA95&bdhRKd5E!r7(b9t!7DxyTiY`)E!FvG27%FK{chAE+X2oQ=QtZPyOcrG`7i>`);!cEQa);fg(0-z?4=MiOL4tRyq~cp*4=CJ4oe>mw|DzP* z{8!Vz()H3ZmH+*cB-9VUj5&{XSwWUyixhjijqW3usx`HTCEQt!$9K1=qBv5}H%K*& z%Cv$4jqKUZ!SBmHZbae61RHqLN{5|o#9D1emKC85e+dVj4XnXt8?I*2MVj$L*)lkn zn#A}kZ=I~vt7%detHsOH-ddOIy~yc!sKWW>qqGX{J=P$=Utfl_p8zL&3v7}g)OU#z2GLFLPiS9AQ|1gROznHQVG*f1?d27q54|ULrBpL1SUu z)O{Pt?!z*ezLWNowx>VQ6cu=vSx~UIrGL*mA-gf^tWUkpg zFOfS{CxQaXwZ@V25_9eL{$8<+owlNRB1t;utBK-37Tn&GnXy zxx&hokYNot4_IRh{%Y7lC5-$wH#upzyR{`4bUv0>d2QazH_<2dZpIJW)(Emh#T-0L zd;N6#bfL|!eD9C1`}&|j)$8UO|4~(z6j@o>@?K_PGk;1rDLRS{-OI&wUX5#P_w?|Q z+XEZ=IAzipJkF+9~Ds6NE$hlda?iZ?&w`p#^enjK-;BCT6Jxa{XNO*1`HzncQfqdR8yH z98Eb7eC)bH8AqbF+OBr+=+kef{sqs?*+@|PuiFv#-&ic^30r+6N-=1;i$Dk@nU5Z1 ztT4H80MgF`S5dJO*wyDxI7?1!nt9E5cTPKr>8Ld?20AVWCh7>JtL}yQtKadf;nACE zhmU;_Nz#ulNCEfOI@Qarag)QmRqlp2?k%!^ywCf}v8g`n|Xviae|?V)nLDEhj?I z4UicQ>ZicEztH*M`!<-7j;e6v867y(@y=$QOenxsot3R?5foQ)HMxUCLEsu=fH`cMfa*Y|=JFBYT* zYl~=K_s)F0T%jd6N%GSm5AiB+nyZyo(BGw4?14Q=V5ZA@YIEz)9z^aHvi;`jeL|gI z?@h-=tMVd_HJ(N|{)YAC;V*9b1tU71V|)F5rr+Yba>5JuS0pKWpkdTQrWm4+xu-5 z60)YA^;>k)dD$lTLy1-n|5T-EYs9=QkZkUz-f_wh%MDX$F@ST$w?X*80Bt`Q9I^gp z-RCvD>>l}B0vd^1)4w0?5n6#0ZeW**9pHC8AHVybYRP9)@$ax@cUMX zDj}0I95bdgd?3na(Y}aDnLVE=A_2?7@mSgr6ohJIW}XYhp~-E<9-qYq@hik>d@q#S zSn`r)P-?(QF_LVElCO??ML7~#U3pT-noDpiQZAK-oqz}Jdi~DXe!X>>P)c0yGs@jR zgX`aGWNjjQva}j2W{RDt73U)gWpyO=8(s} z7`F-oAP&BIP?UVJJjP5auu$M)xmXMy^BWHqmuRLY(0{4W;CS+csf-@76)%U{BV^te zJ?s}GCuE7V&v^Cc&D}^Zicv3b+coO}9h!-!?5U0%AnALE{jid8D;|`zkdS3Af%DnH z@)SQ2`fm1d#${=wK%R_F0IM()gzCLVAx^q6{3eZ&V8ak2Q&R|48}@YYL1@kmni6kp zis<-(>|WC7Hc7$9_(Hw3q!U)ImSxW0iN{F9hL9%_*;7*hZh>ww`NDg0HL0Nl=~8C5 zem;(|pNi#vYxY?&?AwF9{SW%fTSzSOwmdIK!ZCf9V>BOQ56ogggkD1^%AN!L_c|Ru{PJ?*ob6~UcRHh$f+V8gnwzNdlgEcs$7t6mV@BbOq`kDC_u;cnE zn%3a3`tJoO;W&JCC#vEaYKfrZ?Z;W9FVAMA*SE@Na%Yp;6{C#3|HR}pNJ2zQ8M9qE zz8yGNW-Yk7-TS*v$lATFj0$6<5F@$^(h^G|gAzy6Nl89+l#2KeTMLbQMHmrVJL$-B zkgxrgVA7b`D*qMkwTwbUgHMJk5=iA^!r#mfq3D~Di=FL6#ZvgNL$2g-IYhf@*t-nJ z*4ZyiNFcxoF)NQ%TMrrU43pZD10{pV@>-VxvES3k zp&;dhA!1L41gM(%lySc43^iRkcJWLnq%MSpmB2XY^g1FG5WxQqDH!%~bqjPa*e6zY zl08n&z>>rM-b$MS!@&+UNANWTCeaLk@ijl2>QrN+t4m&vRqQ8WmF41Jrn!A=P3B06 zsFxJCBSGBt3#RZmsQc+;MWsC(GwW;l9zmQto+7^9Cc2w`@Kf&d!8l5WH48?9SZA7Y zAjT%H&+Foxn$y;cX5kfJ^GJAmh!6Wk&%>ljQx`-%n$vZ$V^O(nlG- z(M!|@vJ(ne+n{e-HZ?q{l_S+SS z3pUahu}taIB-;Y<3OY?`4Fpb-j?*K~lEPB7?@eZ3vHw0Tj(>2v_T=>}D%UY&ynveD zI=0O*r0(~K{c_(#W`IMj#r}gcYA2{!x{n8$%95%v_Pc0cpR^6A5+XfWWrFt}=C?)J z-~@H>>)!aqNg@H*H=Cg{?rmQ{{nOr0)x`u5HJ+`4`og3!8YnTiV&2_}j2|U#77==) z@_}CZfYs75q2}9zWyOPDSt4gdcY>cQTPXnxH?%UUH)KD;6XKDR0I?|jqO*!jL z9ZcT`xomhLkp-Vi_d?`=670!A^U{YpvU*kbKNbI+o$+|k!EXs5-JZGcy4}cV^^!}{ z>F4Yw-Cv@t@!Je2E5{tc90PzU2kKF4_%H<-bmPxU67Cf_anV5y*^#D&im>cNGp}F>l=D&&UiLQ0|c27CNcxLnbJDrV!y`$qUoq?3P z(^UQf!!Y%3mN-D}V!L(`>9*FN_w%kM-xvA4PGVi%Jo&3gnOf>k0GP{&SrA^6qO&S3 zs%URK=n9`csW}{0CbAdoL;B#_17=~36vb$YlDyL&e7U)@-cK%^D_uMa=C`M(^XdEn zTWQ}@8@{yf$4+|}YY$u_R7bfJo}tGkCy#dotB?m=@<(O5azeyHFwZM=TKgf_lzGtA zVwy%#&``vUuZr?WKw0wDeEpQo;$?Kl&%npsLNF%bLqYSt_Nx^Z+r^(`QUTaZ3xM^d z`%8j{Srab9I`fAM;C|_?faRcJBhk?}{lWWJW5wDRjd-OjCo~7$4W2gcOHcP*-5$74XYKdbn_^(IZT?iL- zUK$^ICjOnb8mGO0hPQI+HTQmM!W;$5|L8n~h%lGiw{8Yve7yNzZeN00N~5mau0-Kn-6m2p{M4LTEJF+XrQxsH!d zqx0YCJ6OCvQqU~IYZNhuju*5kK^JmdkE_mw%gLE@r+ZG9YnYe2zht{^@l5Z%8!=^g z)%GnWlC?R97EvaOhUwjccR@D3s*ao;XGtR0Smg$xWd^`&-&rB-W8qh>>-t8Hm%MfY z@XTJ~C7^OCvk2!$l+@1er{23cgmEX$b!u~>?*+Kun$U_>{|Q}VX+H|bm;*j&@Dr=n_F8gqP znfc&H^Z*(+eAC;yc;e^lia(+TGjGq6^30A5g7pP&wbH_9IM>(N7M{wA!YDTycr6?| zP46IQhq#5zoc$yjtk(xKMlR1@(r)IbP$HoA(E^smJ3F-sMHF2^PdGSmjFTxEMQG-n zU~zYwS#Omu%xSxr$DF^21@YAtWnwnTaUTuvQ$9-hPv^lGMMN70sdG88B;017YYH};G>IJm!^%nlx4BJ%eO~tT$efVanf}CQ-6PC+q`aMxaxV?-s0O< z(YWxUwxSUhpsU$qaj6xSg*k||LRvK^i~xb(42zB!Kyl72Y!vi&f2 zQw}~MF_qWg209&Wn{X5vb3yn#mp{0>wZ;6xCjBL~W_^X=i1S$%d3)CEq)$$!q5anA z-&Y{rf5LV`Tt@qascfFaoDa&gAUN45G`apoe%F1^O3#5IZ0WVOPuM^0D#b3wBXr{< zOT|}THFyCPaO827|Gp{>YrF8}_q^0;#?>*d8~=&!uI`5c+pt6h?w%8DQ{!dF3COqq zWTENPvE@ln!{9<+7@>Z?;%9Xr+J86b>Ya81YU&9Q6H@JP4il z?*h*Jx%`RGNY%PB<5G=gP$hSQv57uQH1D5@w2QSweg^YoVQ1Hjt7nOvY=5SL7~7erNGnt4&pQFBu{x9sUud83D0h=<{9c1Nz5 zpIMNQY2BXBjT#_@mqv_i2=(Qz;F`OWzkUSWDz5fs2XmEbvc~e*#H@X@-?9A0k2pSm zcN6IN$~gEfcn@`jitR8YZ7t$q>=!lXL`^l({%Z8Ni!-oe0sojHvx~o6&RYvjMmFuL z3ZG@vz)nLjw3q(!$;5C*dq=bn_^Dw(M&Z0`YBMzX>Fb(v>+VWz^1l*OFzKT9y8%EI zpQ9ts>!O$QB41&p=G3l!@;|%LtRl_ib)PzzyNNq1jQ~Zv!fmHXZ>GQkjrDRT(Uwy4 zrw#hUHx;eBfyZax94|1a&7rH?J?>^$_9V@50~r_2M+)5# zXj!h8GNcJRY9M6K6a95Wy~v~xbW}^mrAt;tPd5+~7m^GTZBVH9@2SltA*1My8LjuQ znH%cNQiGq3Tqzx;=lK(NiUj_NB1CZhVSmr4JWwA(JB2zWW<&8e%X3;&Lx-3B01mm7 z^!K^?iB1PC$zFNfVq@*7KhNBc=bM8k(6XK%Zywnb&xFU(uGL~!X$d}mYgoVgOL1uR z5H{2nH#_6~h&ry08>Vd=y*kI!Zi!A-z4cdQly@%6IM<#BKw9u8l21oQufJDxH@~P5 zm(s#gcV{ufTZ3&^_*+TLoNjp|mH@FiZ%3l^2p{A5^jau6I{!m={9QIK%4nKApBhnD zQ7!N=F>%w|!xj39HA0fMcfEUO9(RRfxBz>A*tP{cKL)Lj*Ozzb=#|M@?*X(G#~Xk= zGz%3YR@15ac@5Qr4UTi*LqdN*J&|x}vKIrT)To`RNdnwD*LIs?Uiyu1i*v z92~a*r@#`WZ_H05gJ0ck{<-Y}u+fV$e)DToOR@{M)?BD{ZMJH3Fb%R9=k+u?odfCS z7B&T}d?38Jv!75w|KB1rzZ2U8_l=_lsa*xSZJ)_YjP?ZCVkV$CqQRz;C8yjlR@-59 zLiBsayUe1K@`F)QX;N8Cch9TUI!S(BO1tFFq~GH#b#WXR(cSyl3$Y~5b8rkYKs;g0!8ZBG{tJAcMzJ7Y2w!w3+ta|d*)`ZEwokrW* zwynHbo_I6(wm#}1Gw%Sgy~DFoa)BvPVG80DC~+d|$#@byE~x^f3U)w=Fa>IbB=>>f zZ~o9c*RKtSX))>E?si`8Fn*(jG}=l>%*=F+r>apyvj~`VH4buH6zHnT?MzB6Y`LL= zx}49cHReoE>J)xsfubMO*k{&^5U~_%w<+G-OIcAUJzUyo5Yf6bcji zFX~uVZ2VKu#j7YhfJUi$x%>HA_?w|I2|7xCAMB*yl$3PlJMv5)ZY?o4RmDx^Ub77X zdNu^-zE$D)DYuAKm-@ZodWohw+lu+r0}#yR5{8r^D+|bLdAYuk8Ts>-4G6VeWyz6e z{KkX*Wy8B!#^=<~ux|ZUD+wxu;fcY;%;WX(SK=d7%GCP^KEF8Vg4|k>*z@v*|H2lu zSzq6OSFdOl8yT=MpPA8eIh7viN?vdmBk;4du9dZ<-3mC21xo)E&m8HV%=SIY&J2gx^<32uWnau$xT@w*j6 z^bNqVzC9SMHoJ`_kBD+-gYV;6(Ie({7#XBuzU}44K!((E%rN^7sq~fiBwu2d2r0Uc47%gw!RA$8X8U9CQOoCpM_CG7c7=h-hCJl&}$6rFk?Zs%wYDS{zl(Z?B6A->=9vw8i~y<}>7qI~&%c zn(G5w)U8PsQ1PaZ$&t&@x@Jv;^&3Im(|Ar6FPR11j_29Ta z>ik)L5cadDRQO0V_)*35KreWF7D7O*3Z$+U`WW9`i6CqOCwu&{7s0juvFL>9DLkVm z_CIUW_NIOqR zY|%MA&tItT=N=$uLK2fzg#Kz-OvUo}+SxQ=SPsW`eXkDM5y?jhi6@yCPH2cAk~9J2u~`y} zJCv+zQx7;gS}aXXX^_j_E}Q>`w?`UF$2ad;RvhycX2<2ss+kk4?~99|OA!>g)_Yhk z!N!vk_q%D8Vp*sdRk^Z?9(7s08@U3rMU02#m9?9&j93&Zm;rKp^XC)0JfMmO01or| zcXCSEVa6Fk9|Vhp?-YNcEx@NB*n%n| zJuj2@1%e%BdqkUy7QB4WE1ObKi-TAfN0blF##C3;dhS!TFRG}X&)Jdx6PmZns<1#L zCz)QA`LQ%k;OJ+OJ|V(^ZbL<%cRNKxj82Cnb^QGXBAmbG5Q)2SKn7;NdW5H$X#cj; zf{7brpTI_K#@s*OJ)p{r8FGFLMW$3NR}YTf|G-1;M;u%MV-bxYF>mKybAv$-`}?8~ zBQ_9`rG2;&)AK-!kG5Y#s_?ipZZVzT@=O?X6KdPi%EI*}9WxyDlkxqAW8i*V8KK@M z)oA9nZ6rqcVm3p}R$eS95HmKoj)$>$u3@Lvi?n4H@*VD@N;N0~yc5MAH=B_BBQh%Y8SvQ@_(G;4sdcLI|BubDK9R#m@h;kY+*$nH zz5IGlFoU^M!YMi)u)K-=_g@;kFTp$9DH@g)d%J#Ow!^^JZWvUh;)Y_>^@V|#Z?E@0l zof-J?YyY>U2|E4t+Vzt-=bIy8gE8byoPvGxfj)2lGdUf#Yi#RPeEC_gMpoTHK){Jm z(C?bHhML&pvq!$z9oG01Hv}5@WKZs-J!3>*i`=E@*mRrNvq8OiD5U zS+-{8wX5kiAXg6aIx7kKBE|Bc#H7NhFrfY8f}bDCalX3|hP;_RL{j29y<@Kgv|FsHQ^LrD5=$wZnn%lr#{4b zF}yl^TBy)rqwJX@xagVFj3_gES^hqG>&K2_WEYUd>uNOkFe$o{!%!+Yj=7gW(r4kk z{6@b)?cblgJ{_V?apC0d^7IbJT4USBwS!L}ZtD88>vf^%Ol;?)$uy)L?C9dCY7S@( z&YVJ3{k@vlJQ4QW@GNZSGfW$ot2~NR$4BZtC_|F>dC}|PVrRqD^fzM{yh3tE=Dt0! z5nu47HS(|R%Z}nq1=M3<-LBlQ0Z8xS?BYRR3+fOx<^S0?po3#=h+W>`=%-=FpOgvz zk74>-A&gf&15$KUEHlg@VLD}EWI#{t{_c0f-@9Y6gj2<)O_Dn*eoRMzt&rs7Wxw5M zh3i!IU7zF3#m&nwu&zX9M1jPt0B(eG$@rZ9BueaJ!BlayEGOr~4{auGuDJ z{2>2zJ-FXp&Bsn`^XiM>qU2+cj$eACE@z#n!%2F_q}PH;k$;`ymkNja#x`?*+Z~k* zr@?e&Q(AHwt-j#!yMOk(jX=iWfWu!SNO7OvbFYcr+yPqN>j%7VM8OR(m#Py9e~qX{ zD&ko86&8tsd3>ljB^QkRiRwfLLPGbEwbpb5x)O#WrhgZ+^>%v?a2Au#Dg?=ReCK`g zb7AYE-LUX7zC}H+;J>pI<(^N6gRZ}9+RkM9n)uk4-xsMMhsM;LbfsHZVtz~^+P-pB z$cg+lw3$i9_I)}p-i+CIAwhtYv(?O!t#`Ad6xB@bLOYK z3FgDaF81M(R_u#U!aLT%598g3&CDtUJ&$Kgjr~lF9bNo7p7<|YPc(4TIr8K(w6Y|R z?eZuy)c6pU9g2r0vpnTgc-q77t8DxH;EwonGcPlUEdg{HupG7aU`J<;p2!8 zI2y(dx2ua>(t@E!ye?uu$%`@eF-Dc4=WOEC_x`+J{vxqedkdmRJy~ZzR_^)NsHVVJ z0lZ?{2Q}~C;mbnANahOmA{CD!Sa@XE5{%gos?b{!*QL*oC)1H{+6cy@^ z+SyHY^}(Z$k@_!jc3bt0K^;E_`mLu*hJ@%AMaSqOZXwWPx6t<;B4`0WELN<0h`WFP zt{xZ8PoNbRlaRE<@8J8$rSIFz5h3J%7Sh>NM0ppMGDKHes`-@oYbVip1po<`A3)OJ zK58Rs=}iMF1pMoQbgy+8M%C|?o^w^5z)4@H#YQt%nWOQo-vA5i4{hwb%4Rts?}KPJ z@xdtOkw@#fE3vG}?4$yoCon}!;^k?aPP`oXz8v9powfsYng{rcUV@wH%mdm+sBJFY zz}QR{M$U6b`l;21y>igKTX}KSa6B&Zd=?>^I&LgGj=b#x5wU^4vAN}RTqcF^A?x&e z27e5H6*l~s(Qb;{K=p3L&#*hV%n%i7vn3eWop_4R#_$SOO9UB|4#IWZQi+Jy8I77(aemS4Q(@d<)c*lW%;2 z@z8eogy$oAgEQ7p;@-W#r!kQ$-yMW~DZ%ih>ik}?anoq!9riL~fJ8hGHZ!d5Z&8+1 z#M`f{5Dqu(PjR+On;TwAe;#Y<`&S8&N3 zIw}fgLiDy7g@uH^~nXD){u$c~O3Hq3lHL=rtu`#_=UE z1gt^FSxA*F(sfr`f#CsGl~0J1k}@XbOZfQ8uP640wo*X1P6K;3=YJ^B8Ie%9V{(kL&t;r zF zNWJQ3Fi?Fekx1AKE+)S3mNXvi%x&R&y6^cl(q-y9I57yMnIbBf6?2DOpIBny{deY7 z!oAf4Vo(-?3^V~v@y)eRXyluub}Y+z^*D%_JOK)*u2Y>Hlq7;?d1hx$*lYUXT@!7I`Ah?q6&^LJ&6u|&g$lZWtmuWa_C3|LH4do^OxMGXU8uEqf1j=a>%kw>{`SU( z@aCo|j_V?8j2b;e4p5sb$9wjo2!biy(3MQGWt;bmtJ;207CHjpGcv_2#cbQs3g`U? zPl=s)0QDE7zXRBRoOpi4hmt^xdWhG_!enI=QuS+}gL<8e(u8GUE~6isR_`Y`RGEf& zB!!OK=oNoXJSiYndQX@Cw7qdnq+vFWZeXUKg7xjMm8~^aeX+pQv^Ch-Z}{WYSX7TfP(6ignO3MNBO>KWRb-SsC zhSh}xg|+oeueq*cpNPy03r{k#_F-np2G@6qo`{Bf$(&YBjo~7cz z7h&#Dzh6o5I|f#*OoYtI?kjtMJkh_19@!&niM=0WL_7oUu-wa6M)C~=3clGuCJnda;{tQ;3_k5jpz2sDiZcrb2r=&_K!EltPopGRSL_P_09deClX zhItP;HXcJ*t=e&(#XDtUXQ#kk>jA&^Tg&<<(jC4h@>o{k7SjBbsgmCAKb(5Bx!iu0 z8>Kzgjb4Yf<+ugWgKFEP7AR&Mmj;wN3S67dGswYnm^#vN5ZYMtb#apM@5aP7FFfeO zXOrsaSS8}kKI;{9D^qGQ21SL?>c0hBE?CFH3au zL`BMB$9-`!IedtJV{*RENY-BU*3Qbd2ceLaRWI(sRonJ<2+7foPw{*KpZQ8dL0rNo z=C=W-tSwg>S%! zju$_3iSqet;r2!V|Kk8%D#VGxZg&0i^ZQFX_SJ`NPXf&GotMzqCkgFs1zCq0nh%ny wQ{Urv1*ncsFaK|4|2G5q-}>#JUn_9DexY{%h@uDoUL%3Js*Xyfl11qM1L5`C!Th)7EaMKSbVqzVWG5NT3EktQwl z-UEaVQbLE&x#7L{$M>yu*E;L0%${>*=FFZw^XzBuguHpJ`r!VP`!{afc<>6OtaIbW z%^||?QsTRW`$tb%j~h2c6JIGS>OnEMG>Uk$x8GKrV}BuI^#c6$fK>jEo}wMdo&OAh z(TuiLsz;eumU6ge{NFlQqk%0~?dv|D?S@}!d*-9~qQjzJp4&C6>8@KoXEN{%1;K72 zMautmW#$i>uRO3W<|*gr(o|0__%yV0G9xZa6U0 zJ}pvwNBfq_mU>uyw62->um6s`h1rVP{KfWbWGX`+;15m;c1Tp><{Te{(4$OYeAY~R z@`V+V>e~9r1>|WEac&dj7F7d={;c90Dki_%%J4XR;7_5#)0grx%_yv-d(ege8FO>P zo2^4@Us3g_vtn2GM~Ey#7OawV(yH~^Jo!geNl){LbA67z=;-@|0bI;?pZD(I1C>)u z%H8NB?~u|@Erl{k1zLeY8s;8U)uz{nY+_lnOshoN89q?JoJO)~ z)?dJ(jhiz$)Rm$_ThS1TZNs8SV?wvDKzA&OGM~vv`j+#w)5%iy^$SGLngMYB(gALo z3Cv0&TJJR9haX8gS8@>J8{X(gEgO}Fu=&rG^6koFzYk~f-4W)$wLzptXa+&DR=g8{ z-d7KJDWQ^O@PL7Ro|hMC^7TjFZ}#taU`oodi`I@^ZC2U#0GMXIb!9H*)H0xEvMDp# zkdftC()I_EeAaN;2_nE593J_?<%92+b3Q%Us@H($g08y$m59Hj_Pj(8>_*oP%;sB# ze{_)fsx_9<#ra?1$?N3?-n17{3DMi)*Nre1dMr%RG|c4>eAA3I)Mo8BYnrGa1A;sQ z7h#1QNbyLNly~LFT_-t*O!C5R@?xgmMd`%L(#<&}(!nsE>5sAoke669TqV;M*HW|J zEZnT+D-O|;Y~YY_BRdTI$Yhuz^hlm>!^Pk!5)QnC$(gU9+>hB9Zl!0E(3FIuE*0*i7#vbaBc25okEPG7@nV>#U=MTlPXrkSPof`hS zemC*{ussa59aR#P@9p)l?=!U5N$06H@VGR{K|L95Ye+1y4e$8)o^INXKXHnfVNzU5 z>OFhm@o{<>;r4-2qxHD?qqm}mkW*#Fk7%38ro-Q7W#@PH{NLbeg-g^@8W1;*0zEzb zC^4i;e;&`*3k0~7 z(M(^qH=kN}y@5?oACdWztm$I9)7*p;6#&H{=(*-Xm?{|=$!Tp;8tvWY4!ZnaM*Q|p z!;ZifiB)a0ik(A5n?u%jRxwtayTT3yF~!f28$G+mtM{BM1R1K+UEfk-pq|JY0M2xE z{^V1G+m80|bpNFs#`?Ef}msxr7Wn+}lpHOI{0H{*$96QUtw2htub9r9-oL+anmNbon{NL^)sgczjbc z%70?pSE%7nO{7RLYJbAUK}1-tAkdxvQAFrnS6vH9^8ddN{P#TNQzYSag1Uj06;)Nv zsq#LjuGU);>At6H&jTt73N~PVy&+ub$G1_r0~1WGo2$gxE|=$3J^E{4#`*6qvN;YrFu+cInk!a#v4u}{4MytztHBj>}6Pzn(`e*I*)5v zwu%NvmOm?5EkPDA$d>z1Tahx*j38^aqtXre$x7TJkn1>J>|;)7x>@R@QwLKRA+~}#pb>Bdr>Xwi#tUr+ddKkdJJ#$~2`CW@slqeDN zH!?xMxbo3Q&D@IcAFPB4`b-~-sG*Y>A|ks!D1TDJ%f5nNnmLgFSEAs|g8xF=^SD|k z<85z`Z-!X0#|jTD&(|~EKfs1XpmEm4g5?H7qe;M&s-*hzoSfxSDc6#R+zwPM0mx7Q{7(Ivy3z$%jT^;__SFG zm$W7DsL`OXi1l*%#kO^P8z=3s>IT`q zxh&k{D4eWvqT~Fyy?iLlJpKYo7bt{i35k0xy_O&4vk4aIu+rTh(bc+8Fe;UcewHLJ zca|sUXB24cIvUPHeE0C%-!vM z?Opo?+V?0wre7sJum`-_^_kv~2mHw4Jgr$lCB8_a6^&f=`)d{UljelBjSOjqX1DJjtz-}c#tKOCasMUI=s zeVauUR`{F^>|@Gpq1}}*SHH<)lkHfF;(c#?ZTS!eb>a9-cew)IR(FIcJV@?;pJS6@ z=s^^4P*-?`&#>hj>ZMq<2@-y*?g^%>3p2y0SYw|=wzwO4Ej~ia8&tbFX1SO10C41Z z^W}s)E>5WK)j)RMh9Gs2VB`7TZH3lwL$TNT88ehe6ep~5HHIQBm7gbA3 zOZx^Ewds*mhH@Z#1(8v59y{%Qv-pP0Wt63ozOQqjUnP(AqO;ui;o*a0D!kIfWb(I$ z)6s+aUuL=`KccJv{C5CaJF2=gH3W24=CV~P zRPJk$T>JeW8W{TS1L?BiQlPU(066s)KA?X;NLSF+Q(W2<7`iKewprdS_(FAg+m3qZ zY^%29Q&f4M^fQSEw?I$oKr)pMqrbX@>pCY|GNj$t0c>G6hR2Bc(rCAR7IGGv*!qb1 z@Nv?zS#nRmDdv*_CG1QvA`3^KiOY)f3UW;UHF2s(FTRq``^iDr*NtymXaV;lD z5}xtLBo}OYa6Ea@2N=-7dnx8}e}8qHMu$DqvQOdi_~)Y`otvk^L+v0h00%D@(so`PWSMBal@Gd76vGB=)k&(~NK=GgTU&xP!KdIB*hF;|`IM?k`0yw5& z82daXQ2SN*TeOwNB)^8*$&Ls~%a}tD%5hx3*m6_Uoq0R9tgSs?2A^F#GS%GbV#dQ@ z-;tN_mm0`Pv9CsDE<0(sl&{e4{AVrqW;(z7ocx&n60PSAXo9S+G;syB^tXC1$A%gCAE-}8@)s0?Vta}Du%Ji$Zh%FRYb z;v#koi8)R`3ZNejhDFHv2bKmv_33O{{riavV9%F24}(eXGJR>DvIahgSXGIAlN-X; zPW6^red_)oBL1vy%e|yvDag?BoJ0lge1GsMfBGSR)wE7$#_2~0zp%=`4hrr5eT**C z%~V7yY(41-ZNS(oOQ1o~YO@bW{p5nqgOTUAtUa~N%NhRZaMlGLU9K+oGh7t~w9kzU zdKpG5SS+iA33kihk!Oa-y#9<3!=dJ5Qi76KUFLH>F;XcF{0NDTFk_P^rf2K862I)Q zPvPNfL5G+uZCzLqH(9zcLLlJsTW5)Lc;tzPR|~C z`1oQVHOpZ}44`uAmeiztaUm((wg%>l>I#c(0ER!%z)i>`0? zQ7Mv!QVVX-@aa6vaUN9+dsK<{^0V8K>~kxq@jnUhoT21)hD1S?^dK!0JJGxGB=*%`i>8ZKK{I#U5b)9_3Q&)G5fS zDntgB<_hn;&;dRPBVr4NQt}=wJcbLDMzei}Ed~%L2MFUDN`sSV?bv{ZvHj52{2h0! zlvZ;)QVCg4JdbLl4qXrza;BqF;}6MK1r(E?fJI11(WMR;>B-UM zYLGNPm7IivyND@Cl4d18aWqR5^}tL3Lltx85F3=wv>2ui0^=5gKcusQlYBaoqS-88 z65XX34o8yYlY~)7jEOo*&`>eh<}eCVkulg7jS4I&rMBKOgcwB!SoOLtpBh&5cJnQa z8g)zoJ5>VWqYLGRs~FZ`V`6V^2rqXC`+FLk*CELPd{mO$TAp3( zdrmF9x0r1jHtw((L)c845fU46vLNP*?HPiW0|d1|yJVdbT0g^xdE5`&i)b)8y*rHPsC;ZIHHa6_rJbk?$63A-rXj?!D6* zh9POq+_Z}apU<;L&{o9HM+C%^h0qP4Lw4a^_tCUsWAA;2B1C75^ z$R`PYr2L}tX7P={Vk&V4?c0R@k%u~HWvaYK5I&SSpzU_f_wQg?cOjtHr1kCkIpeqp zVkTGl4-6#sZvm)RvT$lI z`$2-<9$wv#W$jR1SzQfO2azlO`q*{gw8BGvu*PNFs$b*Vf@Rq#t`nc%=eH%bo zVJno?WjhNPDiQc2C*hM2*1#>~o;}Gjn+?s%Q6^TBpH1B)8ULZ%LWe;%X`n4#m`x=( z^^~o8dRFVWi(1D~$?-Nro@T(i-RSy6Q%w*_Q5;48=Md|7F4P(V4Jz30b>5;y)S;5Y zLyjn<7#|Hn4OkH2LPBd!(?T>l%0ofUV9O)e7(3E5ENPJHH|6tkU&1_g^&mOWA0cLT z{D9?#?|Im44NxKtWt_ETnyo{fFp6_TT-T5W2VCke@{l2$Ea~SX9(Q{g7bieIKRNYv zb&5ul%{XC)7k6xGpf5kmC_D02v!*;ov!|1q*g0O4Ok=>x<#%*W003gI;ZUHUaOB|s zII4|}S>r|7j)OMfxt54|gTJJ3=iSn0yt&XpH?ch;K69lfa7SbneV;G*4(DyaQG7cK z*R;9`tCP6nWd8?oK?(A&CR2tC6UVtxPmh-crl-*91tj?@^sLqq^^PR=r5VG`aRuUM zBQ{M7ulDFmU%@O_29sIY%#nx8jEjlkVDw@r0(|jj)C(~JER3h7djF6Tzq?p9iij6C ztoS!T>A?>`iL}R+zWfqRen&kUmEBALrPQAtYF8)7-h|uNk=@x2p#&^+*oGH{#0PMY zP{b-=sTN*Jlhx3^rshS2QS#th!I~scet?P)Mizcf>1I z6Z5$8`@g3S(0CVif31ZA?}sjkY^fvu1xw^aBKN_65nrfgBq{s^_d?nuc?tqr3If77 zkhg(<)LZsq_;otjpCZNeZEqPxR31@(b@I2Fnn;dOu{(cx6Pm(l{t`_=CEr|9Y8%Yu z=>M);^;`p+1#~D6*iEP8&9IPTdm=~?lDIpuiLHC3l1{S{$wdbUuSgZli^v$dIWGy> zmV1MlasJq}Up`si@W9SQTJ(~BSHXH`!BiatiEweyEJaj^QxPfWjfs%A4shA&=a@??i^3OBi_j~LckZ_+*7epGI;@c^e|!^s1%aXGmzZ@a{!7v1-m}?7 zR|tdAUsK;INBKu@eFSIk=ftYo+h#Xy!oo^lYW#{w1vn7DM4=Mp&%3G_8OgIDsa=Z? zBrqkDt&8XicHGjPWRJhY5lY(?)Q7wx_Mvq}n)mj@L^zSewUxxOdnWhG3Md%Fmmr5h z|JcOyIoQHG^=tJRwSj+F!{2(SOQ&jp+G4coj_1u~cma<&{-a!^7dj^jdOyoE#)T6c zL4=;G@bV#XxJh|>yHffNvh}^QQ@yuhN7u&cxK>}Nzh}yi@`pu|BaGTwH_u?j97uzYeDfB3)oyId8 z6#K}qVpfO)1odm^b{%X#Jx-V=^&5k@6V!KeqO`Fy**fXbP-W#N&Ne++j$Mgg{1n@z zn`hJ5H6%mv3dBId#Kfdfe}r-5$h8{Y?UUK|_T5|R+^mJU)KlXKd{*VmC-q9zyCzP0 z4f|=6{pj${hM6Bj06@fmsPFl4M!MhSiNe(aZR2qrS;fRZ@8kI(KhI8%#B)b=^=`rP zFOAIF$NMbIT(*{fAQRx&ddOXJ>2O<_U5!ht8+i(f7cv<5ip@_hg~3 zwa33)JOQJXW2r7zQ)+qza>?jQSO}hCNqZ#I*+I3-qDQC|d)c|Kysod-mGH*wk3Vop z?3oBYoFP%pjAkq?3o2%G^kR*1kCap5DbdftfDvCJZ-Zn!!mG9MpB1o+2Cuq83&p!r5~fSuc$=+e93sEC0bpZ`lxdKVp9&48l$I zafA`gkoo3NuCGuXSNE0hZZ4&c>E^e^b}i`dq9RBlcW4&${p6tV@~+FG@oXG1j9K#l z@f!z|0y-$5(9Ve~I1;D0Xc{SiCPYM75NR?TE*I7p8414J{bnn0ewcSPMXP4gaHY4F z;dii|#T~jpwSZ@t(LX&_RWM4q%AncI@G_-c3@T?(W?5N9n6Px+jf7*FNeb?DLz7~r z)(~RPU;g2|e^cUtajurkW!|>2M;obEvWoVjDEX55m6jZEl-$;2p|CG?1qU>x_Um8w zz&HHO)%;IpE5PM5CMP$lsUE97g%~cj(sSj@ zwdPU}Q~$BOtX6EUOS~JyC&lJPn&qb6;=-UdO?_z#4m<)8pjqcDxWA{`AKZhb5Zd-a z-PQA{B8dx>xv4ggH}nD8pw9cUT=Dypo14-V($J)51<>xz)(SSHNgAaI^|VS;tr`8$ z!g8f`Z}8aI7;VE@QIVHDfwJKir=u5zIHn$}Xi|RoSW)H4UHB|CYvKoWw6RJ1m4@2r z+&_6=v~T@y!rVo45_I69mQS#Jf?BQ#*w8F)HIS<4Cr|O%XmSccc=uixBB7x8xP0kk z+uK!r`FzjQ6gJVd^R2egyr!M&YR~m+-`BlS8DU$=Efb1aXjAGlyh*(SWVF(}{51Q) z%$U-A3^YTTusi^AR^I2eS2`&|mO%RW!Q+vGc!EryutNo&&jGwY!CCs3mme1PF zGHM$?QP`Y55U1!;WmY|HQ0XMwFSE?Aa6g&hm#bN(t%0PC=iA3uGT)DCJ$p(B)sU=F zEvgnbo_wC~6D{s#L-ssG7Dn*$^69LakDDG^JoD3K&P3?NN8f+K4iV||S*r{6L~waI zt-9pE1ZA=XTJUqT-g&LLbQY)0}taN`n&0T*JmY2S_Fw`k1 zofAu;{kh!YFy#+o1e5Z|Pz+GgARDFUqYJs1)S5A~SH%dvW!_SfMr4Sz;#$ku1h{Xo4?ump&AkYC&PWE`6m8z{1hgAwXO7-;MHHi_*EOm zgrbhc!}$tpSPyG(sfTSt^Z1!?oVCIt0TqyD|r2%w&Pk)b?O(1sGaZQ z=Pt`h8Qxz%Tb_4N-v-HbNYLnz@#9~*RZE~*GA+Yn&la`Oz^>=tRh4^n6%idlWvt#S z!t91_#Ow5psMD0)D6VNJK+f*p6fZvSfgP>1bbNnbJJStBGbRNbOfD8RT=l$P`h(+a zzK{#@fxV-&3F9igOXMqE8o-Yh<lTFD3jeo5M5 zdF-JL!E)A1Fynfkd#rD_z1S(Q4OP0?KGT#;h0(Wl;P{Q9({WE+#2wleLB%j;9P*%k(Vxk4oT9*7oSV3)NI8!k=m;%LkdKP=4MX4g|?uI zE)znWuo%ItxhU`)PY}hcT*OFSHZ(Q~mhM4zv+cNjxs*H>=A>_n;qv9OvlRrbl(xZi zXk~}0i{pnq`vTdJ({Xsw;@a(wN!xWB>?d!Q0(^!=l>j}u%Co2cxY91TlJ#4(8mP^5 zA=#2oNFbj?_!8=Lyc2%hbF7n|=(vbnq1uz;e3?3sEV1PRb218(=%bU}j#whGSF77Y zlkWZoMM<_{<6+3}Ek9i~<**?EwBXwxvKHksWdSIcIVexWDDB|9pR_=JWhJ?Pi7PLW z@guulAok53Js=jeqSLhN?jd4FUp>g@^7Jk(tXs<{4+g1UNV)-5}oX*b89W&kuaZC zp-&REvw*YsGpm2?%v79+o~ZqmLG;qj3m+2`UtfO%*_Tfkj`d8=HrDMo;wOyKJtD7k zArW}Bys)x%oXB@z=*OWL;0T|yU&e%bF5g&ZQ}+Q?pTv zI+etm{xoo=*>AKt30giT{0fB%$mfqT1VL^Y%JOu=*Ds~(inAbT4`oPWjA463ZOS3G z-I6?(&KN{lB0=DKG!h zH;LMz<_1n+>3qwZnjMj>nI6Sfr1Tw_n&rd$q8$XtfV?EOBsp_(b zsa(P!Lq9?t5)0=a>J-3#`9Nq~9r#l40f0d0P+Yw#&vyFW5dBYmXTnhmczjP|tZ5E8 zHj}(+T`mZLoQHj2Jk-&UHw zp$L*tj$hFA2f|N7tbFrB17JZ)prV<2gSGBtiTUt823)|$Q`fPZ;ZeFDisy|cU1JG$ zp_&ryV4L0YyFinl-6s7JvsKqlX<>8aZwI=xo-K$5cDFW> z@~qVe0`x>b%2ycafpA!2Vf<4uJ4Xu$r=1#T*SjV!0D7X){>6Kzn)0=BYvpNoDk>}} z?&0@35_xEKzB`im2F@CyIpbq;7A?2D!pQV!T|L#1Znn$bT|6qy*6MIRX`3;J=*}(& z(;5#Kq}@V!B=C-vuoi*#J`)CON%PCPV!nIxu;WWVVSLU{C!jr3W0ASi+R*)_SKO(# zAchl>|F<$61X*rtJF?#-d$cChTFEU>UK@M2Nk#dUYC5NlZz3aEHUD}$G24ww-@zfF zXQN#z#^DU;C-cGe9)^l!z|)OHN8F7brRKjRp;L zvOiB?3`%(w1Trycr%vBVm+F`5YFZtU1RQdtjV3mZ%qmdKj#cv9Ub?E5N2;Tq_ou%O zF;kaD1aF)u^sD1j`Sh~oC=i33RJ*T*?rPMK2u{f@p1-nnRNc4Ho_fkNq#bp97H#0{ z@}77Nu^X4}WS{)ITk>#%zVRmyGA;_TKY#XQBg%U8n*7S%HpGxSvU%Lkt?! zrpVwa3#e-wnRx4M%6KXs>B5cmc&Vm&OpmK|-z6VATfDRIj}%9)ub~mbR%Krf*XVE zi@L)-vPjTK(=g%To(J*&yKe8<)& z7iPnJUa?%c?8eGXtszV$>nYN>nh}bJVFci*8uT9skK?Mk`t6i!lJZK|N|e<@a4w$efENGp8XU0;{9p8KY2KFoneda*-FX}hI2LHhxvIu zm#f!sI9FrMJAJb5+CPY~gu~lOhjW=4w=1y287T@^3XS!X&+iCBGDc1gKo1Ag|0XB@ z7lU_a_Y8aSUlJH09f2VEXc$vFUi@Fxb&-|P{XipSWOtk?F3RVVmB|^wwe{GfYO}GO zLLAxhIyw5i!~sIBddjdDzfkVYM`d@h#x~WO}QbZC~OL(TEq#0y6w4VKRJ8O>kiU`$ow_cPz;S(0oapPO}GdnIF(9<`-> z$4qJ_dD@#1;^HE{3dKcF-aJy#X2f|9!weMOCZHqSN&Z{Z28$urleIDw;dlCXn~?Fb zA9o6@)y>`hzZF&r1fMkVT=+1Z&$h&j;AIdTg)y3;Y5rRr|M?Ew0Hzqn)L%ub^}AXu zg~*du_t_8Cd%B!mFXhNpFC~2bZ9h9{M`}JfsqtM(X@pFCGMNlK^?mGfbQIZ+WXwPC zrj_h3XX(SoAAU^~sA0J&9K?9GdKX)wvq+ z5nhau=Ox}#TK%fuC7jt zK7MIe`?i8gF_$#^RH*tu7O=~EzLsk<+NCDA^@9?)oS^Z?mgP%;B6CnQgz z1!XMEH>jp)GY;sUd95*B4$rKwD znrCGpGe8+-s30C${U+GHNxo8lPtCVez*FA@zQdep-?CR-c8fKidKmPap~A6BRD=LP z-!S?z`IP)|e-oM#H)#YP_z=tl=@!PsN$X0?rb9JGlFtZ8Pu6`;^)J8RdjI~uH6uBI zV^;LFLe(cMCrMj>D_W8y2wu`RG?@QWqQY_heVqz?HopK?fx=ArHFk7&S>yA1`6*_F zhbaO5c2xngiYh~P73Iy5>m$Co6)4Vs)^DkU24!6Hi}*ke8|v_o7Yk;|Dy zNA0}RL+3I(K@EsEf~;u~_e=GjQ6Rc7H~ZC?5RIbg=Uqu&h)Cuhe}){Hsk)P890pBo zY6Sz|@Yjhmf{~m^b5@@T+Ni*z%&L?3P(hp8@ z)}8UKy31+5y{!O6oaXwr#ji$d5~A72W#Pq9o%`%XMF6I*JzA^aY z6f&xTIH7(<$ErGbW9*dZP|{j1wm&iZQl@bj0+Az$FsjvXn4@E6%GiBV_Zvei$MFIh zTbkMa!nk^Hi<%+d0qDT2ij^a=A5@ocCbKGBeD3#^N=G&s>ca|r&vcu#_oNi%a5wT? zsH~xofWs9;V|hv|Rt+7)#W;LN2BU0;Pku+gqXO_7uNni5#iT_{!AKPwo#Fi7eM#;U zn?3~wApWd4gAcRvW7zy{+x7bRXonKB(hx*Ev+PS+mu6zH*oAU*ouIV+?i*IC@SnyY zesWwa=iyX(FbYO#qfuo00b3q%k>kPJoHQ4Z^rfm(;p=0a)M{|1wyX-~c7Lcx+eeI~ zAs-vhZCiI%1!>+DYxK(@8%{We&afWn;NK}aCof*veL)CB_7ARWz;$dl^xCZRIgl$n z%=ycLD8z_#L}%>v)Ni9qt`AxxGOQjR%(cvamZbt+L8adDs`Ox1!wyJZpQ zun@V*z_B`JI481dGH1Fyp>ZVTSS)Fv7y@Z82l#dM%HgU5aQkG>`5ipMCMKHO zRK{X3pBQB&Fg=`c+Vb72AQGN=$h!45cZINHotlE{Sn(lvg$Hn`uG8`L3O944^z@;Q zrufkNY))2-!jWR(^l;J`P82pan&+;>VF4S#n!YA@JLR!W6%kv0F{KW=cM`71#gEMw zEFjTxVihW5#4|s{W-Nnidqz@f98X$Xm9js;9N$ps0;Ba+r$$LiI6pXdy{_M}(Rj!6 z%zDW<(!@)e($~lSx$sZVDP>uwTCTL)wOsRN`r{EEpYrVw47yUarbV zSzfc!l9+0zwFs@m92a#npx0*nBFk{p6ajr+W7Y6QTW3M_#gxL=d`YsU`;#L&B!n)t z8Tq|O&#(3h2(dQrs=_e-(IX5_J*5jhJGO~VGuqzYIAGM0?nL&9Rsp znjOBemiXsDf)>;$5d2ThsLxA?&`Zk~&0^YFgE!oKK3cbtc!jO+&)=M~5z@upt(}>k zYCeeeO;&F#eQ{bTV({-7KXG#mTK`&I2;%FY&Hwj$!*!{!El;P&%=%M|@K2o^uT)+u J7X#n<{~w915nliR diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/step/ChildPathStepUnnesting.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/step/ChildPathStepUnnesting.java index 9026de3c7..03b9cf332 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/step/ChildPathStepUnnesting.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/step/ChildPathStepUnnesting.java @@ -18,7 +18,6 @@ import java.io.IOException; -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IPointable; @@ -109,7 +108,7 @@ public boolean step(IPointable result) throws HyracksDataException { * @param result * result * @return found result - * @throws AlgebricksException + * @throws HyracksDataException * Could not save result. */ protected boolean stepNodeTree(TaggedValuePointable tvpInput, int level, IPointable result) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/step/DescendantOrSelfPathStepUnnesting.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/step/DescendantOrSelfPathStepUnnesting.java index 305e26de0..facbee38f 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/step/DescendantOrSelfPathStepUnnesting.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/step/DescendantOrSelfPathStepUnnesting.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.List; -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IPointable; @@ -141,7 +140,7 @@ private boolean processNodeTree(TaggedValuePointable rootTVP, IPointable result) * @param result * result * @return found result - * @throws AlgebricksException + * @throws HyracksDataException * Could not save result. */ protected boolean stepNodeTree(TaggedValuePointable tvpInput, int level, IPointable result) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java index 7c3197a1e..6558274ae 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java @@ -1359,15 +1359,16 @@ public static void writeDateAsString(IDate date, UTF8StringBuilder sb) throws IO } /** - * Writes a number to the DataOutput with zeros as place holders if the number is too small to fill the padding. + * Writes a number to the string builder with zeros as place holders if the number is too small to fill the padding. * * @param valueArg * value * @param paddingArg * padding - * @param dOut - * data output + * @param sb + * string builder * @throws IOException + * Could not save result. */ public static void writeNumberWithPadding(long valueArg, int paddingArg, UTF8StringBuilder sb) throws IOException { long value = valueArg; From 4a38f670cd11231bd18a321cbacdd4f4e109b50a Mon Sep 17 00:00:00 2001 From: Christina Pavlopoulou Date: Tue, 30 May 2017 11:54:31 -0700 Subject: [PATCH 08/14] [VXQUERY-196][VXQUERY-204][VXQUERY-228] Cleaning up indexing query statements 1) Move XPath out of arguments for collection-from-index, leave only the collection path. 2) Hide the index usage from the user by creating a rewrite rule to figure out the existence of index. 3) Remove the unnecessary header tags that collection-from-index creates. --- .../java/org/apache/vxquery/cli/VXQuery.java | 141 ++++++++------ .../apache/vxquery/common/VXQueryCommons.java | 3 +- .../rules/IntroduceCollectionRule.java | 30 ++- .../rewriter/rules/IntroduceIndexingRule.java | 10 +- .../rules/PushChildIntoDataScanRule.java | 7 +- .../rules/PushValueIntoDatascanRule.java | 3 + .../vxquery/functions/builtin-functions.xml | 3 +- .../vxquery/index/IndexDocumentBuilder.java | 14 +- .../VXQueryCollectionOperatorDescriptor.java | 58 +++--- .../metadata/VXQueryIndexingDataSource.java | 16 +- .../VXQueryIndexingOperatorDescriptor.java | 181 ++++++++++-------- .../metadata/VXQueryMetadataProvider.java | 34 +++- .../functions/index/IndexConstructorUtil.java | 69 +++---- ...=> ShowIndexesScalarEvaluatorFactory.java} | 6 +- .../functions/index/VXQueryIndexReader.java | 115 +++++++---- .../IndexCentralizerUtil.java | 54 +++--- .../IndexDirectory.java | 13 +- .../IndexLocator.java | 8 +- .../{updateIndex => update}/Constants.java | 9 +- .../{updateIndex => update}/IndexUpdater.java | 71 +++---- .../{updateIndex => update}/MetaFileUtil.java | 59 +++--- .../{updateIndex => update}/VXQueryIndex.java | 2 +- .../{updateIndex => update}/XmlMetadata.java | 6 +- .../XmlMetadataCollection.java | 2 +- .../json/JnDocScalarEvaluatorFactory.java | 2 +- .../FnDocAvailableScalarEvaluatorFactory.java | 2 +- .../node/FnDocScalarEvaluatorFactory.java | 2 +- .../functions/util/FunctionHelper.java | 21 +- .../vxquery/xmlparser/SAXContentHandler.java | 7 +- .../xmlquery/query/XMLQueryCompiler.java | 4 +- .../vxquery/indexing/MetaFileUtilTest.java | 4 +- .../vxquery/indexing/TestConstants.java | 4 +- .../xmlquery/query/SimpleXQueryTest.java | 2 +- .../org/apache/vxquery/xtest/TestRunner.java | 56 +++++- .../Indexing/Partition-1/useIndex1_user.txt | 2 + .../Indexing/Partition-2/useIndex1_user.txt | 2 + .../Indexing/Partition-4/useIndex1_user.txt | 2 + .../XQuery/Indexing/Partition-1/useIndex1.xq | 4 +- .../Indexing/Partition-1/useIndex1_user.xq | 25 +++ .../XQuery/Indexing/Partition-1/useIndex2.xq | 2 +- .../XQuery/Indexing/Partition-1/useIndex3.xq | 2 +- .../XQuery/Indexing/Partition-1/useIndex4.xq | 2 +- .../XQuery/Indexing/Partition-1/useIndex5.xq | 2 +- .../XQuery/Indexing/Partition-1/useIndex6.xq | 2 +- .../XQuery/Indexing/Partition-1/useIndex7.xq | 4 +- .../XQuery/Indexing/Partition-2/useIndex1.xq | 2 +- .../Indexing/Partition-2/useIndex1_user.xq | 25 +++ .../XQuery/Indexing/Partition-2/useIndex2.xq | 2 +- .../XQuery/Indexing/Partition-2/useIndex3.xq | 2 +- .../XQuery/Indexing/Partition-2/useIndex4.xq | 2 +- .../XQuery/Indexing/Partition-2/useIndex5.xq | 2 +- .../XQuery/Indexing/Partition-2/useIndex6.xq | 2 +- .../XQuery/Indexing/Partition-2/useIndex7.xq | 4 +- .../XQuery/Indexing/Partition-4/useIndex1.xq | 2 +- .../Indexing/Partition-4/useIndex1_user.xq | 25 +++ .../XQuery/Indexing/Partition-4/useIndex2.xq | 2 +- .../XQuery/Indexing/Partition-4/useIndex3.xq | 2 +- .../XQuery/Indexing/Partition-4/useIndex4.xq | 2 +- .../XQuery/Indexing/Partition-4/useIndex5.xq | 2 +- .../XQuery/Indexing/Partition-4/useIndex6.xq | 2 +- .../XQuery/Indexing/Partition-4/useIndex7.xq | 4 +- .../src/test/resources/VXQueryCatalog.xml | 4 +- .../test/resources/cat/IndexingQueries.xml | 15 ++ 63 files changed, 720 insertions(+), 448 deletions(-) rename vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/{ShowIndexScalarEvaluatorFactory.java => ShowIndexesScalarEvaluatorFactory.java} (91%) rename vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/{indexCentralizer => centralizer}/IndexCentralizerUtil.java (78%) rename vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/{indexCentralizer => centralizer}/IndexDirectory.java (89%) rename vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/{indexCentralizer => centralizer}/IndexLocator.java (89%) rename vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/{updateIndex => update}/Constants.java (80%) rename vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/{updateIndex => update}/IndexUpdater.java (87%) rename vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/{updateIndex => update}/MetaFileUtil.java (85%) rename vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/{updateIndex => update}/VXQueryIndex.java (95%) rename vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/{updateIndex => update}/XmlMetadata.java (94%) rename vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/{updateIndex => update}/XmlMetadataCollection.java (97%) create mode 100644 vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt create mode 100644 vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt create mode 100644 vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt create mode 100644 vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq create mode 100644 vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq create mode 100644 vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq diff --git a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java index e0e38433e..25ff9c47d 100644 --- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java +++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java @@ -14,6 +14,7 @@ */ package org.apache.vxquery.cli; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -23,6 +24,7 @@ import java.net.InetAddress; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; @@ -66,12 +68,13 @@ public class VXQuery { private final CmdLineOptions opts; + private final CmdLineOptions indexOpts; private ClusterControllerService cc; private NodeControllerService[] ncs; private IHyracksClientConnection hcc; private IHyracksDataset hds; - + private List collectionList; private ResultSetId resultSetId; private static List timingMessages = new ArrayList<>(); private static long sumTiming; @@ -87,6 +90,16 @@ public class VXQuery { */ public VXQuery(CmdLineOptions opts) { this.opts = opts; + // The index query returns only the result, without any other information. + this.indexOpts = opts; + indexOpts.showAST = false; + indexOpts.showOET = false; + indexOpts.showQuery = false; + indexOpts.showRP = false; + indexOpts.showTET = false; + indexOpts.timing = false; + indexOpts.compileOnly = false; + this.collectionList = new ArrayList(); } /** @@ -168,71 +181,87 @@ private void execute() throws Exception { * @throws SystemException * @throws Exception */ + private void runQueries() throws Exception { - Date start; - Date end; - for (String query : opts.arguments) { - String qStr = slurp(query); - if (opts.showQuery) { - System.err.println(qStr); + List queries = opts.arguments; + // Run the showIndexes query before executing any target query, to store the index metadata + List queriesIndex = new ArrayList(); + queriesIndex.add("vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/showIndexes.xq"); + OutputStream resultStream = new ByteArrayOutputStream(); + executeQuery(queriesIndex.get(0), 1, resultStream, indexOpts); + ByteArrayOutputStream bos = (ByteArrayOutputStream) resultStream; + String result = new String(bos.toByteArray()); + String[] collections = result.split("\n"); + this.collectionList = Arrays.asList(collections); + executeQueries(queries); + } + + public void executeQueries(List queries) throws Exception { + for (String query : queries) { + OutputStream resultStream = System.out; + if (opts.resultFile != null) { + resultStream = new FileOutputStream(new File(opts.resultFile)); } + executeQuery(query, opts.repeatExec, resultStream, opts); + } + } - VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, - opts.showOET, opts.showRP); + public void executeQuery(String query, int repeatedExecution, OutputStream resultStream, CmdLineOptions options) + throws Exception { + PrintWriter writer = new PrintWriter(resultStream, true); + String qStr = slurp(query); + if (opts.showQuery) { + writer.println(qStr); + } + VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, opts.showOET, + opts.showRP); - start = opts.timing ? new Date() : null; + Date start = opts.timing ? new Date() : null; - Map nodeControllerInfos = null; - if (hcc != null) { - nodeControllerInfos = hcc.getNodeControllerInfos(); - } - XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize, - opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf); - resultSetId = createResultSetId(); - CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE), - resultSetId, null); - compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel); - // if -timing argument passed, show the starting and ending times - if (opts.timing) { - end = new Date(); - timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms"); - } - if (opts.compileOnly) { - continue; - } - - Module module = compiler.getModule(); - JobSpecification js = module.getHyracksJobSpecification(); + Map nodeControllerInfos = null; + if (hcc != null) { + nodeControllerInfos = hcc.getNodeControllerInfos(); + } + XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize, + opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf); + resultSetId = createResultSetId(); + CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE), + resultSetId, null); + compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel, this.collectionList); + // if -timing argument passed, show the starting and ending times + Date end = opts.timing ? new Date() : null; + if (opts.timing) { + timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms"); + } + if (opts.compileOnly) { + return; + } - DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext()); - js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory())); + Module module = compiler.getModule(); + JobSpecification js = module.getHyracksJobSpecification(); - OutputStream resultStream = System.out; - if (opts.resultFile != null) { - resultStream = new FileOutputStream(new File(opts.resultFile)); - } + DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext()); + js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory())); - PrintWriter writer = new PrintWriter(resultStream, true); - // Repeat execution for number of times provided in -repeatexec argument - for (int i = 0; i < opts.repeatExec; ++i) { - start = opts.timing ? new Date() : null; - runJob(js, writer); - // if -timing argument passed, show the starting and ending times - if (opts.timing) { - end = new Date(); - long currentRun = end.getTime() - start.getTime(); - if ((i + 1) > opts.timingIgnoreQueries) { - sumTiming += currentRun; - sumSquaredTiming += currentRun * currentRun; - if (currentRun < minTiming) { - minTiming = currentRun; - } - if (maxTiming < currentRun) { - maxTiming = currentRun; - } + // Repeat execution for number of times provided in -repeatexec argument + for (int i = 0; i < repeatedExecution; ++i) { + start = opts.timing ? new Date() : null; + runJob(js, writer); + // if -timing argument passed, show the starting and ending times + if (opts.timing) { + end = new Date(); + long currentRun = end.getTime() - start.getTime(); + if ((i + 1) > opts.timingIgnoreQueries) { + sumTiming += currentRun; + sumSquaredTiming += currentRun * currentRun; + if (currentRun < minTiming) { + minTiming = currentRun; + } + if (maxTiming < currentRun) { + maxTiming = currentRun; } - timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms"); } + timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms"); } } } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java b/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java index ceaf3c79d..400fb157a 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/common/VXQueryCommons.java @@ -35,7 +35,8 @@ public class VXQueryCommons { static { indexingFunctions.add(BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier()); - indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier()); + indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier()); + indexingFunctions.add(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_1.getFunctionIdentifier()); indexingFunctions.add(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier()); indexingFunctions.add(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier()); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java index 20283d836..11d379580 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java @@ -16,12 +16,16 @@ */ package org.apache.vxquery.compiler.rewriter.rules; +import java.util.ArrayList; + import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; import org.apache.vxquery.common.VXQueryCommons; import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext; import org.apache.vxquery.metadata.VXQueryCollectionDataSource; +import org.apache.vxquery.metadata.VXQueryIndexingDataSource; +import org.apache.vxquery.metadata.VXQueryMetadataProvider; import org.apache.vxquery.types.AnyItemType; import org.apache.vxquery.types.Quantifier; import org.apache.vxquery.types.SequenceType; @@ -61,11 +65,35 @@ public class IntroduceCollectionRule extends AbstractCollectionRule { public boolean rewritePre(Mutable opRef, IOptimizationContext context) { VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context; String[] args = getFunctionalArguments(opRef, VXQueryCommons.collectionFunctions); - + VXQueryMetadataProvider metadata = (VXQueryMetadataProvider) context.getMetadataProvider(); if (args != null) { String collectionName = args[0]; // Build the new operator and update the query plan. int collectionId = vxqueryContext.newCollectionId(); + ArrayList collectionTempName = new ArrayList(); + collectionTempName.add(collectionName); + if (collectionName.contains("|")) { + collectionTempName.remove(0); + int index = collectionName.indexOf("|"); + int start = 0; + while (index >= 0) { + collectionTempName.add(collectionName.substring(start, index)); + start = index + 1; + index = collectionName.indexOf("|", index + 1); + if (index == -1) { + collectionTempName.add(collectionName.substring(start)); + } + } + } + if (metadata.hasIndex(collectionTempName)) { + VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collectionName, + SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR), + functionCall.getFunctionIdentifier().getName()); + if (ids != null) { + ids.setTotalDataSources(vxqueryContext.getTotalDataSources()); + return setDataSourceScan(ids, opRef); + } + } VXQueryCollectionDataSource ds = VXQueryCollectionDataSource.create(collectionId, collectionName, SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR)); if (ds != null) { diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java index 5b961317a..6e60d7584 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceIndexingRule.java @@ -33,19 +33,19 @@ public class IntroduceIndexingRule extends AbstractCollectionRule { @Override - public boolean rewritePre(Mutable opRef, IOptimizationContext context) throws AlgebricksException { + public boolean rewritePre(Mutable opRef, IOptimizationContext context) + throws AlgebricksException { VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context; String args[] = getFunctionalArguments(opRef, VXQueryCommons.indexingFunctions); if (args != null) { String collection = args[0]; - String elementPath = args.length > 1?args[1]:null; - // Build the new operator and update the query plan. int collectionId = vxqueryContext.newCollectionId(); - VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collection, elementPath, - SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR), functionCall.getFunctionIdentifier().getName()); + VXQueryIndexingDataSource ids = VXQueryIndexingDataSource.create(collectionId, collection, + SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR), + functionCall.getFunctionIdentifier().getName()); if (ids != null) { ids.setTotalDataSources(vxqueryContext.getTotalDataSources()); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java index dbcce54c6..2773154eb 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushChildIntoDataScanRule.java @@ -55,12 +55,9 @@ public class PushChildIntoDataScanRule extends AbstractPushExpressionIntoDatasca @Override boolean updateDataSource(IVXQueryDataSource datasource, Mutable expression) { - //TODO: indexing needs to be extended to support push child into datascan - if (datasource.usingIndex()) { - return false; - } - boolean added = false; List> finds = new ArrayList>(); + boolean added = false; + ExpressionToolbox.findAllFunctionExpressions(expression, BuiltinOperators.CHILD.getFunctionIdentifier(), finds); for (int i = finds.size(); i > 0; --i) { int typeId = ExpressionToolbox.getTypeExpressionTypeArgument(finds.get(i - 1)); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java index 1d8a55d0b..b9014690d 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/PushValueIntoDatascanRule.java @@ -59,6 +59,9 @@ public class PushValueIntoDatascanRule extends AbstractPushExpressionIntoDatasca @Override boolean updateDataSource(IVXQueryDataSource datasource, Mutable expression) { + if (datasource.usingIndex()) { + return false; + } VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) datasource; boolean added = false; List> finds = new ArrayList>(); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml index d64f42398..4932a781e 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml +++ b/vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml @@ -152,7 +152,6 @@ - @@ -166,7 +165,7 @@ - + diff --git a/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java b/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java index 7524da4a2..1df31dd78 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/index/IndexDocumentBuilder.java @@ -22,7 +22,6 @@ import java.util.List; import org.apache.commons.lang3.StringUtils; -import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.data.std.primitive.BooleanPointable; import org.apache.hyracks.data.std.primitive.BytePointable; import org.apache.hyracks.data.std.primitive.DoublePointable; @@ -56,12 +55,10 @@ import org.apache.vxquery.datamodel.accessors.nodes.TextOrCommentNodePointable; import org.apache.vxquery.datamodel.values.ValueTag; import org.apache.vxquery.runtime.functions.cast.CastToStringOperation; -import org.apache.vxquery.runtime.functions.index.updateIndex.Constants; +import org.apache.vxquery.runtime.functions.index.update.Constants; import org.apache.vxquery.serializer.XMLSerializer; public class IndexDocumentBuilder extends XMLSerializer { - private final IPointable treePointable; - private final PointablePool pp; private NodeTreePointable ntp; @@ -88,16 +85,11 @@ public ComplexItem(StringField sfin, String idin) { } //TODO: Handle Processing Instructions, PrefixedNames, and Namepsace entries - public IndexDocumentBuilder(IPointable tree, IndexWriter inWriter, String file) { - this.treePointable = tree; + public IndexDocumentBuilder(TaggedValuePointable tvp, IndexWriter inWriter, String file) { writer = inWriter; this.filePath = file; - //convert to tagged value pointable - TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); - tvp.set(treePointable.getByteArray(), 0, treePointable.getLength()); - //get bytes and info from doc pointer bstart = tvp.getByteArray(); sstart = tvp.getStartOffset(); @@ -105,7 +97,7 @@ public IndexDocumentBuilder(IPointable tree, IndexWriter inWriter, String file) doc = new Document(); - results = new ArrayList(); + results = new ArrayList<>(); pp = PointablePoolFactory.INSTANCE.createPointablePool(); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index 623b48cf5..a3756d5b4 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -59,7 +59,6 @@ import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender; import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor; -import org.apache.hyracks.dataflow.common.comm.util.FrameUtils; import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor; import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable; import org.apache.hyracks.hdfs.ContextFactory; @@ -129,38 +128,14 @@ public void open() throws HyracksDataException { public void nextFrame(ByteBuffer buffer) throws HyracksDataException { fta.reset(buffer); String collectionModifiedName = collectionName.replace("${nodeId}", nodeId); - Reader input; + if (!collectionModifiedName.contains("hdfs:/")) { File collectionDirectory = new File(collectionModifiedName); // check if directory is in the local file system if (collectionDirectory.exists()) { // Go through each tuple. if (collectionDirectory.isDirectory()) { - for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) { - Iterator it = FileUtils.iterateFiles(collectionDirectory, - new VXQueryIOFileFilter(), TrueFileFilter.INSTANCE); - while (it.hasNext()) { - File file = it.next(); - String fileName = file.getName().toLowerCase(); - if (fileName.endsWith(".xml")) { - if (LOGGER.isLoggable(Level.FINE)) { - LOGGER.fine("Starting to read XML document: " + file.getAbsolutePath()); - } - parser.parseElements(file, writer, tupleIndex); - } else if (fileName.endsWith(".json")) { - if (LOGGER.isLoggable(Level.FINE)) { - LOGGER.fine("Starting to read JSON document: " + file.getAbsolutePath()); - } - try { - jsonAbvs.reset(); - input = new InputStreamReader(new FileInputStream(file)); - jparser.parse(input, jsonAbvs, writer, appender); - } catch (FileNotFoundException e) { - throw new HyracksDataException(e.toString()); - } - } - } - } + xmlAndJsonCollection(collectionDirectory); } else { throw new HyracksDataException("Invalid directory parameter (" + nodeId + ":" + collectionDirectory.getAbsolutePath() + ") passed to collection."); @@ -272,6 +247,35 @@ public void nextFrame(ByteBuffer buffer) throws HyracksDataException { } } + public void xmlAndJsonCollection(File directory) throws HyracksDataException { + Reader input; + for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) { + Iterator it = FileUtils.iterateFiles(directory, new VXQueryIOFileFilter(), + TrueFileFilter.INSTANCE); + while (it.hasNext()) { + File file = it.next(); + String fileName = file.getName().toLowerCase(); + if (fileName.endsWith(".xml")) { + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("Starting to read XML document: " + file.getAbsolutePath()); + } + parser.parseElements(file, writer, tupleIndex); + } else if (fileName.endsWith(".json")) { + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("Starting to read JSON document: " + file.getAbsolutePath()); + } + try { + jsonAbvs.reset(); + input = new InputStreamReader(new FileInputStream(file)); + jparser.parse(input, jsonAbvs, writer, appender); + } catch (FileNotFoundException e) { + throw new HyracksDataException(e.toString()); + } + } + } + } + } + @Override public void fail() throws HyracksDataException { writer.fail(); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java index ea69cfd0b..d55530dcf 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingDataSource.java @@ -31,14 +31,11 @@ */ public class VXQueryIndexingDataSource extends AbstractVXQueryDataSource { - private String elementPath; private String function; - private VXQueryIndexingDataSource(int id, String collection, String elementPath, Object[] types, - String functionCall) { + private VXQueryIndexingDataSource(int id, String collection, Object[] types, String functionCall) { this.dataSourceId = id; this.collectionName = collection; - this.elementPath = elementPath; this.function = functionCall; this.collectionPartitions = collectionName.split(DELIMITER); this.types = types; @@ -56,13 +53,8 @@ public IPhysicalPropertiesVector computePropertiesVector(List s this.valueSeq = new ArrayList<>(); } - public static VXQueryIndexingDataSource create(int id, String collection, String index, Object type, - String function) { - return new VXQueryIndexingDataSource(id, collection, index, new Object[] { type }, function); - } - - public String getElementPath() { - return elementPath; + public static VXQueryIndexingDataSource create(int id, String collection, Object type, String function) { + return new VXQueryIndexingDataSource(id, collection, new Object[] { type }, function); } public String getFunctionCall() { @@ -71,7 +63,7 @@ public String getFunctionCall() { @Override public String toString() { - return "VXQueryIndexingDataSource [collectionName=" + collectionName + ", elementPath=" + elementPath + return "VXQueryIndexingDataSource [collectionName=" + collectionName + ", elementPath=" + this.childSeq + ", function=" + function + "]"; } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java index ac92a0eed..356371340 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java @@ -16,15 +16,13 @@ */ package org.apache.vxquery.metadata; -import java.io.DataInputStream; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Map; +import java.util.List; import java.util.logging.Logger; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -import org.apache.hyracks.api.client.NodeControllerInfo; import org.apache.hyracks.api.comm.IFrame; import org.apache.hyracks.api.comm.IFrameFieldAppender; import org.apache.hyracks.api.comm.VSizeFrame; @@ -35,23 +33,20 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.data.std.api.IPointable; -import org.apache.hyracks.data.std.primitive.UTF8StringPointable; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender; import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor; -import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; import org.apache.hyracks.dataflow.common.comm.util.FrameUtils; import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor; import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable; import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; -import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; import org.apache.vxquery.datamodel.values.XDMConstants; import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.functions.BuiltinFunctions; import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil; import org.apache.vxquery.runtime.functions.index.VXQueryIndexReader; -import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil; -import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater; +import org.apache.vxquery.runtime.functions.index.centralizer.IndexCentralizerUtil; +import org.apache.vxquery.runtime.functions.index.update.IndexUpdater; import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; import org.apache.vxquery.xmlparser.TreeNodeIdProvider; @@ -61,18 +56,18 @@ public class VXQueryIndexingOperatorDescriptor extends AbstractSingleActivityOpe private short dataSourceId; private short totalDataSources; private String[] collectionPartitions; - private String elementPath; private final String functionCall; + private List childSeq; public VXQueryIndexingOperatorDescriptor(IOperatorDescriptorRegistry spec, VXQueryIndexingDataSource ds, - RecordDescriptor rDesc, String hdfsConf, Map nodeControllerInfos) { + RecordDescriptor rDesc) { super(spec, 1, 1); this.functionCall = ds.getFunctionCall(); collectionPartitions = ds.getPartitions(); dataSourceId = (short) ds.getDataSourceId(); totalDataSources = (short) ds.getTotalDataSources(); recordDescriptors[0] = rDesc; - this.elementPath = ds.getElementPath(); + childSeq = ds.getChildSeq(); } @Override @@ -87,10 +82,11 @@ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources); final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); final String collectionName = collectionPartitions[partition % collectionPartitions.length]; - String collectionModifiedName = collectionName.replace("${nodeId}", nodeId); + final String collectionModifiedName = collectionName.replace("${nodeId}", nodeId); IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil( ctx.getIOManager().getIODevices().get(0).getMount()); indexCentralizerUtil.readIndexDirectory(); + final IPointable result = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); return new AbstractUnaryInputUnaryOutputOperatorNodePushable() { @Override @@ -103,100 +99,115 @@ public void open() throws HyracksDataException { public void nextFrame(ByteBuffer buffer) throws HyracksDataException { fta.reset(buffer); - IPointable result = new TaggedValuePointable(); - - final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); - final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY - .createPointable(); - - final ByteBufferInputStream bbis = new ByteBufferInputStream(); - final DataInputStream di = new DataInputStream(bbis); - final SequenceBuilder sb = new SequenceBuilder(); final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage(); - String indexModifiedName; + abvs.reset(); + abvsFileNode.reset(); + if (collectionModifiedName.contains("hdfs://")) { throw new HyracksDataException("Indexing support for HDFS not yet implemented."); } else { if (functionCall.equals( BuiltinFunctions.FN_BUILD_INDEX_ON_COLLECTION_1.getFunctionIdentifier().getName())) { - indexModifiedName = indexCentralizerUtil.putIndexForCollection(collectionModifiedName); - File collectionDirectory = new File(collectionModifiedName); - - //check if directory is in the local file system - if (collectionDirectory.exists() && collectionDirectory.isDirectory()) { - IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil(); - try { - indexConstructorUtil.evaluate(collectionModifiedName, indexModifiedName, result, - stringp, bbis, di, sb, abvs, nodeIdProvider, abvsFileNode, nodep, false, - nodeId); - XDMConstants.setTrue(result); - FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), - result.getStartOffset(), result.getLength()); - } catch (SystemException e) { - throw new HyracksDataException("Could not create index for collection: " - + collectionName + " in dir: " + indexModifiedName + " " + e.getMessage()); - } - } else { - throw new HyracksDataException("Cannot find Collection Directory (" + nodeId + ":" - + collectionDirectory.getAbsolutePath() + ")"); - } - } else if (functionCall - .equals(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier().getName())) { - indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName); - IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs, - nodeIdProvider, abvsFileNode, nodep, nodeId); try { - updater.setup(); - updater.updateIndex(); - updater.updateMetadataFile(); - updater.exit(); - XDMConstants.setTrue(result); - FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), - result.getStartOffset(), result.getLength()); + createIndex(result, abvs, abvsFileNode); } catch (IOException e) { - throw new HyracksDataException( - "Could not update index in " + indexModifiedName + " " + e.getMessage()); + throw new HyracksDataException(e); } + } else if (functionCall + .equals(BuiltinFunctions.FN_UPDATE_INDEX_1.getFunctionIdentifier().getName())) { + updateIndex(result, abvs, abvsFileNode); } else if (functionCall .equals(BuiltinFunctions.FN_DELETE_INDEX_1.getFunctionIdentifier().getName())) { - indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName); - IndexUpdater updater = new IndexUpdater(indexModifiedName, result, stringp, bbis, di, sb, abvs, - nodeIdProvider, abvsFileNode, nodep, nodeId); - indexCentralizerUtil.deleteEntryForCollection(collectionModifiedName); - try { - updater.setup(); - updater.deleteAllIndexes(); - XDMConstants.setTrue(result); - FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), - result.getStartOffset(), result.getLength()); - } catch (IOException e) { - throw new HyracksDataException( - "Could not delete index in " + indexModifiedName + " " + e.getMessage()); - } - + deleteIndex(result, abvs, abvsFileNode); } else if (functionCall - .equals(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_2.getFunctionIdentifier().getName())) { - indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName); - VXQueryIndexReader indexReader = new VXQueryIndexReader(ctx, indexModifiedName, elementPath); - try { - indexReader.init(); - while (indexReader.step(result)) { - FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), - result.getStartOffset(), result.getLength()); - } - } catch (AlgebricksException e) { - throw new HyracksDataException("Could not read index."); - } - + .equals(BuiltinFunctions.FN_COLLECTION_FROM_INDEX_1.getFunctionIdentifier().getName()) + || functionCall + .equals(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier().getName())) { + usingIndex(result); } else { throw new HyracksDataException("Unsupported function call (" + functionCall + ")"); } } } + public void createIndex(IPointable result, ArrayBackedValueStorage abvs, + ArrayBackedValueStorage abvsFileNode) throws IOException { + String indexModifiedName = indexCentralizerUtil.putIndexForCollection(collectionModifiedName); + File collectionDirectory = new File(collectionModifiedName); + + //check if directory is in the local file system + if (collectionDirectory.exists() && collectionDirectory.isDirectory()) { + IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil(); + try { + indexConstructorUtil.evaluate(collectionModifiedName, indexModifiedName, result, abvs, + nodeIdProvider, abvsFileNode, false, nodeId); + XDMConstants.setTrue(result); + FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(), + result.getLength()); + } catch (SystemException e) { + throw new HyracksDataException("Could not create index for collection: " + collectionName + + " in dir: " + indexModifiedName + " " + e.getMessage(), e); + } + } else { + throw new HyracksDataException("Cannot find Collection Directory (" + nodeId + ":" + + collectionDirectory.getAbsolutePath() + ")"); + } + } + + public void updateIndex(IPointable result, ArrayBackedValueStorage abvs, + ArrayBackedValueStorage abvsFileNode) throws HyracksDataException { + String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName); + IndexUpdater updater = new IndexUpdater(indexModifiedName, result, abvs, nodeIdProvider, abvsFileNode, + nodeId); + try { + updater.setup(); + updater.updateIndex(); + updater.updateMetadataFile(); + updater.exit(); + XDMConstants.setTrue(result); + FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(), + result.getLength()); + } catch (IOException e) { + throw new HyracksDataException( + "Could not update index in " + indexModifiedName + " " + e.getMessage(), e); + } + } + + public void deleteIndex(IPointable result, ArrayBackedValueStorage abvs, + ArrayBackedValueStorage abvsFileNode) throws HyracksDataException { + String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName); + IndexUpdater updater = new IndexUpdater(indexModifiedName, result, abvs, nodeIdProvider, abvsFileNode, + nodeId); + indexCentralizerUtil.deleteEntryForCollection(collectionModifiedName); + try { + updater.setup(); + updater.deleteAllIndexes(); + XDMConstants.setTrue(result); + FrameUtils.appendFieldToWriter(writer, appender, result.getByteArray(), result.getStartOffset(), + result.getLength()); + } catch (IOException e) { + throw new HyracksDataException( + "Could not delete index in " + indexModifiedName + " " + e.getMessage(), e); + } + } + + public void usingIndex(IPointable result) throws HyracksDataException { + String indexModifiedName = indexCentralizerUtil.getIndexForCollection(collectionModifiedName); + VXQueryIndexReader indexReader = new VXQueryIndexReader(ctx, indexModifiedName, childSeq, appender); + try { + indexReader.init(); + for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) { + while (indexReader.step(result, writer, tupleIndex)) { + } + } + } catch (AlgebricksException e) { + throw new HyracksDataException("Could not read index.", e); + } + } + @Override public void fail() throws HyracksDataException { writer.fail(); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java index f6644d6f6..5bb9d1adf 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java @@ -59,15 +59,17 @@ public class VXQueryMetadataProvider implements IMetadataProvider sourceFileMap; private final StaticContext staticCtx; private final String hdfsConf; + private final List collections; private final Map nodeControllerInfos; public VXQueryMetadataProvider(String[] nodeList, Map sourceFileMap, StaticContext staticCtx, - String hdfsConf, Map nodeControllerInfos) { + String hdfsConf, Map nodeControllerInfos, List collections) { this.nodeList = nodeList; this.sourceFileMap = sourceFileMap; this.staticCtx = staticCtx; this.hdfsConf = hdfsConf; this.nodeControllerInfos = nodeControllerInfos; + this.collections = collections; } @Override @@ -111,8 +113,7 @@ public Pair getScannerRuntim constraint = getClusterLocations(nodeList, ds.getPartitionCount()); } else { rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]); - scanner = new VXQueryIndexingOperatorDescriptor(jobSpec, (VXQueryIndexingDataSource) ds, rDesc, - this.hdfsConf, this.nodeControllerInfos); + scanner = new VXQueryIndexingOperatorDescriptor(jobSpec, (VXQueryIndexingDataSource) ds, rDesc); constraint = getClusterLocations(nodeList, ds.getPartitionCount()); } @@ -142,7 +143,7 @@ public static AlgebricksAbsolutePartitionConstraint getClusterLocations(String[] @Override public Pair getWriteFileRuntime(IDataSink sink, int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc) - throws AlgebricksException { + throws AlgebricksException { throw new UnsupportedOperationException(); } @@ -168,7 +169,7 @@ public Pair getIndexDeleteRu IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List primaryKeys, List secondaryKeys, List additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec) - throws AlgebricksException { + throws AlgebricksException { throw new UnsupportedOperationException(); } @@ -234,7 +235,7 @@ public Pair getTokenizerRunt JobGenContext context, JobSpecification spec, boolean bulkload) throws AlgebricksException { throw new UnsupportedOperationException(); } - + @Override public Pair getInsertRuntime(IDataSource dataSource, IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List keys, @@ -243,7 +244,7 @@ public Pair getInsertRuntime JobSpecification jobSpec, boolean bulkload) throws AlgebricksException { throw new UnsupportedOperationException(); } - + @Override public Pair getUpsertRuntime(IDataSource dataSource, IOperatorSchema inputSchema, IVariableTypeEnvironment typeEnv, List keys, @@ -252,7 +253,7 @@ public Pair getUpsertRuntime JobSpecification jobSpec) throws AlgebricksException { throw new UnsupportedOperationException(); } - + @Override public Pair getIndexUpsertRuntime( IDataSourceIndex dataSourceIndex, IOperatorSchema propagatedSchema, @@ -263,11 +264,26 @@ public Pair getIndexUpsertRu JobSpecification spec) throws AlgebricksException { throw new UnsupportedOperationException(); } - + @Override public Map getConfig() { return new HashMap<>(); } + public List getIndexCollections() { + return collections; + + } + + public boolean hasIndex(ArrayList collections) { + boolean indexExists = false; + for (String collection : collections) { + indexExists = getIndexCollections().contains(collection); + if (!indexExists) { + break; + } + } + return indexExists; + } } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java index 470649669..2c2575254 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java @@ -16,10 +16,14 @@ */ package org.apache.vxquery.runtime.functions.index; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.concurrent.ConcurrentHashMap; + import org.apache.hyracks.data.std.api.IPointable; -import org.apache.hyracks.data.std.primitive.UTF8StringPointable; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; -import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -31,38 +35,31 @@ import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.index.IndexDocumentBuilder; -import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil; -import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata; +import org.apache.vxquery.runtime.functions.index.update.MetaFileUtil; +import org.apache.vxquery.runtime.functions.index.update.XmlMetadata; import org.apache.vxquery.runtime.functions.util.FunctionHelper; import org.apache.vxquery.xmlparser.IParser; import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; import org.apache.vxquery.xmlparser.XMLParser; -import java.io.DataInputStream; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.util.concurrent.ConcurrentHashMap; - public class IndexConstructorUtil { - boolean isMetaFilePresent = false; - MetaFileUtil metaFileUtil; - ConcurrentHashMap metadataMap = new ConcurrentHashMap<>(); + private final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); + private final SequenceBuilder sb = new SequenceBuilder(); + private boolean isMetaFilePresent = false; + private MetaFileUtil metaFileUtil; + private ConcurrentHashMap metadataMap = new ConcurrentHashMap<>(); - public void evaluate(String collectioFolder, String indexFolder, IPointable result, UTF8StringPointable - stringp, ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, - ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep, - boolean isElementPath, String nodeId) throws SystemException { + public void evaluate(String collectioFolder, String indexFolder, IPointable result, ArrayBackedValueStorage abvs, + ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, boolean isElementPath, + String nodeId) throws IOException { - metaFileUtil = new MetaFileUtil(indexFolder); -// metaFileUtil = .create(indexFolder); - isMetaFilePresent = metaFileUtil.isMetaFilePresent(); - metaFileUtil.setCollection(collectioFolder); + metaFileUtil = new MetaFileUtil(indexFolder); + isMetaFilePresent = metaFileUtil.isMetaFilePresent(); + metaFileUtil.setCollection(collectioFolder); File collectionDirectory = new File(collectioFolder); if (!collectionDirectory.exists()) { - throw new RuntimeException("The collection directory (" + collectioFolder + ") does not exist."); + throw new IOException("The collection directory (" + collectioFolder + ") does not exist."); } try { @@ -80,8 +77,7 @@ public void evaluate(String collectioFolder, String indexFolder, IPointable resu IndexWriter writer = new IndexWriter(dir, iwc); //Add files to index - indexXmlFiles(collectionDirectory, writer, isElementPath, nodep, abvsFileNode, nodeIdProvider, sb, bbis, di, - nodeId); + indexXmlFiles(collectionDirectory, writer, isElementPath, abvsFileNode, nodeIdProvider, sb, nodeId); if (!isMetaFilePresent) { // Write metadata map to a file. @@ -101,14 +97,13 @@ public void evaluate(String collectioFolder, String indexFolder, IPointable resu } } - /*This function goes recursively one file at a time. First it turns the file into an ABVS document node, then + /* + * This function goes recursively one file at a time. First it turns the file into an ABVS document node, then * it indexes that document node. */ public void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath, - TaggedValuePointable nodep, ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, - SequenceBuilder sb, ByteBufferInputStream bbis, DataInputStream di, String nodeId) - throws SystemException, IOException { - + ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, SequenceBuilder sb, String nodeId) + throws IOException { SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy, HH:mm:ss"); for (File file : collectionDirectory.listFiles()) { @@ -116,8 +111,7 @@ public void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean if (readableXmlFile(file.getPath())) { abvsFileNode.reset(); - IndexDocumentBuilder ibuilder = getIndexBuilder(file, writer, nodep, abvsFileNode, nodeIdProvider, bbis, - di, nodeId); + IndexDocumentBuilder ibuilder = getIndexBuilder(file, writer, abvsFileNode, nodeIdProvider, nodeId); ibuilder.printStart(); if (!isMetaFilePresent) { @@ -131,22 +125,21 @@ public void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean } else if (file.isDirectory()) { // Consider all XML file in sub directories. - indexXmlFiles(file, writer, isElementPath, nodep, abvsFileNode, nodeIdProvider, sb, bbis, di, nodeId); + indexXmlFiles(file, writer, isElementPath, abvsFileNode, nodeIdProvider, sb, nodeId); } } } public boolean readableXmlFile(String path) { - return (path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz")); + return path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz"); } - public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep, - ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, ByteBufferInputStream bbis, - DataInputStream di, String nodeId) throws IOException { + public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, ArrayBackedValueStorage abvsFileNode, + ITreeNodeIdProvider nodeIdProvider, String nodeId) throws IOException { //Get the document node IParser parser = new XMLParser(false, nodeIdProvider, nodeId); - FunctionHelper.readInDocFromString(file.getPath(), bbis, di, abvsFileNode, parser); + FunctionHelper.readInDocFromString(file.getPath(), abvsFileNode, parser); nodep.set(abvsFileNode.getByteArray(), abvsFileNode.getStartOffset(), abvsFileNode.getLength()); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java similarity index 91% rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java index 6677bd90a..6b18b3305 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java @@ -30,12 +30,12 @@ import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator; import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory; -import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil; +import org.apache.vxquery.runtime.functions.index.centralizer.IndexCentralizerUtil; -public class ShowIndexScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory { +public class ShowIndexesScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory { private static final long serialVersionUID = 1L; - public ShowIndexScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) { + public ShowIndexesScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) { super(args); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java index 875084958..cf781ab4d 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java @@ -16,11 +16,17 @@ */ package org.apache.vxquery.runtime.functions.index; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.api.comm.IFrameFieldAppender; +import org.apache.hyracks.api.comm.IFrameWriter; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; -import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; @@ -32,21 +38,21 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.FSDirectory; +import org.apache.vxquery.context.DynamicContext; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.index.IndexAttributes; +import org.apache.vxquery.runtime.functions.util.FunctionHelper; +import org.apache.vxquery.types.ElementType; +import org.apache.vxquery.types.NameTest; +import org.apache.vxquery.types.NodeType; +import org.apache.vxquery.types.SequenceType; import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; import org.apache.vxquery.xmlparser.SAXContentHandler; import org.apache.vxquery.xmlparser.TreeNodeIdProvider; import org.xml.sax.Attributes; import org.xml.sax.SAXException; -import java.io.DataInputStream; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; - public class VXQueryIndexReader { private ArrayBackedValueStorage nodeAbvs = new ArrayBackedValueStorage(); @@ -55,10 +61,7 @@ public class VXQueryIndexReader { private int indexLength; private String elementPath; private String indexName; - - private ByteBufferInputStream bbis = new ByteBufferInputStream(); - private DataInputStream di = new DataInputStream(bbis); - + private List childSequenceTypes; private IndexReader reader; private IndexSearcher searcher; private QueryParser parser; @@ -68,14 +71,40 @@ public class VXQueryIndexReader { private Document doc; private List fields; private IHyracksTaskContext ctx; + private String[] childLocalName = null; + private IFrameFieldAppender appender; + private boolean firstElement; - public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, String elementPath) { + public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, List childSeq, + IFrameFieldAppender appender) { this.ctx = context; this.indexName = indexPath; - this.elementPath = elementPath; + this.appender = appender; + final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); + childSequenceTypes = new ArrayList<>(); + for (int typeCode : childSeq) { + childSequenceTypes.add(dCtx.getStaticContext().lookupSequenceType(typeCode)); + } + childLocalName = new String[childSequenceTypes.size()]; + int index = 0; + StringBuilder stb = new StringBuilder(); + stb.append("/"); + for (SequenceType sType : childSequenceTypes) { + NodeType nodeType = (NodeType) sType.getItemType(); + ElementType eType = (ElementType) nodeType; + NameTest nameTest = eType.getNameTest(); + childLocalName[index] = FunctionHelper.getStringFromBytes(nameTest.getLocalName()); + + stb.append(childLocalName[index]); + if (index != childSequenceTypes.size() - 1) { + stb.append("/"); + } + ++index; + } + elementPath = stb.toString(); } - public boolean step(IPointable result) throws AlgebricksException { + public boolean step(IPointable result, IFrameWriter writer, int tupleIndex) throws AlgebricksException { /*each step will create a tuple for a single xml file * This is done using the parse function * checkoverflow is used throughout. This is because memory might not be @@ -88,6 +117,8 @@ public boolean step(IPointable result) throws AlgebricksException { //TODO: now we get back the entire document doc = searcher.doc(hits[indexPlace].doc); fields = doc.getFields(); + handler.setupElementWriter(writer, tupleIndex); + this.firstElement = true; parse(nodeAbvs); } catch (IOException e) { throw new AlgebricksException(e); @@ -103,7 +134,7 @@ public void init() throws SystemException { int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); - handler = new SAXContentHandler(false, nodeIdProvider, true); + handler = new SAXContentHandler(false, nodeIdProvider, appender, childSequenceTypes); nodeAbvs.reset(); indexPlace = 0; @@ -125,7 +156,7 @@ public void init() throws SystemException { String queryString = elementPath.replaceAll("/", "."); queryString = "item:" + queryString + "*"; - int lastslash = elementPath.lastIndexOf("/"); + int lastslash = elementPath.lastIndexOf('/'); elementPath = elementPath.substring(0, lastslash) + ":" + elementPath.substring(lastslash + 1); elementPath = elementPath.replaceAll("/", ".") + ".element"; @@ -135,31 +166,25 @@ public void init() throws SystemException { //TODO: Right now it only returns 1000000 results results = searcher.search(query, 1000000); - } catch (Exception e) { - throw new SystemException(null); + throw new SystemException(null, e); } hits = results.scoreDocs; - System.out.println("found: " + results.totalHits); indexPlace = 0; indexLength = hits.length; - } public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException { try { - handler.startDocument(); - for (int i = 0; i < fields.size(); i++) { String fieldValue = fields.get(i).stringValue(); if (fieldValue.equals(elementPath)) { + handler.startDocument(); + this.firstElement = true; buildElement(abvsFileNode, i); } } - - handler.endDocument(); - handler.writeDocument(abvsFileNode); } catch (Exception e) { throw new IOException(e); } @@ -167,6 +192,7 @@ public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException { private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) throws SAXException { int whereIFinish = fieldNum; + int firstFinish; IndexableField field = fields.get(fieldNum); String contents = field.stringValue(); String uri = ""; @@ -176,18 +202,37 @@ private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) thr String type = contents.substring(lastDot + 1); String lastBit = contents.substring(firstColon + 1, lastDot); - if (type.equals("textnode")) { + if (this.firstElement) { + this.firstElement = false; + firstFinish = whereIFinish - this.childSequenceTypes.size() + 1; + String firstBit = contents.substring(1, firstColon); + List names = new ArrayList<>(); + List values = new ArrayList<>(); + List uris = new ArrayList<>(); + List localNames = new ArrayList<>(); + List types = new ArrayList<>(); + List qNames = new ArrayList<>(); + firstFinish = findAttributeChildren(firstFinish, names, values, uris, localNames, types, qNames); + Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames); + + handler.startElement(uri, firstBit, firstBit, atts); + buildElement(abvsFileNode, firstFinish + 1); + handler.endElement(uri, firstBit, firstBit); + + } + + if ("textnode".equals(type)) { char[] charContents = lastBit.toCharArray(); handler.characters(charContents, 0, charContents.length); } - if (type.equals("element")) { - List names = new ArrayList(); - List values = new ArrayList(); - List uris = new ArrayList(); - List localNames = new ArrayList(); - List types = new ArrayList(); - List qNames = new ArrayList(); + if ("element".equals(type)) { + List names = new ArrayList<>(); + List values = new ArrayList<>(); + List uris = new ArrayList<>(); + List localNames = new ArrayList<>(); + List types = new ArrayList<>(); + List qNames = new ArrayList<>(); whereIFinish = findAttributeChildren(whereIFinish, names, values, uris, localNames, types, qNames); Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames); @@ -264,7 +309,7 @@ boolean isChild(IndexableField child, IndexableField adult) { String adultPath = adultId.substring(0, lastDotAdult); adultPath = adultPath.replaceFirst(":", "."); - return (childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + ".")); + return childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + "."); } boolean isDirectChildAttribute(IndexableField child, IndexableField adult) { @@ -278,7 +323,7 @@ boolean isDirectChildAttribute(IndexableField child, IndexableField adult) { String childType = childSegments[childSegments.length - 1]; - return (childPath.startsWith(adultPath + ":") && childType.equals("attribute")); + return childPath.startsWith(adultPath + ":") && "attribute".equals(childType); } } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java similarity index 78% rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java index 51510d59a..1d584337e 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.vxquery.runtime.functions.index.indexCentralizer; +package org.apache.vxquery.runtime.functions.index.centralizer; import java.io.DataOutput; import java.io.File; @@ -49,46 +49,54 @@ */ public class IndexCentralizerUtil { - private final String FILE_NAME = "VXQuery-Index-Directory.xml"; + private static final String FILE_NAME = "VXQuery-Index-Directory.xml"; private final List collections = new ArrayList<>(); - private final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil"); - private File XML_FILE; - private String INDEX_LOCATION; - private static ConcurrentHashMap indexCollectionMap = new ConcurrentHashMap<>(); + private static final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil"); + private File xmlFile; + private String indexPath; + public static ConcurrentHashMap indexCollectionMap = new ConcurrentHashMap<>(); + private static final StringValueBuilder svb = new StringValueBuilder(); + private final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); + private final DataOutput output = abvs.getDataOutput(); public IndexCentralizerUtil(File index) { - this.INDEX_LOCATION = index.getPath(); + indexPath = index.getPath(); if (!index.exists()) { try { FileUtils.forceMkdir(index); } catch (IOException e) { - LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + INDEX_LOCATION + " " + e); + LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + indexPath + " " + e); } } - XML_FILE = new File(index.getPath() + "/" + FILE_NAME); + xmlFile = new File(index.getPath() + "/" + FILE_NAME); } /** * Get the index directory containing index of the given collection * - * @param collection : Collection folder + * @param collection + * : Collection folder * @return Index folder. */ public String getIndexForCollection(String collection) { - return indexCollectionMap.get(collection).getIndex(); + if (indexCollectionMap.size() > 0 && indexCollectionMap.containsKey(collection)) { + return indexCollectionMap.get(collection).getIndex(); + } + return null; } /** * Put the index location corresponding to given collection. * Index location is created by using the last 100 characters of collection. * - * @param collection : Collection directory + * @param collection + * : Collection directory * @return index */ public String putIndexForCollection(String collection) { int length = collection.replaceAll("/", "").length(); String index = collection.replaceAll("/", ""); - index = INDEX_LOCATION + "/" + (length > 100 ? index.substring(length - 100) : index); + index = indexPath + "/" + (length > 100 ? index.substring(length - 100) : index); IndexLocator il = new IndexLocator(); il.setCollection(collection); il.setIndex(index); @@ -102,7 +110,8 @@ public String putIndexForCollection(String collection) { /** * Remove the entry for given collection directory. * - * @param collection : Collection directory + * @param collection + * : Collection directory */ public void deleteEntryForCollection(String collection) { indexCollectionMap.remove(collection); @@ -110,14 +119,15 @@ public void deleteEntryForCollection(String collection) { /** * Prints all collections which have an index created. - * @param sb : The output is stored in a sequence - * @throws IOException : If writing the dataOutput generates {@link IOException} + * + * @param sb + * : The output is stored in a sequence + * @throws IOException + * : If writing the dataOutput generates {@link IOException} */ public void getAllCollections(SequenceBuilder sb) throws IOException { for (String s : collections) { - StringValueBuilder svb = new StringValueBuilder(); - ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); - DataOutput output = abvs.getDataOutput(); + abvs.reset(); output.write(ValueTag.XS_STRING_TAG); svb.write(s, output); sb.addItem(abvs); @@ -128,11 +138,11 @@ public void getAllCollections(SequenceBuilder sb) throws IOException { * Read the collection, index directory file and populate the HashMap. */ public void readIndexDirectory() { - if (this.XML_FILE.exists()) { + if (xmlFile.exists()) { try { JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class); Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); - IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(this.XML_FILE); + IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(xmlFile); for (IndexLocator il : indexDirectory.getDirectory()) { indexCollectionMap.put(il.getCollection(), il); @@ -153,7 +163,7 @@ public void writeIndexDirectory() { List indexLocators = new ArrayList<>(indexCollectionMap.values()); id.setDirectory(indexLocators); try { - FileOutputStream fileOutputStream = new FileOutputStream(this.XML_FILE); + FileOutputStream fileOutputStream = new FileOutputStream(this.xmlFile); JAXBContext context = JAXBContext.newInstance(IndexDirectory.class); Marshaller jaxbMarshaller = context.createMarshaller(); jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java similarity index 89% rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java index 54d9ad98e..d1189269c 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java @@ -14,19 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.vxquery.runtime.functions.index.indexCentralizer; +package org.apache.vxquery.runtime.functions.index.centralizer; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; @XmlRootElement(name = "indexes") @XmlAccessorType(XmlAccessType.FIELD) -public class IndexDirectory implements Serializable{ +public class IndexDirectory implements Serializable { + private static final long serialVersionUID = 1L; @XmlElement(name = "index", type = IndexLocator.class) private List directory = new ArrayList<>(); @@ -35,7 +37,6 @@ public List getDirectory() { return directory; } - public void setDirectory(List directory) { this.directory = directory; } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java similarity index 89% rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java index 1a33c8b60..49dbecd88 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java @@ -14,17 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.vxquery.runtime.functions.index.indexCentralizer; +package org.apache.vxquery.runtime.functions.index.centralizer; + +import java.io.Serializable; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; @XmlAccessorType(XmlAccessType.FIELD) @XmlRootElement(name = "Entry") -public class IndexLocator implements Serializable{ +public class IndexLocator implements Serializable { + private static final long serialVersionUID = 1L; @XmlAttribute private String collection; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java similarity index 80% rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java index 2a4574768..0346a62a4 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java @@ -14,12 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.vxquery.runtime.functions.index.updateIndex; +package org.apache.vxquery.runtime.functions.index.update; /** * Constants used in updating index */ public class Constants { - public static String FIELD_PATH = "path"; - public static String META_FILE_NAME = "vxquery_index.xml"; + public static final String FIELD_PATH = "path"; + public static final String META_FILE_NAME = "vxquery_index.xml"; + + private Constants() { + } } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java similarity index 87% rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java index d3b9fdf52..65a8325ec 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java @@ -14,12 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.vxquery.runtime.functions.index.updateIndex; +package org.apache.vxquery.runtime.functions.index.update; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.apache.hyracks.data.std.api.IPointable; -import org.apache.hyracks.data.std.primitive.UTF8StringPointable; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; -import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.lucene.index.IndexWriter; @@ -27,7 +34,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; -import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; @@ -36,16 +42,6 @@ import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil; import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; -import java.io.DataInputStream; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - /** * Update the index if the source files are changed. */ @@ -53,48 +49,37 @@ public class IndexUpdater { private MetaFileUtil metaFileUtil; private ConcurrentHashMap metadataMap; private IPointable result; - private ByteBufferInputStream bbis; - private DataInputStream di; - private SequenceBuilder sb; + private final SequenceBuilder sb = new SequenceBuilder(); private ArrayBackedValueStorage abvs; private ITreeNodeIdProvider nodeIdProvider; private ArrayBackedValueStorage abvsFileNode; - private TaggedValuePointable nodep; private String nodeId; private IndexWriter indexWriter; private Set pathsFromFileList; private String collectionFolder; private String indexFolder; - private Logger LOGGER = Logger.getLogger("Index Updater"); + private final Logger LOGGER = Logger.getLogger("Index Updater"); private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil(); - public IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp, ByteBufferInputStream bbis, - DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider, - ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep, String nodeId) { + public IndexUpdater(String indexFolder, IPointable result, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider, + ArrayBackedValueStorage abvsFileNode, String nodeId) { this.indexFolder = indexFolder; this.result = result; - this.bbis = bbis; - this.di = di; - this.sb = sb; this.abvs = abvs; this.nodeIdProvider = nodeIdProvider; this.abvsFileNode = abvsFileNode; - this.nodep = nodep; this.nodeId = nodeId; this.pathsFromFileList = new HashSet<>(); } /** * Perform the initial configuration for index update/ delete processes. - * - * @throws SystemException - * : If getting the index folder generates {@link SystemException} + * * @throws IOException * : If getting the index folder generates {@link IOException} */ - public void setup() throws SystemException, IOException { - + public void setup() throws IOException { // Read the metadata file and load the metadata map into memory. metaFileUtil = new MetaFileUtil(indexFolder); metaFileUtil.readMetadataFile(); @@ -114,14 +99,14 @@ public void setup() throws SystemException, IOException { /** * Wrapper for update index function. - * + * * @throws IOException * : If the directory doesn't exist */ public void updateIndex() throws IOException { File collectionDirectory = new File(collectionFolder); if (!collectionDirectory.exists()) { - throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist."); + throw new IOException("The collection directory (" + collectionFolder + ") does not exist."); } //Execute update index process @@ -134,7 +119,7 @@ public void updateIndex() throws IOException { /** * Close opened IndexWriter and terminate the index update/ delete process. - * + * * @throws IOException * : If exiting the index folder generates {@link IOException} */ @@ -149,7 +134,7 @@ public void exit() throws IOException { /** * Functional wrapper to update Metadata file. - * + * * @throws IOException * : If updating metadata folder generates {@link IOException} */ @@ -190,8 +175,8 @@ private void updateIndex(File collection) throws IOException { //Update index corresponding to the xml file. indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath())); - indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep, - abvsFileNode, nodeIdProvider, bbis, di, nodeId); + indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, abvsFileNode, + nodeIdProvider, nodeId); indexDocumentBuilder.printStart(); if (LOGGER.isDebugEnabled()) { @@ -207,8 +192,8 @@ private void updateIndex(File collection) throws IOException { // In this case, the xml file has not added to the index. (It is a newly added file) // Therefore generate a new index for this file and add it to the existing index. - indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep, abvsFileNode, - nodeIdProvider, bbis, di, nodeId); + indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, abvsFileNode, nodeIdProvider, + nodeId); indexDocumentBuilder.printStart(); if (LOGGER.isDebugEnabled()) { @@ -235,8 +220,8 @@ private void updateIndex(File collection) throws IOException { * @throws IOException * : If getting the file info generates {@link IOException} */ - private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException { - + private XmlMetadata updateEntry(File file, XmlMetadata metadataArg) throws IOException { + XmlMetadata metadata = metadataArg; if (metadata == null) { metadata = new XmlMetadata(); } @@ -286,7 +271,7 @@ private void deleteIndexOfDeletedFiles(Set pathsFromMap, Set pat * When deleting indexes, if any error occurred, the process will be rolled back and all the indexes will be * restored. * Otherwise the changes will be committed. - * + * * @throws SystemException * : An attempt to divide by zero */ @@ -310,7 +295,7 @@ public void deleteAllIndexes() throws SystemException { sb.finish(); result.set(abvs); } catch (IOException e1) { - throw new SystemException(ErrorCode.FOAR0001); + throw new SystemException(ErrorCode.FOAR0001, e1); } } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java similarity index 85% rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java index 0dfb54aec..5f4135567 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java @@ -14,16 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.vxquery.runtime.functions.index.updateIndex; +package org.apache.vxquery.runtime.functions.index.update; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -import javax.xml.bind.DatatypeConverter; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; -import javax.xml.bind.Unmarshaller; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -37,13 +29,22 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.xml.bind.DatatypeConverter; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + /** * Utility class for writing, reading metadata file and generating checksum. */ public class MetaFileUtil { private File metaFile; - private Logger LOGGER = Logger.getLogger("MetadataFileUtil"); + private static final Logger LOGGER = Logger.getLogger("MetadataFileUtil"); private String index; private String collection; private ConcurrentHashMap indexMap = new ConcurrentHashMap<>(); @@ -65,8 +66,11 @@ public boolean isMetaFilePresent() { * Update the content of the metadata map. * If the current collection data is present, replace it. * Otherwise insert new. - * @param metadataMap : Set of XmlMetaData objects. - * @param index : The path to index location. + * + * @param metadataMap + * : Set of XmlMetaData objects. + * @param index + * : The path to index location. */ public void updateMetadataMap(ConcurrentHashMap metadataMap, String index) { this.indexMap = metadataMap; @@ -109,22 +113,22 @@ public void readMetadataFile() { * Write the content of the ConcurrentHashMap to the xml metadata file. */ public void writeMetadataToFile() { - XmlMetadataCollection collection = new XmlMetadataCollection(); + XmlMetadataCollection xmlMetadataCollection = new XmlMetadataCollection(); List metadataList = new ArrayList<>(); for (Map.Entry entry : this.indexMap.entrySet()) { metadataList.add(entry.getValue()); } - collection.setMetadataList(metadataList); - collection.setCollection(this.collection); - collection.setIndexLocation(this.index); - try{ + xmlMetadataCollection.setMetadataList(metadataList); + xmlMetadataCollection.setCollection(collection); + xmlMetadataCollection.setIndexLocation(this.index); + try { FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile); JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class); Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); - jaxbMarshaller.marshal(collection, fileOutputStream); + jaxbMarshaller.marshal(xmlMetadataCollection, fileOutputStream); if (LOGGER.isDebugEnabled()) { LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!"); @@ -135,18 +139,18 @@ public void writeMetadataToFile() { } } - } - /** * Generate MD5 checksum string for a given file. * - * @param file : File which the checksum should be generated. + * @param file + * : File which the checksum should be generated. * @return : Checksum String - * @throws IOException : The file is not available + * @throws IOException + * : The file is not available */ - public String generateMD5(File file) throws IOException { + public String generateMD5(File file) throws IOException { try { MessageDigest md = MessageDigest.getInstance("MD5"); md.update(Files.readAllBytes(file.toPath())); @@ -168,12 +172,12 @@ public String generateMD5(File file) throws IOException { public boolean deleteMetaDataFile() { try { Files.delete(Paths.get(metaFile.getCanonicalPath())); - if (LOGGER.isDebugEnabled()){ + if (LOGGER.isDebugEnabled()) { LOGGER.log(Level.DEBUG, "Metadata file deleted!"); } return true; } catch (IOException e) { - if (LOGGER.isTraceEnabled()){ + if (LOGGER.isTraceEnabled()) { LOGGER.log(Level.ERROR, "Metadata file could not be deleted!"); } return false; @@ -182,6 +186,7 @@ public boolean deleteMetaDataFile() { /** * Get the collection for a given index location. + * * @return collection folder for a given index. */ public String getCollection() { @@ -190,7 +195,9 @@ public String getCollection() { /** * Set the entry for given index and collection. - * @param collection : path to corresponding collection + * + * @param collection + * : path to corresponding collection */ public void setCollection(String collection) { this.collection = collection; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java similarity index 95% rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java index fa92b2f36..fc967633a 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.vxquery.runtime.functions.index.updateIndex; +package org.apache.vxquery.runtime.functions.index.update; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java similarity index 94% rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java index b6da6d9bd..063120cc6 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java @@ -14,12 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.vxquery.runtime.functions.index.updateIndex; +package org.apache.vxquery.runtime.functions.index.update; + +import java.io.Serializable; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; /** * Class to store metadata related to an XML file. @@ -32,6 +33,7 @@ @XmlRootElement(name = "file") @XmlAccessorType(XmlAccessType.FIELD) public class XmlMetadata implements Serializable { + private static final long serialVersionUID = 1L; private String path; private String md5; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java similarity index 97% rename from vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java rename to vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java index 1f5c3e99e..a1ca77646 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.vxquery.runtime.functions.index.updateIndex; +package org.apache.vxquery.runtime.functions.index.update; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java index 665c812b8..85ef4ca06 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java @@ -64,7 +64,7 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S tvp.getValue(stringp); try { IParser parser = new JSONParser(); - FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser); + FunctionHelper.readInDocFromPointable(stringp, abvs, parser); } catch (IOException e) { throw new SystemException(ErrorCode.FODC0002, e); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java index db908f638..15fd62474 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java @@ -78,7 +78,7 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S tvp.getValue(stringp); try { IParser parser = new XMLParser(false, nodeIdProvider, nodeId); - FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser); + FunctionHelper.readInDocFromPointable(stringp, abvs, parser); XDMConstants.setTrue(result); } catch (Exception e) { XDMConstants.setFalse(result); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java index 5f08a8eef..2fd1755ae 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java @@ -79,7 +79,7 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S try { // Only one document should be parsed so its ok to have a unique parser. IParser parser = new XMLParser(false, nodeIdProvider, nodeId); - FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser); + FunctionHelper.readInDocFromPointable(stringp, abvs, parser); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java index 6558274ae..1d66c4eb8 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java @@ -16,7 +16,6 @@ */ package org.apache.vxquery.runtime.functions.util; -import java.io.DataInputStream; import java.io.DataOutput; import java.io.File; import java.io.FileInputStream; @@ -37,7 +36,7 @@ import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.data.std.util.GrowableArray; import org.apache.hyracks.data.std.util.UTF8StringBuilder; -import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; +import org.apache.hyracks.util.string.UTF8StringUtil; import org.apache.vxquery.context.DynamicContext; import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; import org.apache.vxquery.datamodel.accessors.TypedPointables; @@ -485,6 +484,15 @@ public static boolean arraysEqual(byte[] bytes1, int offset1, int length1, byte[ return true; } + public static String getStringFromBytes(byte[] bytes) { + if (bytes == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + UTF8StringUtil.toString(sb, bytes, 0); + return sb.toString(); + } + public static boolean compareTaggedValues(AbstractValueComparisonOperation aOp, TaggedValuePointable tvp1, TaggedValuePointable tvp2, DynamicContext dCtx, TypedPointables tp1, TypedPointables tp2) throws SystemException { @@ -1215,13 +1223,12 @@ public static void printUTF8String(UTF8StringPointable stringp) { System.err.println(" printUTF8String END"); } - public static void readInDocFromPointable(UTF8StringPointable stringp, ByteBufferInputStream bbis, - DataInputStream di, ArrayBackedValueStorage abvs, IParser parser) throws IOException { - readInDocFromString(stringp.toString(), bbis, di, abvs, parser); + public static void readInDocFromPointable(UTF8StringPointable stringp, ArrayBackedValueStorage abvs, + IParser parser) throws IOException { + readInDocFromString(stringp.toString(), abvs, parser); } - public static void readInDocFromString(String fName, ByteBufferInputStream bbis, DataInputStream di, - ArrayBackedValueStorage abvs, IParser parser) throws IOException { + public static void readInDocFromString(String fName, ArrayBackedValueStorage abvs, IParser parser) throws IOException { Reader input; if (!fName.contains("hdfs:/")) { File file = new File(fName); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java index 84c8ddf25..9e21f5399 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/SAXContentHandler.java @@ -129,8 +129,6 @@ public SAXContentHandler(boolean attachTypes, ITreeNodeIdProvider nodeIdProvider public SAXContentHandler(boolean attachTypes, ITreeNodeIdProvider nodeIdProvider, IFrameFieldAppender appender, List childSequenceTypes) { this(attachTypes, nodeIdProvider, false); - - // Frame writing variables this.appender = appender; setChildPathSteps(childSequenceTypes); } @@ -297,10 +295,11 @@ public void startDocument() throws SAXException { /** * The filter settings here are similar to one in the class linked below. - * + * + * @throws SAXException * @see org.apache.vxquery.runtime.functions.step.NodeTestFilter.java */ - private boolean startElementChildPathStep(String uri, String localName) { + private boolean startElementChildPathStep(String uri, String localName) throws SAXException { if (subElement != null && depth <= subElement.length) { // Check path step if it exists. subElement[depth - 1] = true; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java index 0c252b4f3..d3d02aee5 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java @@ -222,7 +222,7 @@ private void setNodeList() { } } - public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel) + public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel, List collections) throws AlgebricksException, SystemException { moduleNode = XMLQueryParser.parse(name, query); listener.notifyParseResult(moduleNode); @@ -230,7 +230,7 @@ public void compile(String name, Reader query, CompilerControlBlock ccb, int opt pprinter = new LogicalOperatorPrettyPrintVisitor(new AlgebricksAppendable(), new VXQueryLogicalExpressionPrettyPrintVisitor(module.getModuleContext())); VXQueryMetadataProvider mdProvider = new VXQueryMetadataProvider(nodeList, ccb.getSourceFileMap(), - module.getModuleContext(), this.hdfsConf, nodeControllerInfos); + module.getModuleContext(), this.hdfsConf, nodeControllerInfos, collections); compiler = cFactory.createCompiler(module.getBody(), mdProvider, 0); listener.notifyTranslationResult(module); XMLQueryTypeChecker.typeCheckModule(module); diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java index 60f39f879..45d553fa6 100644 --- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java +++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java @@ -23,8 +23,8 @@ import javax.xml.bind.JAXBException; import org.apache.commons.io.FileUtils; -import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil; -import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata; +import org.apache.vxquery.runtime.functions.index.update.MetaFileUtil; +import org.apache.vxquery.runtime.functions.index.update.XmlMetadata; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java index b79107e2f..1b3e0b771 100644 --- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java +++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java @@ -14,8 +14,6 @@ */ package org.apache.vxquery.indexing; -import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata; - import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -27,6 +25,8 @@ import java.nio.file.Paths; import java.util.concurrent.ConcurrentHashMap; +import org.apache.vxquery.runtime.functions.index.update.XmlMetadata; + /** * TestConstants and methods which will be used in indexing test cases. */ diff --git a/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java b/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java index 7646f9734..1ee35a474 100644 --- a/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java +++ b/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java @@ -142,6 +142,6 @@ private static void runTestInternal(String testName, String query) throws Except XMLQueryCompiler compiler = new XMLQueryCompiler(null, nodeControllerInfos, 65536); CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE), new ResultSetId(System.nanoTime()), null); - compiler.compile(testName, new StringReader(query), ccb, Integer.MAX_VALUE); + compiler.compile(testName, new StringReader(query), ccb, Integer.MAX_VALUE, null); } } diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java index e4ba6eb92..fa0a90034 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java @@ -14,16 +14,22 @@ */ package org.apache.vxquery.xtest; +import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumSet; +import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.api.client.NodeControllerInfo; import org.apache.hyracks.api.comm.IFrame; @@ -53,13 +59,14 @@ public class TestRunner { private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})"); - + private List collectionList; private XTestOptions opts; private IHyracksClientConnection hcc; private IHyracksDataset hds; public TestRunner(XTestOptions opts) throws UnknownHostException { this.opts = opts; + this.collectionList = new ArrayList(); } public void open() throws Exception { @@ -67,8 +74,46 @@ public void open() throws Exception { hds = TestClusterUtil.getDataset(); } + protected static TestConfiguration getIndexConfiguration(TestCase testCase) { + XTestOptions opts = new XTestOptions(); + opts.verbose = false; + opts.threads = 1; + opts.showQuery = true; + opts.showResult = true; + opts.hdfsConf = "src/test/resources/hadoop/conf"; + opts.catalog = StringUtils.join(new String[] { "src", "test", "resources", "VXQueryCatalog.xml" }, + File.separator); + TestConfiguration indexConf = new TestConfiguration(); + indexConf.options = opts; + String baseDir = new File(opts.catalog).getParent(); + try { + String root = new File(baseDir).getCanonicalPath(); + indexConf.testRoot = new File(root + "/./"); + indexConf.resultOffsetPath = new File(root + "/./ExpectedResults/"); + indexConf.sourceFileMap = testCase.getSourceFileMap(); + indexConf.xqueryFileExtension = ".xq"; + indexConf.xqueryxFileExtension = "xqx"; + indexConf.xqueryQueryOffsetPath = new File(root + "/./Queries/XQuery/"); + } catch (IOException e) { + e.printStackTrace(); + } + return indexConf; + + } + public TestCaseResult run(final TestCase testCase) { TestCaseResult res = new TestCaseResult(testCase); + TestCase testCaseIndex = new TestCase(getIndexConfiguration(testCase)); + testCaseIndex.setFolder("Indexing/Partition-1/"); + testCaseIndex.setName("showIndexes"); + runQuery(testCaseIndex, res); + String[] collections = res.result.split("\n"); + this.collectionList = Arrays.asList(collections); + runQueries(testCase, res); + return res; + } + + public void runQuery(TestCase testCase, TestCaseResult res) { if (opts.verbose) { System.err.println("Starting " + testCase.getXQueryDisplayName()); } @@ -78,6 +123,7 @@ public TestCaseResult run(final TestCase testCase) { try { try { if (opts.showQuery) { + FileInputStream query = new FileInputStream(testCase.getXQueryFile()); System.err.println("***Query for " + testCase.getXQueryDisplayName() + ": "); System.err.println(IOUtils.toString(query, "UTF-8")); @@ -98,7 +144,7 @@ public TestCaseResult run(final TestCase testCase) { CompilerControlBlock ccb = new CompilerControlBlock( new StaticContextImpl(RootStaticContextImpl.INSTANCE), new ResultSetId(testCase.getXQueryDisplayName().hashCode()), testCase.getSourceFileMap()); - compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel); + compiler.compile(testCase.getXQueryDisplayName(), in, ccb, opts.optimizationLevel, collectionList); JobSpecification spec = compiler.getModule().getHyracksJobSpecification(); in.close(); @@ -172,7 +218,11 @@ public TestCaseResult run(final TestCase testCase) { System.err.println(res.result); } } - return res; + + } + + public void runQueries(TestCase testCase, TestCaseResult res) { + runQuery(testCase, res); } public void close() throws Exception { diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt new file mode 100644 index 000000000..baf9dcade --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1_user.txt @@ -0,0 +1,2 @@ +2003-03-03T00:00:00.000TMINGHCND:AS00000000313.75a +2003-03-03T00:00:00.000TMAXGHCND:AS00000000333a \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt new file mode 100644 index 000000000..baf9dcade --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/useIndex1_user.txt @@ -0,0 +1,2 @@ +2003-03-03T00:00:00.000TMINGHCND:AS00000000313.75a +2003-03-03T00:00:00.000TMAXGHCND:AS00000000333a \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt new file mode 100644 index 000000000..baf9dcade --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-4/useIndex1_user.txt @@ -0,0 +1,2 @@ +2003-03-03T00:00:00.000TMINGHCND:AS00000000313.75a +2003-03-03T00:00:00.000TMAXGHCND:AS00000000333a \ No newline at end of file diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq index 63fdda7ee..96a767146 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1.xq @@ -16,10 +16,10 @@ under the License. :) (: Search Lucene Index :) -for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data +for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data let $datetime := xs:dateTime(fn:data($r/date)) where $r/station eq "GHCND:AS000000003" and fn:year-from-dateTime($datetime) ge 2000 and fn:month-from-dateTime($datetime) eq 3 and fn:day-from-dateTime($datetime) eq 3 -return $r \ No newline at end of file +return $r diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq new file mode 100644 index 000000000..7a2bb2c2a --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex1_user.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: Search Lucene Index :) +for $r in collection-from-index("src/test/resources/TestSources/ghcnd")/dataCollection/data +let $datetime := xs:dateTime(fn:data($r/date)) +where $r/station eq "GHCND:AS000000003" + and fn:year-from-dateTime($datetime) ge 2000 + and fn:month-from-dateTime($datetime) eq 3 + and fn:day-from-dateTime($datetime) eq 3 +return $r diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq index cf41536c1..464c1cc68 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex2.xq @@ -19,6 +19,6 @@ (: Find all reading for hurricane force wind warning or extreme wind warning. :) (: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :) (: meters per second). (Wind value is in tenth of a meter per second) :) -for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data +for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744 return $r diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq index 5c99d9aff..eeb019bc5 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex3.xq @@ -19,7 +19,7 @@ (: Find the annual precipitation (PRCP) for a Seattle using the airport :) (: station (US000000002) for 2002. :) fn:sum( - for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data + for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data where $r/station eq "GHCND:US000000002" and $r/dataType eq "PRCP" and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002 diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq index 39e5d175b..06284aec8 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex4.xq @@ -18,7 +18,7 @@ (: Search Lucene Index :) (: Find the highest recorded temperature (TMAX) in Celsius. :) fn:max( - for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data + for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data where $r/dataType eq "TMAX" return $r/value ) div 10 diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq index 63aeca5e8..3a4ae0507 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex5.xq @@ -18,6 +18,6 @@ (: Search Lucene Index :) (: Find all the weather readings for Washington state for a specific day :) (: 2002-2-2. :) -for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data +for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000") return $r diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq index c81f27153..9090eddf7 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex6.xq @@ -18,7 +18,7 @@ (: Search Lucene Index :) (: Find all the weather readings for Washington state for a specific day :) (: 2002-2-2. :) -for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station +for $s in collection("src/test/resources/TestSources/ghcnd")/stationCollection/station where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1")) order by $s/id return $s diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq index dd6b5f920..7c703b66b 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-1/useIndex7.xq @@ -18,8 +18,8 @@ (: Search Lucene Index :) (: Find all the weather readings for Washington state for a specific day :) (: 2002-2-2. :) -for $s in collection-from-index("src/test/resources/TestSources/ghcnd", "/stationCollection/station")/station -for $r in collection-from-index("src/test/resources/TestSources/ghcnd", "/dataCollection/data")/data +for $s in collection("src/test/resources/TestSources/ghcnd")/stationCollection/station +for $r in collection("src/test/resources/TestSources/ghcnd")/dataCollection/data where $s/id eq $r/station and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1")) diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq index fecb56df7..471824032 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1.xq @@ -16,7 +16,7 @@ under the License. :) (: Search Lucene Index :) -for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data +for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data let $datetime := xs:dateTime(fn:data($r/date)) where $r/station eq "GHCND:AS000000003" and fn:year-from-dateTime($datetime) ge 2000 diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq new file mode 100644 index 000000000..0e42155a0 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex1_user.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: Search Lucene Index :) +for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data +let $datetime := xs:dateTime(fn:data($r/date)) +where $r/station eq "GHCND:AS000000003" + and fn:year-from-dateTime($datetime) ge 2000 + and fn:month-from-dateTime($datetime) eq 3 + and fn:day-from-dateTime($datetime) eq 3 +return $r diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq index 75c7a6475..37e5626bc 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex2.xq @@ -19,6 +19,6 @@ (: Find all reading for hurricane force wind warning or extreme wind warning. :) (: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :) (: meters per second). (Wind value is in tenth of a meter per second) :) -for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data +for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744 return $r diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq index 28f747367..358d3f38d 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex3.xq @@ -19,7 +19,7 @@ (: Find the annual precipitation (PRCP) for a Seattle using the airport :) (: station (US000000002) for 2002. :) fn:sum( - for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data + for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data where $r/station eq "GHCND:US000000002" and $r/dataType eq "PRCP" and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002 diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq index 317a1416c..bd5ba1c8b 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex4.xq @@ -18,7 +18,7 @@ (: Search Lucene Index :) (: Find the highest recorded temperature (TMAX) in Celsius. :) fn:max( - for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data + for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data where $r/dataType eq "TMAX" return $r/value ) div 10 diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq index 2deb4c3ef..77f6c2f30 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex5.xq @@ -18,6 +18,6 @@ (: Search Lucene Index :) (: Find all the weather readings for Washington state for a specific day :) (: 2002-2-2. :) -for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data +for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000") return $r diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq index a0ce1e9ad..c1a45f433 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex6.xq @@ -18,6 +18,6 @@ (: Search Lucene Index :) (: Find all the weather readings for Washington state for a specific day :) (: 2002-2-2. :) -for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station +for $s in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/stationCollection/station where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1")) return $s diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq index b3e622cd6..a776ab920 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-2/useIndex7.xq @@ -18,8 +18,8 @@ (: Search Lucene Index :) (: Find all the weather readings for Washington state for a specific day :) (: 2002-2-2. :) -for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/stationCollection/station")/station -for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2", "/dataCollection/data")/data +for $s in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/stationCollection/station +for $r in collection("src/test/resources/TestSources/ghcnd/half_1|src/test/resources/TestSources/ghcnd/half_2")/dataCollection/data where $s/id eq $r/station and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1")) diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq index 0cccbc52a..2bc9ce759 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1.xq @@ -16,7 +16,7 @@ under the License. :) (: Search Lucene Index :) -for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data +for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data let $datetime := xs:dateTime(fn:data($r/date)) where $r/station eq "GHCND:AS000000003" and fn:year-from-dateTime($datetime) ge 2000 diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq new file mode 100644 index 000000000..e740365a2 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex1_user.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: Search Lucene Index :) +for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data +let $datetime := xs:dateTime(fn:data($r/date)) +where $r/station eq "GHCND:AS000000003" + and fn:year-from-dateTime($datetime) ge 2000 + and fn:month-from-dateTime($datetime) eq 3 + and fn:day-from-dateTime($datetime) eq 3 +return $r diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq index c282e313f..a1b86acba 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex2.xq @@ -19,6 +19,6 @@ (: Find all reading for hurricane force wind warning or extreme wind warning. :) (: The warnings occur when the wind speed (AWND) exceeds 110 mph (49.1744 :) (: meters per second). (Wind value is in tenth of a meter per second) :) -for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data +for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744 return $r diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq index 33ea1c954..9cc2b8eaf 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex3.xq @@ -19,7 +19,7 @@ (: Find the annual precipitation (PRCP) for a Seattle using the airport :) (: station (US000000002) for 2002. :) fn:sum( - for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data + for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data where $r/station eq "GHCND:US000000002" and $r/dataType eq "PRCP" and fn:year-from-dateTime(xs:dateTime(fn:data($r/date))) eq 2002 diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq index d213082aa..dd26e87cc 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex4.xq @@ -18,7 +18,7 @@ (: Search Lucene Index :) (: Find the highest recorded temperature (TMAX) in Celsius. :) fn:max( - for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data + for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data where $r/dataType eq "TMAX" return $r/value ) div 10 diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq index 1d98682ac..4a9d224bd 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex5.xq @@ -18,6 +18,6 @@ (: Search Lucene Index :) (: Find all the weather readings for Washington state for a specific day :) (: 2002-2-2. :) -for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data +for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data where xs:dateTime(fn:data($r/date)) eq xs:dateTime("2002-02-02T00:00:00.000") return $r diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq index abe21840a..440707916 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex6.xq @@ -18,6 +18,6 @@ (: Search Lucene Index :) (: Find all the weather readings for Washington state for a specific day :) (: 2002-2-2. :) -for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station +for $s in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/stationCollection/station where (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1")) return $s diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq index 7b40ca00b..664b15034 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Indexing/Partition-4/useIndex7.xq @@ -18,8 +18,8 @@ (: Search Lucene Index :) (: Find all the weather readings for Washington state for a specific day :) (: 2002-2-2. :) -for $s in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/stationCollection/station")/station -for $r in collection-from-index("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4", "/dataCollection/data")/data +for $s in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/stationCollection/station +for $r in collection("src/test/resources/TestSources/ghcnd/half_1/quarter_1|src/test/resources/TestSources/ghcnd/half_1/quarter_2|src/test/resources/TestSources/ghcnd/half_2/quarter_3|src/test/resources/TestSources/ghcnd/half_2/quarter_4")/dataCollection/data where $s/id eq $r/station and (some $x in $s/locationLabels satisfies ($x/type eq "ST" and fn:upper-case(fn:data($x/displayName)) eq "STATE 1")) diff --git a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml index 5ecdb9436..8e2813587 100644 --- a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml +++ b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml @@ -353,7 +353,7 @@ &LibrariesInJSONiq; - + Serialize Function Queries @@ -365,7 +365,7 @@ &SerializationQueries; - + XML in JSON diff --git a/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml b/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml index 7cf6bf667..cc6b65be5 100644 --- a/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml +++ b/vxquery-xtest/src/test/resources/cat/IndexingQueries.xml @@ -35,6 +35,11 @@ useIndex1.txt + + Get Collection From Lucene Index + + useIndex1_user.txt + Get Collection From Lucene Index @@ -95,6 +100,11 @@ useIndex1.txt + + Get Collection From Lucene Index + + useIndex1_user.txt + Get Collection From Lucene Index @@ -145,6 +155,11 @@ useIndex1.txt + + Get Collection From Lucene Index + + useIndex1_user.txt + Get Collection From Lucene Index From 0ed20c53ee6fb16110c3c29c6bc2f2ffe477c094 Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sun, 7 May 2017 00:21:55 -0700 Subject: [PATCH 09/14] Add missing dependencies --- pom.xml | 70 ++++++++++++++++++++++++++++++++++++++++-- vxquery-cli/pom.xml | 23 +++++++++++--- vxquery-core/pom.xml | 55 +++++++++++++++++++++++++++++++++ vxquery-server/pom.xml | 10 ++++++ vxquery-xtest/pom.xml | 45 +++++++++++++++++++++++++++ 5 files changed, 197 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 5f03a49bc..6fef7c6ba 100644 --- a/pom.xml +++ b/pom.xml @@ -222,7 +222,37 @@ args4j args4j - 2.0.9 + 2.33 + + + + log4j + log4j + 1.2.17 + + + + javax.xml.bind + jaxb-api + 2.2.2 + + + + xml-apis + xml-apis + 1.3.04 + + + + stax + stax-api + 1.0.1 + + + + javax.servlet + servlet-api + 2.5 @@ -231,6 +261,12 @@ ${hyracks.version} + + org.apache.hyracks + hyracks-util + ${hyracks.version} + + org.apache.hyracks hyracks-control-common @@ -328,12 +364,24 @@ jar + + org.apache.hadoop + hadoop-common + 2.7.0 + + org.apache.hadoop hadoop-hdfs 2.7.0 + + org.apache.hadoop + hadoop-mapreduce-client-core + 2.7.0 + + ant ant-trax @@ -383,6 +431,12 @@ 2.8.4 + + com.fasterxml.jackson.core + jackson-databind + 2.8.4 + + commons-io commons-io @@ -395,6 +449,18 @@ 1.4 + + commons-lang + commons-lang + 2.6 + + + + com.google.guava + guava + 11.0.2 + + junit junit @@ -670,7 +736,7 @@ UTF-8 UTF-8 - 0.3.0 + 0.3.2-SNAPSHOT 0.11 diff --git a/vxquery-cli/pom.xml b/vxquery-cli/pom.xml index 52c32ad9d..48bd63fa2 100644 --- a/vxquery-cli/pom.xml +++ b/vxquery-cli/pom.xml @@ -87,6 +87,16 @@ compile + + args4j + args4j + + + + commons-io + commons-io + + org.apache.hyracks hyracks-api @@ -122,19 +132,24 @@ hyracks-control-common + + org.apache.hyracks + hyracks-dataflow-common + + org.apache.hyracks hyracks-dataflow-std - org.apache.hyracks - hyracks-hdfs-core + org.apache.hyracks + hyracks-hdfs-core - org.apache.hyracks - hyracks-hdfs-2.x + org.apache.hyracks + hyracks-hdfs-2.x diff --git a/vxquery-core/pom.xml b/vxquery-core/pom.xml index 38908d7e9..67d45e36e 100644 --- a/vxquery-core/pom.xml +++ b/vxquery-core/pom.xml @@ -175,6 +175,21 @@ commons-lang3 + + org.apache.hyracks + algebricks-common + + + + org.apache.hyracks + algebricks-core + + + + org.apache.hyracks + algebricks-runtime + + org.apache.hyracks algebricks-compiler @@ -195,6 +210,11 @@ hyracks-api + + org.apache.hyracks + hyracks-util + + org.apache.hyracks hyracks-data-std @@ -261,6 +281,11 @@ jackson-core + + com.fasterxml.jackson.core + jackson-databind + + commons-io commons-io @@ -271,12 +296,42 @@ commons-codec + + commons-lang + commons-lang + + + + com.google.guava + guava + + junit junit test + + javax.xml.bind + jaxb-api + + + + xml-apis + xml-apis + + + + stax + stax-api + + + + log4j + log4j + + lucene-core org.apache.lucene diff --git a/vxquery-server/pom.xml b/vxquery-server/pom.xml index e572c3605..5cc9b5d84 100644 --- a/vxquery-server/pom.xml +++ b/vxquery-server/pom.xml @@ -143,6 +143,11 @@ 0.7-SNAPSHOT + + org.apache.hyracks + hyracks-api + + org.apache.hyracks hyracks-control-cc @@ -152,6 +157,11 @@ org.apache.hyracks hyracks-control-nc + + + args4j + args4j + diff --git a/vxquery-xtest/pom.xml b/vxquery-xtest/pom.xml index a00bec201..b8b7a8603 100644 --- a/vxquery-xtest/pom.xml +++ b/vxquery-xtest/pom.xml @@ -143,6 +143,26 @@ 0.7-SNAPSHOT + + args4j + args4j + + + + xml-apis + xml-apis + + + + stax + stax-api + + + + javax.servlet + servlet-api + + org.apache.hyracks hyracks-api @@ -153,6 +173,11 @@ hyracks-client + + org.apache.hyracks + hyracks-control-common + + org.apache.hyracks hyracks-control-cc @@ -168,6 +193,11 @@ hyracks-dataflow-std + + org.apache.hyracks + hyracks-dataflow-common + + org.apache.hyracks hyracks-hdfs-2.x @@ -178,11 +208,26 @@ commons-io + + org.apache.commons + commons-lang3 + + + + org.apache.hadoop + hadoop-common + + org.apache.hadoop hadoop-hdfs + + org.apache.hadoop + hadoop-mapreduce-client-core + + org.mortbay.jetty jetty From f706f0b28f930ed8bf68cd17d1ca6dd0c227e56d Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sun, 7 May 2017 01:07:08 -0700 Subject: [PATCH 10/14] Fixes for current Hyracks 0.3.2-SNAPSHOT mvn package works --- .../java/org/apache/vxquery/cli/VXQuery.java | 33 +++++++++--------- .../VXQueryCollectionOperatorDescriptor.java | 6 ++-- .../VXQueryIndexingOperatorDescriptor.java | 2 +- .../FnDocAvailableScalarEvaluatorFactory.java | 2 +- .../node/FnDocScalarEvaluatorFactory.java | 2 +- .../apache/vxquery/xtest/TestClusterUtil.java | 34 +++++++++---------- .../org/apache/vxquery/xtest/TestRunner.java | 2 +- 7 files changed, 40 insertions(+), 41 deletions(-) diff --git a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java index 25ff9c47d..dfb440783 100644 --- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java +++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java @@ -21,7 +21,7 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.io.StringReader; -import java.net.InetAddress; +import java.net.Inet4Address; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; @@ -314,33 +314,32 @@ protected ResultSetId createResultSetId() { * @throws Exception */ public void startLocalHyracks() throws Exception { - String localAddress = InetAddress.getLocalHost().getHostAddress(); + String localAddress = Inet4Address.getLoopbackAddress().getHostAddress(); CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = localAddress; - ccConfig.clientNetPort = 39000; - ccConfig.clusterNetIpAddress = localAddress; - ccConfig.clusterNetPort = 39001; - ccConfig.httpPort = 39002; - ccConfig.profileDumpPeriod = 10000; + ccConfig.setClientListenAddress(localAddress); + ccConfig.setClientListenPort(39000); + ccConfig.setClusterListenAddress(localAddress); + ccConfig.setClusterListenPort(39001); + ccConfig.setConsoleListenPort(39002); + ccConfig.setProfileDumpPeriod(10000); cc = new ClusterControllerService(ccConfig); cc.start(); ncs = new NodeControllerService[opts.localNodeControllers]; for (int i = 0; i < ncs.length; i++) { - NCConfig ncConfig = new NCConfig(); - ncConfig.ccHost = "localhost"; - ncConfig.ccPort = 39001; - ncConfig.clusterNetIPAddress = localAddress; - ncConfig.dataIPAddress = localAddress; - ncConfig.resultIPAddress = localAddress; - ncConfig.nodeId = "nc" + (i + 1); + NCConfig ncConfig = new NCConfig("nc" + (i + 1)); + ncConfig.setClusterAddress("localhost"); + ncConfig.setClusterPort(39001); + ncConfig.setClusterListenAddress(localAddress); + ncConfig.setDataListenAddress(localAddress); + ncConfig.setResultListenAddress(localAddress); //TODO: enable index folder as a cli option for on-the-fly indexing queries - ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString(); + ncConfig.setIODevices(new String[] { Files.createTempDirectory(ncConfig.getNodeId()).toString() }); ncs[i] = new NodeControllerService(ncConfig); ncs[i].start(); } - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + hcc = new HyracksConnection(ccConfig.getClientListenAddress(), ccConfig.getClientListenPort()); } /** diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index a3756d5b4..d214fd781 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -23,7 +23,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.net.InetAddress; +import java.net.Inet4Address; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -108,7 +108,7 @@ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final IFrameFieldAppender appender = new FrameFixedFieldTupleAppender(fieldOutputCount); final short partitionId = (short) ctx.getTaskAttemptId().getTaskId().getPartition(); final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources); - final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); + final String nodeId = ctx.getJobletContext().getServiceContext().getNodeId(); final DynamicContext dCtx = (DynamicContext) ctx.getJobletContext().getGlobalJobData(); final ArrayBackedValueStorage jsonAbvs = new ArrayBackedValueStorage(); final String collectionName = collectionPartitions[partition % collectionPartitions.length]; @@ -157,7 +157,7 @@ public void nextFrame(ByteBuffer buffer) throws HyracksDataException { try { hdfs.scheduleSplits(); ArrayList schedule = hdfs - .getScheduleForNode(InetAddress.getLocalHost().getHostAddress()); + .getScheduleForNode(Inet4Address.getLoopbackAddress().getHostAddress()); List splits = hdfs.getSplits(); List fileSplits = new ArrayList<>(); for (int i : schedule) { diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java index 356371340..31ef46d5d 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java @@ -80,7 +80,7 @@ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final IFrameFieldAppender appender = new FrameFixedFieldTupleAppender(fieldOutputCount); final short partitionId = (short) ctx.getTaskAttemptId().getTaskId().getPartition(); final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider(partitionId, dataSourceId, totalDataSources); - final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); + final String nodeId = ctx.getJobletContext().getServiceContext().getNodeId(); final String collectionName = collectionPartitions[partition % collectionPartitions.length]; final String collectionModifiedName = collectionName.replace("${nodeId}", nodeId); IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil( diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java index 15fd62474..6d63d5fe2 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java @@ -57,7 +57,7 @@ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvalu final DataInputStream di = new DataInputStream(bbis); final int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); - final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); + final String nodeId = ctx.getJobletContext().getServiceContext().getNodeId(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java index 2fd1755ae..e3157afdb 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java @@ -57,7 +57,7 @@ protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvalu final DataInputStream di = new DataInputStream(bbis); final int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); - final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); + final String nodeId = ctx.getJobletContext().getServiceContext().getNodeId(); return new AbstractTaggedValueArgumentScalarEvaluator(args) { @Override diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java index 0e5b48105..148dfa540 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.IOException; +import java.net.Inet4Address; import java.net.InetAddress; import java.net.UnknownHostException; @@ -45,26 +46,25 @@ private TestClusterUtil() { } public static CCConfig createCCConfig() throws UnknownHostException { - String publicAddress = InetAddress.getLocalHost().getHostAddress(); + String publicAddress = Inet4Address.getLoopbackAddress().getHostAddress(); CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = publicAddress; - ccConfig.clientNetPort = CLIENT_NET_PORT; - ccConfig.clusterNetIpAddress = publicAddress; - ccConfig.clusterNetPort = CLUSTER_NET_PORT; - ccConfig.profileDumpPeriod = PROFILE_DUMP_PERIOD; + ccConfig.setClientListenAddress(publicAddress); + ccConfig.setClientListenPort(CLIENT_NET_PORT); + ccConfig.setClusterListenAddress(publicAddress); + ccConfig.setClusterListenPort(CLUSTER_NET_PORT); + ccConfig.setProfileDumpPeriod(PROFILE_DUMP_PERIOD); return ccConfig; } public static NCConfig createNCConfig() throws UnknownHostException { - String publicAddress = InetAddress.getLocalHost().getHostAddress(); - NCConfig ncConfig1 = new NCConfig(); - ncConfig1.ccHost = CC_HOST; - ncConfig1.ccPort = CLUSTER_NET_PORT; - ncConfig1.clusterNetIPAddress = publicAddress; - ncConfig1.dataIPAddress = publicAddress; - ncConfig1.resultIPAddress = publicAddress; - ncConfig1.nodeId = NODE_ID; - ncConfig1.ioDevices = IO_DEVICES; + String publicAddress = Inet4Address.getLoopbackAddress().getHostAddress(); + NCConfig ncConfig1 = new NCConfig(NODE_ID); + ncConfig1.setClusterAddress(CC_HOST); + ncConfig1.setClusterPort(CLUSTER_NET_PORT); + ncConfig1.setClusterListenAddress(publicAddress); + ncConfig1.setDataPublicAddress(publicAddress); + ncConfig1.setResultPublicAddress(publicAddress); + ncConfig1.setIODevices(new String[] { IO_DEVICES }); return ncConfig1; } @@ -75,11 +75,11 @@ public static ClusterControllerService startCC(XTestOptions opts) throws IOExcep File ccRoot = File.createTempFile(TestRunner.class.getName(), ".data", outDir); ccRoot.delete(); ccRoot.mkdir(); - ccConfig.ccRoot = ccRoot.getAbsolutePath(); + ccConfig.setRootDir(ccRoot.getAbsolutePath()); try { ClusterControllerService cc = new ClusterControllerService(ccConfig); cc.start(); - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + hcc = new HyracksConnection(ccConfig.getClientListenAddress(), ccConfig.getClientListenPort()); hds = new HyracksDataset(hcc, opts.frameSize, opts.threads); return cc; } catch (Exception e) { diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java index fa0a90034..a367bd3e8 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java @@ -158,7 +158,7 @@ public void runQuery(TestCase testCase, TestCaseResult res) { IFrame frame = new VSizeFrame(resultDisplayFrameMgr); IHyracksDatasetReader reader = hds.createReader(jobId, ccb.getResultSetId()); // TODO(tillw) remove this loop once the IHyracksDatasetReader reliably returns the correct exception - while (reader.getResultStatus() == DatasetJobRecord.Status.RUNNING) { + while (reader.getResultStatus().getState() == DatasetJobRecord.State.RUNNING) { Thread.sleep(1); } IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor(); From bfbe2959cbc6010bb79c8afa881ceb2c211c1f96 Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sat, 13 May 2017 16:28:42 -0700 Subject: [PATCH 11/14] More fixes for current Hyracks 0.3.2-SNAPSHOT --- .../algebricks/VXQueryComparatorFactoryProvider.java | 6 ++++++ .../rewriter/rules/IntroduceTwoStepAggregateRule.java | 9 +++++---- .../metadata/VXQueryCollectionOperatorDescriptor.java | 2 +- .../metadata/VXQueryIndexingOperatorDescriptor.java | 4 ++-- .../index/ShowIndexesScalarEvaluatorFactory.java | 2 +- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryComparatorFactoryProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryComparatorFactoryProvider.java index b7196cfc1..be76a22e9 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryComparatorFactoryProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryComparatorFactoryProvider.java @@ -30,6 +30,12 @@ public IBinaryComparatorFactory getBinaryComparatorFactory(Object type, boolean return new BinaryComparatorFactory(type, ascending); } + @Override + public IBinaryComparatorFactory getBinaryComparatorFactory(Object type, boolean ascending, boolean ignoreCase) + throws AlgebricksException { + throw new UnsupportedOperationException(); + } + private static class BinaryComparatorFactory implements IBinaryComparatorFactory { private static final long serialVersionUID = 1L; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceTwoStepAggregateRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceTwoStepAggregateRule.java index 806b53292..df76805a4 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceTwoStepAggregateRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceTwoStepAggregateRule.java @@ -20,9 +20,6 @@ import java.util.Map; import org.apache.commons.lang3.mutable.Mutable; -import org.apache.vxquery.functions.BuiltinFunctions; -import org.apache.vxquery.functions.BuiltinOperators; - import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -37,6 +34,8 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator; import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; +import org.apache.vxquery.functions.BuiltinFunctions; +import org.apache.vxquery.functions.BuiltinOperators; /** * The rule searches for aggregate operators with an aggregate function @@ -107,9 +106,11 @@ public boolean rewritePre(Mutable opRef, IOptimizationContext if (aggregateFunctionCall.isTwoStep()) { return false; } - aggregateFunctionCall.setTwoStep(true); + aggregateFunctionCall = new AggregateFunctionCallExpression(aggregateFunctionCall.getFunctionInfo(), true, + aggregateFunctionCall.getArguments()); aggregateFunctionCall.setStepOneAggregate(AGGREGATE_MAP.get(functionCall.getFunctionIdentifier()).first); aggregateFunctionCall.setStepTwoAggregate(AGGREGATE_MAP.get(functionCall.getFunctionIdentifier()).second); + aggregate.getExpressions().get(0).setValue(aggregateFunctionCall); return true; } return false; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index d214fd781..005209dba 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -92,7 +92,7 @@ public VXQueryCollectionOperatorDescriptor(IOperatorDescriptorRegistry spec, Abs totalDataSources = (short) ds.getTotalDataSources(); childSeq = ds.getChildSeq(); valueSeq = ds.getValueSeq(); - recordDescriptors[0] = rDesc; + outRecDescs[0] = rDesc; this.tag = ds.getTag(); this.hdfsConf = hdfsConf; this.nodeControllerInfos = nodeControllerInfos; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java index 31ef46d5d..929a6fc8e 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryIndexingOperatorDescriptor.java @@ -66,7 +66,7 @@ public VXQueryIndexingOperatorDescriptor(IOperatorDescriptorRegistry spec, VXQue collectionPartitions = ds.getPartitions(); dataSourceId = (short) ds.getDataSourceId(); totalDataSources = (short) ds.getTotalDataSources(); - recordDescriptors[0] = rDesc; + outRecDescs[0] = rDesc; childSeq = ds.getChildSeq(); } @@ -84,7 +84,7 @@ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final String collectionName = collectionPartitions[partition % collectionPartitions.length]; final String collectionModifiedName = collectionName.replace("${nodeId}", nodeId); IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil( - ctx.getIOManager().getIODevices().get(0).getMount()); + ctx.getIoManager().getIODevices().get(0).getMount()); indexCentralizerUtil.readIndexDirectory(); final IPointable result = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java index 6b18b3305..6004d7902 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexesScalarEvaluatorFactory.java @@ -52,7 +52,7 @@ protected void evaluate(TaggedValuePointable[] args, IPointable result) throws S abvs.reset(); sb.reset(abvs); IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil( - ctx.getIOManager().getIODevices().get(0).getMount()); + ctx.getIoManager().getIODevices().get(0).getMount()); indexCentralizerUtil.readIndexDirectory(); indexCentralizerUtil.getAllCollections(sb); sb.finish(); From 20bd06f830a49a8ab0cf8ed769ad023dffad2799 Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sun, 14 May 2017 22:56:05 -0700 Subject: [PATCH 12/14] Enable verbose logging for VXQueryIT --- .../src/test/java/org/apache/vxquery/xtest/VXQueryIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java index c6f54f636..e8aee89cb 100644 --- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java +++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java @@ -45,6 +45,7 @@ public static XTestOptions getOptions() { XTestOptions options = getDefaultTestOptions(); options.catalog = XQTS_CATALOG; options.previousTestResults = StringUtils.join(new String[] { "results", "xqts.txt" }, File.separator); + options.verbose = true; return options; } From 41918ae112b941dc1343cad7d707c5796212d2bb Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sat, 27 May 2017 20:15:21 -0700 Subject: [PATCH 13/14] Add option to set implicit timezone for tests - The implicit timezone for VXQueryIT is set to "PST". --- .../apache/vxquery/context/DynamicContextImpl.java | 9 +++++++++ .../vxquery/context/DynamicContextImplFactory.java | 11 +++++------ .../accessors/atomic/XSDateTimePointable.java | 5 ++++- .../java/org/apache/vxquery/xtest/TestRunner.java | 13 +++++++++++++ .../java/org/apache/vxquery/xtest/XTestOptions.java | 3 +++ .../java/org/apache/vxquery/xtest/VXQueryIT.java | 2 +- 6 files changed, 35 insertions(+), 8 deletions(-) diff --git a/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImpl.java b/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImpl.java index 6a2a93059..7805c7213 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImpl.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImpl.java @@ -24,6 +24,7 @@ import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.data.std.api.IValueReference; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.vxquery.datamodel.accessors.atomic.XSDateTimePointable; public class DynamicContextImpl implements DynamicContext { private StaticContext sCtx; @@ -56,6 +57,14 @@ public void setCurrentDateTime(IValueReference value) { @Override public void getCurrentDateTime(IPointable value) { + if (currentDateTime == null) { + // if not set, get it from the JVM + final int dtLen = XSDateTimePointable.TYPE_TRAITS.getFixedLength(); + currentDateTime = new byte[dtLen]; + XSDateTimePointable datetimep = new XSDateTimePointable(); + datetimep.set(currentDateTime, 0, dtLen); + datetimep.setCurrentDateTime(); + } value.set(currentDateTime, 0, currentDateTime.length); } diff --git a/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImplFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImplFactory.java index a30c5adaa..c00ffc15c 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImplFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/context/DynamicContextImplFactory.java @@ -69,12 +69,11 @@ public DynamicContext createDynamicContext(IHyracksJobletContext ctx) { static IDynamicContextFactory createInstance(DynamicContextImpl dCtx) { IStaticContextFactory scFactory = dCtx.getStaticContext().createFactory(); - final int dtLen = XSDateTimePointable.TYPE_TRAITS.getFixedLength(); - byte[] currentDateTime = new byte[dtLen]; - XSDateTimePointable datetimep = new XSDateTimePointable(); - datetimep.set(currentDateTime, 0, dtLen); - datetimep.setCurrentDateTime(); - + VoidPointable vp = new VoidPointable(); + dCtx.getCurrentDateTime(vp); + byte[] currentDateTime = new byte[vp.getLength()]; + vp.copyInto(currentDateTime); + Map vMap = dCtx.getVariableMap(); int nVars = vMap.size(); QName[] variableNames = new QName[nVars]; diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java index 0f3a44a2e..40cf55b66 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java @@ -76,7 +76,10 @@ public ITypeTraits getTypeTraits() { }; public void setCurrentDateTime() { - Calendar cal = Calendar.getInstance(); + setCurrentDateTime(Calendar.getInstance()); + } + + public void setCurrentDateTime(Calendar cal) { TimeZone tz = cal.getTimeZone(); int year = cal.get(Calendar.YEAR); int month = cal.get(Calendar.MONTH); diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java index a367bd3e8..a2cf709a9 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java @@ -22,9 +22,11 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.EnumSet; import java.util.List; import java.util.Map; +import java.util.TimeZone; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -51,6 +53,7 @@ import org.apache.vxquery.context.DynamicContextImpl; import org.apache.vxquery.context.RootStaticContextImpl; import org.apache.vxquery.context.StaticContextImpl; +import org.apache.vxquery.datamodel.accessors.atomic.XSDateTimePointable; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.result.ResultUtils; @@ -149,6 +152,16 @@ public void runQuery(TestCase testCase, TestCaseResult res) { in.close(); DynamicContext dCtx = new DynamicContextImpl(compiler.getModule().getModuleContext()); + + if (opts.timezone != null) { + final int dtLen = XSDateTimePointable.TYPE_TRAITS.getFixedLength(); + byte[] currentDateTime = new byte[dtLen]; + XSDateTimePointable datetimep = new XSDateTimePointable(); + datetimep.set(currentDateTime, 0, dtLen); + datetimep.setCurrentDateTime(Calendar.getInstance(TimeZone.getTimeZone(opts.timezone))); + dCtx.setCurrentDateTime(datetimep); + } + spec.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory())); spec.setMaxReattempts(0); diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java index 496b74a8f..d0f2760ad 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java @@ -29,6 +29,9 @@ public class XTestOptions { @Option(name = "-catalog", required = true, usage = "Test catalog XML.") String catalog; + @Option(name = "-tz", required = true, usage = "Implicit timezone for test execution (default: JVM timezone).") + String timezone; + @Option(name = "-threads", required = false, usage = "Number of threads. (default: 1)") int threads; diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java index e8aee89cb..e6dfa22d5 100644 --- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java +++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryIT.java @@ -45,7 +45,7 @@ public static XTestOptions getOptions() { XTestOptions options = getDefaultTestOptions(); options.catalog = XQTS_CATALOG; options.previousTestResults = StringUtils.join(new String[] { "results", "xqts.txt" }, File.separator); - options.verbose = true; + options.timezone = "PST"; return options; } From fbd9f691eeaf0b4c1e4a1e9a43dd7129d80628bf Mon Sep 17 00:00:00 2001 From: Till Westmann Date: Sat, 27 May 2017 20:19:18 -0700 Subject: [PATCH 14/14] The implicit timezone for tests is not required. --- .../src/main/java/org/apache/vxquery/xtest/XTestOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java index d0f2760ad..34cd5b1ee 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java @@ -29,7 +29,7 @@ public class XTestOptions { @Option(name = "-catalog", required = true, usage = "Test catalog XML.") String catalog; - @Option(name = "-tz", required = true, usage = "Implicit timezone for test execution (default: JVM timezone).") + @Option(name = "-tz", required = false, usage = "Implicit timezone for test execution (default: JVM timezone).") String timezone; @Option(name = "-threads", required = false, usage = "Number of threads. (default: 1)")