Implemented runtime for the string function "fn:matches", "fn:replace" and "fn:tokenize" for issue VXQUERY-67.

pom.xml

@@ -508,7 +508,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <hyracks.version>0.2.16-incubating</hyracks.version>
+        <hyracks.version>0.2.17-incubating</hyracks.version>
         <apache-rat-plugin.version>0.11</apache-rat-plugin.version>
     </properties>
 

vxquery-core/src/main/java/org/apache/vxquery/functions/builtin-functions.xml

@@ -527,6 +527,7 @@
         <param name="input" type="xs:string?"/>
         <param name="pattern" type="xs:string"/>
         <return type="xs:boolean"/>
+        <runtime type="scalar" class="org.apache.vxquery.runtime.functions.strings.FnMatchesEvaluatorFactory"/>
     </function>
 
     <!-- fn:matches($input  as xs:string?, $pattern as xs:string, $flags as xs:string)  as xs:boolean -->
@@ -535,6 +536,7 @@
         <param name="pattern" type="xs:string"/>
         <param name="flags" type="xs:string"/>
         <return type="xs:boolean"/>
+        <runtime type="scalar" class="org.apache.vxquery.runtime.functions.strings.FnMatchesEvaluatorFactory"/>
     </function>
 
     <!-- fn:max($arg as  xs:anyAtomicType*) as xs:anyAtomicType?  -->
@@ -751,6 +753,7 @@
         <param name="pattern" type="xs:string"/>
         <param name="replacement" type="xs:string"/>
         <return type="xs:string"/>
+        <runtime type="scalar" class="org.apache.vxquery.runtime.functions.strings.FnReplaceEvaluatorFactory"/>
     </function>
 
     <!-- fn:replace($input  as xs:string?, $pattern as xs:string, $replacement as xs:string, $flags as xs:string)  as xs:string -->
@@ -760,6 +763,7 @@
         <param name="replacement" type="xs:string"/>
         <param name="flags" type="xs:string"/>
         <return type="xs:string"/>
+        <runtime type="scalar" class="org.apache.vxquery.runtime.functions.strings.FnReplaceEvaluatorFactory"/>
     </function>
 
     <!-- fn:resolve-QName($qname  as xs:string?, $element as element())  as xs:QName?  -->
@@ -1018,6 +1022,7 @@
         <param name="input" type="xs:string?"/>
         <param name="pattern" type="xs:string"/>
         <return type="xs:string*"/>
+        <runtime type="scalar" class="org.apache.vxquery.runtime.functions.strings.FnTokenizeEvaluatorFactory"/>
     </function>
 
     <!-- fn:tokenize($input  as xs:string?, $pattern as xs:string, $flags as xs:string)  as xs:string* -->
@@ -1026,6 +1031,7 @@
         <param name="pattern" type="xs:string"/>
         <param name="flags" type="xs:string"/>
         <return type="xs:string*"/>
+        <runtime type="scalar" class="org.apache.vxquery.runtime.functions.strings.FnTokenizeEvaluatorFactory"/>
     </function>
 
     <!-- fn:trace($value as  item()*, $label as xs:string)  as item()* -->

vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/strings/FnMatchesEvaluatorFactory.java

@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.strings;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+import org.apache.vxquery.datamodel.accessors.SequencePointable;
+import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
+import org.apache.vxquery.datamodel.values.ValueTag;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
+import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
+import org.apache.vxquery.runtime.functions.util.FunctionHelper;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+public class FnMatchesEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
+    private static final long serialVersionUID = 1L;
+
+    public FnMatchesEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+        super(args);
+    }
+
+    @Override
+    protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
+            throws AlgebricksException {
+        final UTF8StringPointable stringp1 = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
+        final UTF8StringPointable stringp2 = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
+        final UTF8StringPointable stringp3 = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
+        final StringBuilder builder1 = new StringBuilder();
+        final StringBuilder builder2 = new StringBuilder();
+        final StringBuilder builder3 = new StringBuilder();
+        final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable();
+        final TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
+
+        return new AbstractTaggedValueArgumentScalarEvaluator(args) {
+            private Pattern pattern = null;
+            private Matcher matcher = null;
+            private String s1, s2, s3;
+
+            @Override
+            protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
+                // Default result is false.
+                byte[] booleanResult = new byte[2];
+                booleanResult[0] = ValueTag.XS_BOOLEAN_TAG;
+                booleanResult[1] = 0;
+
+                TaggedValuePointable tvp1 = args[0];
+                TaggedValuePointable tvp2 = args[1];
+
+                // Only accept strings as input.
+                PatternMatchingEvaluatorUtils.checkInput(tvp, tvp1, seqp, stringp1);
+                stringp1.toString(builder1);
+                s1 = builder1.toString();
+                PatternMatchingEvaluatorUtils.checkInput(tvp, tvp2, seqp, stringp2);
+                stringp2.toString(builder2);
+                s2 = builder2.toString();
+                // Third parameter is optional.
+                if (args.length > 2) {
+                    TaggedValuePointable tvp3 = args[2];
+                    if (!FunctionHelper.isDerivedFromString(tvp3.getTag())) {
+                        throw new SystemException(ErrorCode.FORG0006);
+                    }
+                    tvp3.getValue(stringp3);
+                    stringp3.toString(builder3);
+                    s3 = builder3.toString();
+                    try {
+                        if (s3.contains("q")) {
+                            s2 = Pattern.quote(s2);
+                        }
+                        pattern = Pattern.compile(s2, PatternMatchingEvaluatorUtils.toFlag(s3));
+                    } catch (PatternSyntaxException e) {
+                        throw new SystemException(ErrorCode.FORX0002);
+                    } catch (IllegalArgumentException e) {
+                        throw new SystemException(ErrorCode.FORX0001);
+                    }
+                }
+                if (pattern == null) {
+                    try {
+                        pattern = Pattern.compile(s2);
+                    } catch (PatternSyntaxException e) {
+                        throw new SystemException(ErrorCode.FORX0002);
+                    }
+                }
+
+                matcher = pattern.matcher(builder1);
+
+                if (matcher.find()) {
+                    booleanResult[1] = 1;
+                }
+                result.set(booleanResult, 0, 2);
+            }
+        };
+    }
+}

vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/strings/FnReplaceEvaluatorFactory.java

@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.runtime.functions.strings;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.vxquery.datamodel.accessors.SequencePointable;
+import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
+import org.apache.vxquery.datamodel.values.ValueTag;
+import org.apache.vxquery.exceptions.ErrorCode;
+import org.apache.vxquery.exceptions.SystemException;
+import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator;
+import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory;
+import org.apache.vxquery.runtime.functions.util.FunctionHelper;
+
+public class FnReplaceEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory {
+    private static final long serialVersionUID = 1L;
+
+    public FnReplaceEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+        super(args);
+    }
+
+    @Override
+    protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args)
+            throws AlgebricksException {
+        final UTF8StringPointable stringp1 = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
+        final UTF8StringPointable stringp2 = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
+        final UTF8StringPointable stringp3 = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
+        final UTF8StringPointable stringp4 = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
+        final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+        final StringBuilder builder1 = new StringBuilder();
+        final StringBuilder builder2 = new StringBuilder();
+        final StringBuilder builder3 = new StringBuilder();
+        final StringBuilder builder4 = new StringBuilder();
+        final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable();
+        final TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
+
+        return new AbstractTaggedValueArgumentScalarEvaluator(args) {
+            private Pattern pattern = null;
+            private Matcher matcher = null;
+
+            @Override
+            protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException {
+                try {
+                    // Byte Format: Type (1 byte) + String Length (2 bytes) + String.
+                    DataOutput out = abvs.getDataOutput();
+                    out.write(ValueTag.XS_STRING_TAG);
+
+                    // Default values for the length and update later
+                    out.write(0);
+                    out.write(0);
+
+                    TaggedValuePointable tvp1 = args[0];
+                    TaggedValuePointable tvp2 = args[1];
+                    TaggedValuePointable tvp3 = args[2];
+
+                    PatternMatchingEvaluatorUtils.checkInput(tvp, tvp1, seqp, stringp1);
+                    stringp1.toString(builder1);
+                    PatternMatchingEvaluatorUtils.checkInput(tvp, tvp2, seqp, stringp2);
+                    stringp2.toString(builder2);
+                    String s2 = builder2.toString();
+                    PatternMatchingEvaluatorUtils.checkInput(tvp, tvp3, seqp, stringp3);
+                    stringp3.toString(builder3);
+
+                    // Fourth parameter is optional.
+                    if (args.length > 3) {
+                        TaggedValuePointable tvp4 = args[3];
+                        if (!FunctionHelper.isDerivedFromString(tvp4.getTag())) {
+                            throw new SystemException(ErrorCode.FORG0006);
+                        }
+                        tvp4.getValue(stringp4);
+                        stringp4.toString(builder4);
+                        String s4 = builder4.toString();
+                        try {
+                            if (s4.contains("q")) {
+                                s2 = Pattern.quote(s2);
+                            }
+                            pattern = Pattern.compile(s2, PatternMatchingEvaluatorUtils.toFlag(s4));
+                        } catch (PatternSyntaxException e) {
+                            throw new SystemException(ErrorCode.FORX0002);
+                        } catch (IllegalArgumentException e) {
+                            throw new SystemException(ErrorCode.FORX0002);
+                        }
+                    }
+                    if (pattern == null) {
+                        try {
+                            pattern = Pattern.compile(builder2.toString());
+                        } catch (PatternSyntaxException e) {
+                            throw new SystemException(ErrorCode.FORX0002);
+                        }
+                    }
+
+                    matcher = pattern.matcher(builder1.toString());
+                    try {
+                        String replaced = matcher.replaceAll(builder3.toString());
+                        out.write(replaced.getBytes(StandardCharsets.UTF_8));
+                    } catch (IndexOutOfBoundsException e) {
+                        throw new SystemException(ErrorCode.FORX0003);
+                    } catch (IllegalArgumentException e) {
+                        throw new SystemException(ErrorCode.FORX0004);
+                    }
+
+                    abvs.getByteArray()[1] = (byte) (((abvs.getLength() - 3) >>> 8) & 0xFF);
+                    abvs.getByteArray()[2] = (byte) (((abvs.getLength() - 3) >>> 0) & 0xFF);
+
+                    result.set(abvs.getByteArray(), abvs.getStartOffset(), abvs.getLength());
+                } catch (IOException e) {
+                    throw new SystemException(ErrorCode.SYSE0001, e);
+                }
+            }
+        };
+    }
+}