diff --git a/build.sbt b/build.sbt old mode 100644 new mode 100755 index d3c904a..6d40410 --- a/build.sbt +++ b/build.sbt @@ -45,7 +45,8 @@ libraryDependencies ++= Seq( "javax.activation" % "javax.activation-api" % "1.2.0", // JAXB - Needed for Java 9+ since it is no longer automatically available "joda-time" % "joda-time" % "2.9.1", "org.joda" % "joda-convert" % "1.8", // Required by joda-time when using Scala - "org.mongodb" % "bson" % "3.3.0" + "org.mongodb" % "bson" % "3.3.0", + "org.yaml" % "snakeyaml" % "1.23", // YAML => JSON ) libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.5" % "test" diff --git a/src/main/scala/fm/serializer/json/JSONInput.scala b/src/main/scala/fm/serializer/json/JSONInput.scala old mode 100644 new mode 100755 diff --git a/src/main/scala/fm/serializer/yaml/YAML.scala b/src/main/scala/fm/serializer/yaml/YAML.scala new file mode 100755 index 0000000..ed0bd2e --- /dev/null +++ b/src/main/scala/fm/serializer/yaml/YAML.scala @@ -0,0 +1,113 @@ +/* + * Copyright 2019 Frugal Mechanic (http://frugalmechanic.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fm.serializer.yaml + +import fm.common.IOUtils +import fm.common.Logging +import fm.serializer.{Deserializer, Serializer} +import java.io._ +import java.nio.charset.StandardCharsets.UTF_8 +import org.yaml.snakeyaml.DumperOptions +import org.yaml.snakeyaml.emitter.Emitter +import org.yaml.snakeyaml.error.YAMLException +import org.yaml.snakeyaml.resolver.Resolver +import org.yaml.snakeyaml.serializer.{Serializer => YAMLSerializer} + +object YAML extends Logging { + // TODO: Figure out if these can/should be tweaked for our output. + private def makeDumperOptions(): DumperOptions = { + val options: DumperOptions = new DumperOptions() + + options.setDefaultScalarStyle(DumperOptions.ScalarStyle.PLAIN) + options.setDefaultFlowStyle(DumperOptions.FlowStyle.AUTO) + + options + } + + private[this] val defaultDumperOptions = makeDumperOptions + private[this] val minimalDumperOptions = makeDumperOptions + private[this] val prettyDumperOptions = makeDumperOptions + private[this] val defaultResolver = new Resolver() + + // writeToBsonWriter(v, new BsonBinaryWriter(buf)) + + def toYAML[@specialized T](v: T)(implicit serializer: Serializer[T]): String = { + toYAML(v, defaultDumperOptions) + } + + def toMinimalYAML[@specialized T](v: T)(implicit serializer: Serializer[T]): String = { + toYAML(v, minimalDumperOptions) + } + + def toPrettyYAML[@specialized T](v: T)(implicit serializer: Serializer[T]): String = { + toYAML(v, prettyDumperOptions) + } + + def toYAML[@specialized T](v: T, options: DumperOptions)(implicit serializer: Serializer[T]): String = { + val writer: StringWriter = new StringWriter() + writeYAML(v, writer, options) + writer.toString + } + + def toBytes[@specialized T](v: T)(implicit serializer: Serializer[T]): Array[Byte] = { + toBytes(v, defaultDumperOptions) + } + + def toBytes[@specialized T](v: T, options: DumperOptions)(implicit serializer: Serializer[T]): Array[Byte] = { + toYAML(v, options).getBytes(UTF_8) + } + + def fromYAML[@specialized T](yaml: String)(implicit deserializer: Deserializer[T]): T = { + fromYAML(yaml, YAMLOptions.default) + } + + def fromYAML[@specialized T](yaml: String, options: YAMLOptions)(implicit deserializer: Deserializer[T]): T = { + val input: YAMLReaderInput = new YAMLReaderInput(new StringReader(yaml), options) + deserializer.deserializeRaw(input) + } + + def fromBytes[@specialized T](bytes: Array[Byte])(implicit deserializer: Deserializer[T]): T = { + fromBytes(bytes, YAMLOptions.default) + } + + def fromBytes[@specialized T](bytes: Array[Byte], options: YAMLOptions)(implicit deserializer: Deserializer[T]): T = { + val bis: ByteArrayInputStream = new ByteArrayInputStream(bytes) + val charset: String = IOUtils.detectCharsetName(bis, useMarkReset = true).getOrElse(UTF_8.name) + val reader: InputStreamReader = new InputStreamReader(bis, charset) + + fromReader(reader, options) + } + + def fromReader[@specialized T](reader: Reader)(implicit deserializer: Deserializer[T]): T = { + fromReader(reader, YAMLOptions.default) + } + + def fromReader[@specialized T](reader: Reader, options: YAMLOptions)(implicit deserializer: Deserializer[T]): T = { + deserializer.deserializeRaw(new YAMLReaderInput(reader, options)) + } + + private def writeYAML[T](v: T, writer: Writer, dumperOptions: DumperOptions)(implicit serializer: Serializer[T]): Unit = { + val yamlSerializer: YAMLSerializer = new YAMLSerializer(new Emitter(writer, dumperOptions), defaultResolver, dumperOptions, null) + + //try { + yamlSerializer.open() + serializer.serializeRaw(new YAMLOutput(yamlSerializer, dumperOptions), v) + yamlSerializer.close() + //} catch { + // case ex: Exception => throw new YAMLException(ex) // TODO: better exception? + //} + } +} \ No newline at end of file diff --git a/src/main/scala/fm/serializer/yaml/YAMLInput.scala b/src/main/scala/fm/serializer/yaml/YAMLInput.scala new file mode 100755 index 0000000..54838a9 --- /dev/null +++ b/src/main/scala/fm/serializer/yaml/YAMLInput.scala @@ -0,0 +1,296 @@ +/* + * Copyright 2019 Frugal Mechanic (http://frugalmechanic.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fm.serializer.yaml + +import fm.serializer.{CollectionInput, FieldInput, Input} +import fm.common.Implicits._ +import fm.common.Logging +import org.yaml.snakeyaml.events._ +import fm.serializer.base64.Base64 +import org.yaml.snakeyaml.nodes.Tag + +// (Correct) YAML Uses Spaces for indentation: https://yaml.org/spec/1.1/#id871998 +abstract class YAMLInput(options: YAMLOptions) extends Input with Logging { + def allowStringMap: Boolean = true + + // + // HELPERS + // + + protected def debug(): Unit + + protected def hasNext(): Boolean + + /** Current Event, does not consume it */ + protected def headOption(): Option[Event] + private def head: Event = headOption.get + + /** Return peek and advance to the next event */ + protected def next(): Event + + /** Checks for end of document or end of stream */ + protected def isEOF(event: Event): Boolean + + /** Verifies the type and reads to next event */ + protected def readNextRequired(tpe: Event.ID): Unit + + /* + private def readTag(): String = { + head match { + case scalar: ScalarEvent => scalar.getTag + case collection: CollectionStartEvent => collection.getTag + case map: MappingStartEvent => map.getTag + //case node: NodeEvent => node.getTag + case e: Event => throw new IllegalStateException(s"Expected to read an event with a tag, but got Event: $e") + } + } + + private def readAnchor(): String = { + head match { + case scalar: ScalarEvent => scalar.getAnchor + case collection: CollectionStartEvent => collection.getAnchor + case map: MappingStartEvent => map.getAnchor + //case node: NodeEvent => node.getTag + case e: Event => throw new IllegalStateException(s"Expected to read an event with a tag, but got Event: $e") + } + } + */ + + private def isEOFOrMappingEnd(event: Event): Boolean = isEOF(event) || event.is(Event.ID.MappingEnd) + + def setIsChildren(): Unit + + private def isEndMarker(event: Event): Boolean = { + isEOFOrMappingEnd(event) || event.is(Event.ID.SequenceEnd) + } + + def readFieldName(): String = { + logger.error(s"readFieldName!") + + if (nextValueIsNull) return null + + val ret: String = head match { + case scalar: ScalarEvent => scalar.getValue + //case node: NodeEvent => node.getTag + case e: Event => throw new IllegalStateException(s"Expected to read an event with a tag, but got Event: $e") + } + + + next() + + logger.error(s"readFieldName(): ${ret}, next is: $head") + ret + } + + // + // FIELD Input + // + def readFieldNumber(nameToNumMap: Map[String, Int]): Int = { + logger.error(s"readFieldNumber!") + val name: String = readFieldName() + + if (null == name) return 0 + + try { + // Exact name match + nameToNumMap(name) + } catch { + case _: NoSuchElementException => + // TODO: possibly require that the map be pre-populated with the lower case versions so we don't have to search through it + val lowerName: String = name.toLowerCase + if (nameToNumMap.contains(lowerName)) nameToNumMap(lowerName) + else nameToNumMap.find{ case (n,i) => n.toLowerCase == lowerName }.map{ _._2 }.getOrElse(-1) + } + } + + + def skipUnknownField(): Unit = { + logger.error(s"skipUnknownField!") + require(headOption.isDefined, "Expected headOption") + head match { + case e: ScalarEvent => readRawString() + case e: MappingStartEvent => readRawObject(skipRawObjectFun) + case e: SequenceStartEvent => readRawCollection(skipRawCollectionFun) + case e: Event => throw new IllegalStateException(s"Trying to skip an unknown field, no handler for event: $e") + } + } + + // This avoids the object creations in skipUnknownField() + private[this] val skipRawObjectFun: Function1[FieldInput, AnyRef] = new Function1[FieldInput, AnyRef] { + def apply(in: FieldInput): AnyRef = { + while(in.readFieldNumber(Map.empty) != 0) in.skipUnknownField() + null + } + } + + // This avoids the object creations in skipUnknownField() + private[this] val skipRawCollectionFun: Function1[CollectionInput, AnyRef] = new Function1[CollectionInput, AnyRef] { + def apply(in: CollectionInput): AnyRef = { + while(in.hasAnotherElement) skipUnknownField() + null + } + } + + // + // COLLECTION Input + // + def hasAnotherElement: Boolean = head.is(Event.ID.Scalar) + + // + // RAW Input + // + + private def readStringValue(): String = { + logger.error(s"readStringValue(), nextValueIsNull: ${nextValueIsNull}") + + if (nextValueIsNull) null + else { + val ret = head match { + case scalar: ScalarEvent => scalar.getValue + case e: Event => throw new IllegalStateException(s"Expected to read a scalar value, but read: $e") + } + + next() + + logger.error(s"readStringValue() ret: $ret and next: $head") + ret + } + } + + // Basic Types + def readRawBool(): Boolean = readStringValue().parseBoolean.get + def readRawFloat(): Float = readStringValue().toFloat + def readRawDouble(): Double = readStringValue().toDouble + + def readRawString(): String = { + if (options.internStrings) readStringValue().intern() else readStringValue() + } + + // Bytes + def readRawByteArray(): Array[Byte] = { + val s: String = readStringValue() + if (null == s) null else Base64.decode(s) + } + + // Ints + def readRawInt(): Int = readStringValue().toInt + def readRawUnsignedInt(): Int = readRawInt() + def readRawSignedInt(): Int = readRawInt() + def readRawFixedInt(): Int = readRawInt() + + // Longs + def readRawLong(): Long = readStringValue().toLong + def readRawUnsignedLong(): Long = readRawLong() + def readRawSignedLong(): Long = readRawLong() + def readRawFixedLong(): Long = readRawLong() + + def nextValueIsNull: Boolean = { + headOption.exists{ isEndMarker } || (head match { + case scalar: ScalarEvent => + val ret: Boolean = Tag.NULL.equals(scalar.getTag) || scalar.getValue.isNullOrBlank || (scalar.getValue === "null") + if (ret) next() + ret + //case beginMapping: MappingStartEvent => true + //case beginCollection: SequenceStartEvent => true + case event: Event => false + }) + } + + // Objects + def readRawObject[T](f: FieldInput => T): T = { + logger.error("readRawObject!") + + if (nextValueIsNull) return null.asInstanceOf[T] + + readNextRequired(Event.ID.MappingStart) + + logger.error("readRawObject, after mapping start check") + + if (head.is(Event.ID.MappingEnd)) { + next() + null.asInstanceOf[T] + } else { + val res: T = f(this) + logger.error(s"readRawObject after mapping start. head now: $head, and ${res}") + readNextRequired(Event.ID.MappingEnd) + + res + } + } + + // Collections + def readRawCollection[T](f: CollectionInput => T): T = { + logger.error(s"readRawCollection. $head, nextValueIsNull: ${nextValueIsNull}") + + if (nextValueIsNull/* && !head.is(Event.ID.SequenceStart)*/) return null.asInstanceOf[T] + + + readNextRequired(Event.ID.SequenceStart) + + logger.error(s"readRawCollection after sequence start. $head") + if (head.is(Event.ID.SequenceEnd)) { + next() + null.asInstanceOf[T] + } else { + val res: T = f(this) + + logger.error(s"readRawCollection after sequence start. head now: $head, and ${res}, ${f}") + readNextRequired(Event.ID.SequenceEnd) + + res + } + } + + // + // NESTED Input + // + + // Basic Types + def readNestedBool(): Boolean = readRawBool() + + def readNestedFloat(): Float = readRawFloat() + + def readNestedDouble(): Double = readRawDouble() + + def readNestedString(): String = readRawString() + + // Bytes + def readNestedByteArray(): Array[Byte] = readRawByteArray() + + // Ints + def readNestedInt(): Int = readRawInt() + def readNestedUnsignedInt(): Int = readNestedInt() + def readNestedSignedInt(): Int = readNestedInt() + def readNestedFixedInt(): Int = readNestedInt() + + // Longs + def readNestedLong(): Long = readRawLong() + def readNestedUnsignedLong(): Long = readNestedLong() + def readNestedSignedLong(): Long = readNestedLong() + def readNestedFixedLong(): Long = readNestedLong() + + // Objects + def readNestedObject[T](f: FieldInput => T): T = { + logger.error(s"readNestedObject") + readRawObject(f) + } + + // Collections + def readNestedCollection[T](f: CollectionInput => T): T = { + logger.error(s"readNestedCollection") + readRawCollection(f) + } +} diff --git a/src/main/scala/fm/serializer/yaml/YAMLOptions.scala b/src/main/scala/fm/serializer/yaml/YAMLOptions.scala new file mode 100755 index 0000000..d3c76ea --- /dev/null +++ b/src/main/scala/fm/serializer/yaml/YAMLOptions.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2019 Frugal Mechanic (http://frugalmechanic.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fm.serializer.yaml + +object YAMLOptions { + val default: YAMLOptions = YAMLOptions() + val internStrings: YAMLOptions = YAMLOptions(internStrings = true) + + // // Serializer(Emitable emitter, Resolver resolver, DumperOptions opts, Tag rootTag) +} + +/** + * @param internStrings Call String.intern() on any strings read + */ +final case class YAMLOptions( + internStrings: Boolean = false + // TODO: Add the YAMLOutput options into here and maybe split between serializer/deserialier options +) { + +} \ No newline at end of file diff --git a/src/main/scala/fm/serializer/yaml/YAMLOutput.scala b/src/main/scala/fm/serializer/yaml/YAMLOutput.scala new file mode 100755 index 0000000..dd2d572 --- /dev/null +++ b/src/main/scala/fm/serializer/yaml/YAMLOutput.scala @@ -0,0 +1,308 @@ +/* + * Copyright 2019 Frugal Mechanic (http://frugalmechanic.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fm.serializer.yaml + +import fm.common.Logging +import fm.common.Implicits._ +import fm.serializer.{FieldOutput, NestedOutput, Output} +import java.util.{ArrayList => JavaArrayList} +import org.yaml.snakeyaml.representer.Representer +import org.yaml.snakeyaml.nodes._ +import org.yaml.snakeyaml.serializer.{Serializer => YAMLSerializer} +import org.yaml.snakeyaml.DumperOptions + +object YAMLOutput { + private val representer: Representer = { + val res: Representer = new Representer() + + // Not exactly sure why these defaults don't get picked up without being explicit here... + res.setDefaultScalarStyle(DumperOptions.ScalarStyle.PLAIN) + res.setDefaultFlowStyle(DumperOptions.FlowStyle.AUTO) + + res + } + + private def NullNode(): Node = representer.represent(null) + //new ScalarNode(Tag.NULL, "null", null, null, DumperOptions.ScalarStyle.PLAIN) + + private case class JavaArrayListStack[T]() extends SimpleStack[JavaArrayList[T]] { + def createNewHead: JavaArrayList[T] = { + val newHead: JavaArrayList[T] = new JavaArrayList[T]() + push(newHead) + newHead + } + } + + private case class ObjectStack() extends SimpleStack[MappingNode] + + // 2.12 scaladocs says Stack is deprecated and you should just use a List *shrug* + private sealed trait SimpleStack[T] { + // This is a pseudo-stack for keeping track of potential nested-arrays, in a FIFO List order. + // The complete Node (with children) must be created before we can do a writer.serialize(node: SequenceNode) + private var _stack: List[T] = List.empty[T] + + def head(): T = _stack.head + + def push(newHead: T): Unit = { + _stack = newHead +: _stack + } + + def pop(): Unit = { + require(_stack.nonEmpty, s"Attempted to pop ${this}, but it's empty!") + _stack = _stack.tail + } + + def isEmpty(): Boolean = _stack.isEmpty + def nonEmpty(): Boolean = _stack.nonEmpty + + override def toString(): String = s"${this.getClass.toString}(${_stack.toString})" + } +} + +final class YAMLOutput(writer: YAMLSerializer, options: DumperOptions) extends Output with Logging { + import YAMLOutput._ + + def allowStringMap: Boolean = true + + // we can't stream nested nodes, so + + // This is a pseudo-stack for keeping track of potential nested-arrays, in a FIFO List order. + // The complete Node (with children) must be created before we can do a writer.serialize(node: SequenceNode) or writer.serialize(node: MappingNode) + private val collectionStack: JavaArrayListStack[Node] = JavaArrayListStack[Node]() + private val mapChildrenStack: JavaArrayListStack[NodeTuple] = JavaArrayListStack[NodeTuple]() + + implicit def toNodeValue[T <: AnyVal](value: T): Node = representer.represent(value) + implicit def toNodeValue(value: String): Node = representer.represent(value) + implicit def toNodeValue(value: Array[Byte]): Node = representer.represent(value) + + // + // Inherited from RawOutput, these should only be used to write elements into a Collection + // + + // Basic Types + + private def addValueToCollection(value: Node): Unit = collectionStack.head.add(value) + + def writeRawBool(value: Boolean): Unit = addValueToCollection(value) + def writeRawFloat(value: Float): Unit = addValueToCollection(value) + def writeRawDouble(value: Double): Unit = addValueToCollection(value) + def writeRawString(value: String): Unit = addValueToCollection(value) + def writeRawByteArray(value: Array[Byte]): Unit = addValueToCollection(value) + + // Ints + def writeRawInt(value: Int): Unit = addValueToCollection(value) + def writeRawUnsignedInt(value: Int): Unit = addValueToCollection(value) + def writeRawSignedInt(value: Int): Unit = addValueToCollection(value) + def writeRawFixedInt(value: Int): Unit = addValueToCollection(value) + + // Longs + def writeRawLong(value: Long): Unit = addValueToCollection(value) + def writeRawUnsignedLong(value: Long): Unit = addValueToCollection(value) + def writeRawSignedLong(value: Long): Unit = addValueToCollection(value) + def writeRawFixedLong(value: Long): Unit = addValueToCollection(value) + + /** + * For writing objects. Note: that the obj is passed in for null handling + * by the implementation. If the object is not null then the function f + * will be called so the caller can write out the fields + */ + def writeRawObject[T](obj: T)(f: (FieldOutput, T) => Unit): Unit = { + require( + ( + (mapChildrenStack.isEmpty() && collectionStack.nonEmpty()) || + (mapChildrenStack.nonEmpty() === collectionStack.nonEmpty()) + ), + s"""|writeRawObject($obj) but mapChildrenStack and collectionStack conflict.: + | + |mapChildren.isEmpty && collectionStack.nonEmpty: ${(mapChildrenStack.isEmpty() && collectionStack.nonEmpty())} + |mapChildren.nonEmpty && collectionStack.nonEmpty: ${(mapChildrenStack.nonEmpty() && collectionStack.nonEmpty())} + |mapChildren.isEmpty && collectionStack.isEmpty: ${(mapChildrenStack.isEmpty() && collectionStack.isEmpty())} + | + |mapChildrenStack: + | + |$mapChildrenStack + | + |collectionStack: + | + |$collectionStack""".stripMargin + ) + + if (null == obj) { + if (collectionStack.nonEmpty()) addValueToCollection(NullNode) + else writer.serialize(NullNode) + } else { + val mapChildren: JavaArrayList[NodeTuple] = mapChildrenStack.createNewHead + val mapNode: MappingNode = new MappingNode(Tag.MAP, mapChildren, options.getDefaultFlowStyle) + + // Nested objects require creating the complete object Node so we can add it to the collection, + // and finish writing the enclosing yaml array + f(this, obj) + + mapChildrenStack.pop + + if (collectionStack.nonEmpty()) addValueToCollection(mapNode) + else writer.serialize(mapNode) + } + } + + + def writeRawCollection[T](col: T)(f: (NestedOutput, T) => Unit): Unit = { + require( + ( + (mapChildrenStack.isEmpty() && collectionStack.nonEmpty()) || + (mapChildrenStack.nonEmpty() === collectionStack.nonEmpty()) + ), + s"""|writeRawCollection($col) but mapChildrenStack and collectionStack conflict: + | + |mapChildren.isEmpty && collectionStack.nonEmpty: ${(mapChildrenStack.isEmpty() && collectionStack.nonEmpty())} + |mapChildren.nonEmpty == collectionStack.nonEmpty: ${(mapChildrenStack.nonEmpty() === collectionStack.nonEmpty())} + | + |mapChildrenStack: + | + |$mapChildrenStack + | + |collectionStack: + | + |${collectionStack}""".stripMargin + ) + + if (null == col) { + if (collectionStack.nonEmpty) addValueToCollection(NullNode) + else writer.serialize(NullNode) + } else { + val seqChildren: JavaArrayList[Node] = collectionStack.createNewHead + val seqNode: SequenceNode = new SequenceNode(Tag.SEQ, seqChildren, options.getDefaultFlowStyle) + + f(this, col) + + collectionStack.pop + + if (collectionStack.nonEmpty) addValueToCollection(seqNode) + else writer.serialize(seqNode) + } + } + + // + // Inherited from NestedOutput + // + + // Basic Types + def writeNestedBool(value: Boolean): Unit = writeRawBool(value) + def writeNestedFloat(value: Float): Unit = writeRawFloat(value) + def writeNestedDouble(value: Double): Unit = writeRawDouble(value) + def writeNestedString(value: String): Unit = writeRawString(value) + + // Bytes + def writeNestedByteArray(value: Array[Byte]): Unit = writeRawByteArray(value) + + // Ints + def writeNestedInt(value: Int): Unit = writeRawInt(value) + def writeNestedUnsignedInt(value: Int): Unit = writeRawUnsignedInt(value) + def writeNestedSignedInt(value: Int): Unit = writeRawSignedInt(value) + def writeNestedFixedInt(value: Int): Unit = writeRawFixedInt(value) + + // Longs + def writeNestedLong(value: Long): Unit = writeRawLong(value) + def writeNestedUnsignedLong(value: Long): Unit = writeRawUnsignedLong(value) + def writeNestedSignedLong(value: Long): Unit = writeRawSignedLong(value) + def writeNestedFixedLong(value: Long): Unit = writeRawFixedLong(value) + + def writeNestedObject[T](obj: T)(f: (FieldOutput, T) => Unit): Unit = { + writeRawObject(obj)(f) + } + def writeNestedCollection[T](col: T)(f: (NestedOutput, T) => Unit): Unit = writeRawCollection(col)(f) + + // + // Inherited from FieldOutput + // + + private def writeField(keyNode: Node, valueNode: Node): Unit = { + val nodeTuple = new NodeTuple(keyNode, valueNode) + writeField(nodeTuple) + } + + private def writeField(node: NodeTuple): Unit = { + logger.error(s"writeField: $node") + require(mapChildrenStack.nonEmpty, s"Attempted to write a NodeTuple ($node) but the MappingNode values stack is empty") + mapChildrenStack.head.add(node) + } + + // Basic Types + def writeFieldBool(number: Int, name: String, value: Boolean): Unit = writeField(name, value) + def writeFieldFloat(number: Int, name: String, value: Float): Unit = writeField(name, value) + def writeFieldDouble(number: Int, name: String, value: Double): Unit = writeField(name, value) + def writeFieldString(number: Int, name: String, value: String): Unit = writeField(name, value) + + // Bytes + def writeFieldByteArray(number: Int, name: String, value: Array[Byte]): Unit = writeField(name, value) + + // Ints + def writeFieldInt(number: Int, name: String, value: Int): Unit = writeField(name, value) + def writeFieldUnsignedInt(number: Int, name: String, value: Int): Unit = writeField(name, value) + def writeFieldSignedInt(number: Int, name: String, value: Int): Unit = writeField(name, value) + def writeFieldFixedInt(number: Int, name: String, value: Int): Unit = writeField(name, value) + + // Longs + def writeFieldLong(number: Int, name: String, value: Long): Unit = writeField(name, value) + def writeFieldUnsignedLong(number: Int, name: String, value: Long): Unit = writeField(name, value) + def writeFieldSignedLong(number: Int, name: String, value: Long): Unit = writeField(name, value) + def writeFieldFixedLong(number: Int, name: String, value: Long): Unit = writeField(name, value) + + // Objects + def writeFieldObject[T](number: Int, name: String, obj: T)(f: (FieldOutput, T) => Unit): Unit = { + require(mapChildrenStack.nonEmpty, s"writeFieldCollection($number, $name, $obj), but mapChildrenStack.isEmpty") + + logger.error(s"writeFieldObject($number, $name, $obj)") + + if (null == obj) { + writeFieldNull(number, name) + } else { + val mapChildren: JavaArrayList[NodeTuple] = mapChildrenStack.createNewHead + val mapNode: MappingNode = new MappingNode(Tag.MAP, mapChildren, options.getDefaultFlowStyle) + + f(this, obj) + + mapChildrenStack.pop + + writeField(name, mapNode) + } + } + + // Collections + def writeFieldCollection[T](number: Int, name: String, col: T)(f: (NestedOutput, T) => Unit): Unit = { + require(mapChildrenStack.nonEmpty, s"writeFieldCollection($number, $name, $col), but mapChildrenStack.isEmpty") + + logger.error(s"writeFieldCollection($number, $name, $col)") + + if (null == col) { + logger.error(s"writeFieldCollection($number, $name, $col) col is null") + writeFieldNull(number, name) + } else { + logger.error(s"writeFieldCollection($number, $name, $col) col is NOT null") + val seqChildren: JavaArrayList[Node] = collectionStack.createNewHead + val seqNode: SequenceNode = new SequenceNode(Tag.SEQ, seqChildren, options.getDefaultFlowStyle) + + f(this, col) + + collectionStack.pop + + writeField(name, seqNode) + } + } + + // Null + def writeFieldNull(number: Int, name: String): Unit = writeField(name, NullNode()) +} \ No newline at end of file diff --git a/src/main/scala/fm/serializer/yaml/YAMLReadInput.scala b/src/main/scala/fm/serializer/yaml/YAMLReadInput.scala new file mode 100755 index 0000000..3f8a634 --- /dev/null +++ b/src/main/scala/fm/serializer/yaml/YAMLReadInput.scala @@ -0,0 +1,89 @@ +/* + * Copyright 2019 Frugal Mechanic (http://frugalmechanic.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fm.serializer.yaml + +import fm.common.Logging +import fm.common.Util +import java.io.Reader +import org.yaml.snakeyaml.Yaml +import org.yaml.snakeyaml.events.Event +import scala.collection.JavaConverters._ + +final class YAMLReaderInput(reader: Reader, options: YAMLOptions) extends YAMLInput(options) with Logging { + private val yaml: Yaml = new Yaml() + + + private val (time: Long, events: Iterator[Event]) = Util.time { + yaml.parse(reader).iterator().asScala + } + + protected var headOption: Option[Event] = { + // Initialize headOption, can't use readNextRequired + val streamStart: Event = events.next() + require(streamStart.is(Event.ID.StreamStart), s"Expected StreamStart, but got: $streamStart") + + val documentStart: Event = events.next() + require(documentStart.is(Event.ID.DocumentStart), s"Expected StreamStart, but got: $documentStart") + + Some(events.next()) + } + + var currentField: String = "" + + logger.debug(s"SnakeYAML took $time ms to parse reader.") + + protected def debug(): Unit = { + events.foreach { e: Event => logger.error(s"Event: $e") } + } + protected def isEOF(event: Event): Boolean = event.is(Event.ID.DocumentEnd) || event.is(Event.ID.StreamEnd) + + private var done: Boolean = false + + protected def hasNext(): Boolean = !done && events.hasNext && !headOption.exists{ isEOF } + + protected def readNextRequired(tpe: Event.ID): Unit = { + require(headOption.exists{ _.is(tpe)}, s"Expected a ${tpe}, but got: $headOption") + next() + } + /** The snakeyaml Events contain both the Tag and the Value, so we need to re-use a single event + * for both readFieldName and readRaw* values. + **/ + + private var isChildren: Boolean = false + def setIsChildren(): Unit = isChildren = true + + // DocumentEndEvent + /** Return peek and advance to the next character */ + protected def next(): Event = { + require(headOption.isDefined, "EOF") + + val ret: Event = headOption.get + + headOption = if (hasNext) { + val nextEvent: Event = events.next + + if (isEOF(nextEvent)) { + done = true + None + } else { + Some(nextEvent) + } + } else None + + if (isChildren) logger.error(s"IS CHILDREN: next(): $ret and next next: $headOption") + ret + } +} diff --git a/src/test/scala/fm/serializer/TestSerializer.scala b/src/test/scala/fm/serializer/TestSerializer.scala old mode 100644 new mode 100755 index 3b6d3ef..49d7813 --- a/src/test/scala/fm/serializer/TestSerializer.scala +++ b/src/test/scala/fm/serializer/TestSerializer.scala @@ -15,6 +15,7 @@ */ package fm.serializer +import fm.common.Logging import fm.common.{IP, ImmutableArray, ImmutableDate, UUID} import fm.serializer.validation.{Validation, ValidationError, ValidationResult} import java.io.File @@ -413,6 +414,7 @@ trait TestSerializer[BYTES] extends FunSuite with Matchers with AppendedClues { test("Foo") { val foo: Foo = Foo() val bytes: BYTES = serialize(foo) + logger.error(s"Foo: ${fm.serializer.yaml.YAML.toYAML(foo).slice(0,300)}") val foo2: Foo = deserialize[Foo](bytes) // TestMinimalJSON doesn't play well with this @@ -748,4 +750,4 @@ trait TestSerializer[BYTES] extends FunSuite with Matchers with AppendedClues { // Since this is an AnyVal it must be defined separately case class WrappedInt(value: Int) extends AnyVal case class IntHolder(count: Int) -case class WrappedHolder(count: WrappedInt) \ No newline at end of file +case class WrappedHolder(count: WrappedInt) diff --git a/src/test/scala/fm/serializer/yaml/TestYAML.scala b/src/test/scala/fm/serializer/yaml/TestYAML.scala new file mode 100755 index 0000000..1418e6a --- /dev/null +++ b/src/test/scala/fm/serializer/yaml/TestYAML.scala @@ -0,0 +1,118 @@ +/* + * Copyright 2014 Frugal Mechanic (http://frugalmechanic.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fm.serializer.yaml + +import fm.serializer.{Deserializer, Field, Serializer} + +final class TestDefaultYAML extends TestYAML { + def serialize[T](v: T)(implicit ser: Serializer[T]): String = YAML.toYAML[T](v) + + test("String Map - Treated as object") { + val map: Map[String,Int] = Map("foo" -> 123, "bar" -> 321) + val yaml: String = serialize(map) + + yaml should equal ( + """|foo: 123, + |bar: 321""".stripMargin) + + deserialize[Map[String,Int]](yaml) should equal (map) + } + + test("Int Map - Treated as array of tuples") { + val map: Map[Int,String] = Map(123 -> "foo", 312 -> "bar") + val yaml: String = serialize(map) + + yaml should equal ( + """|[ + | - _1: 123 + | _2: foo + | - _1: 312 + | _2: bar + |]""".stripMargin) + + deserialize[Map[Int,String]](yaml) should equal (map) + } + + test("Boolean types") { + case class BooleanKey(key: Boolean) + deserialize[BooleanKey]("""key: yes""") should equal(BooleanKey(true)) + deserialize[BooleanKey]("""key: Yes""") should equal(BooleanKey(true)) + deserialize[BooleanKey]("""key: YES""") should equal(BooleanKey(true)) + + deserialize[BooleanKey]("""key: true""") should equal(BooleanKey(true)) + deserialize[BooleanKey]("""key: True""") should equal(BooleanKey(true)) + deserialize[BooleanKey]("""key: TRUE""") should equal(BooleanKey(true)) + + deserialize[BooleanKey]("""key: no""") should equal(BooleanKey(false)) + deserialize[BooleanKey]("""key: No""") should equal(BooleanKey(false)) + deserialize[BooleanKey]("""key: NO""") should equal(BooleanKey(false)) + + deserialize[BooleanKey]("""key: false""") should equal(BooleanKey(false)) + deserialize[BooleanKey]("""key: False""") should equal(BooleanKey(false)) + deserialize[BooleanKey]("""key: FALSE""") should equal(BooleanKey(false)) + } + + case class Unquoted(name: String, int: Int, long: Long) + + test("Unquoted Field Names") { + deserialize[Unquoted]( + """|name: null + |int: 123 + |long: 123123123123123""".stripMargin) should equal(Unquoted(null, 123, 123123123123123L)) + + deserialize[Unquoted]( + """|name: nullnot + |int: 123 + |long: 123123123123123""".stripMargin) should equal(Unquoted("nullnot", 123, 123123123123123L)) + + deserialize[Unquoted]( + """|name: foo + |int: 123 + |long: 123123123123123""".stripMargin) should equal(Unquoted("foo", 123, 123123123123123L)) + + deserialize[Unquoted]( + """|name: foo + |int: "123" + |long: "123123123123123"""".stripMargin) should equal(Unquoted("foo", 123, 123123123123123L)) + } + + case class AlternateName(@Field("type") tpe: String, @Field foo: Int) + + test("Alternate Field Name") { + val instance: AlternateName = AlternateName("the_type_field", 123) + val yaml: String = + """|"type": "the_type_field" + |"foo": 123""".stripMargin + + serialize(instance) should equal (yaml) + deserialize[AlternateName](yaml) should equal (instance) + } +} + + +final class TestMinimalYAML extends TestYAML { + def serialize[T](v: T)(implicit ser: Serializer[T]): String = YAML.toMinimalYAML[T](v) + override def ignoreNullRetainTest: Boolean = true +} + +final class TestPrettyYAML extends TestYAML { + def serialize[T](v: T)(implicit ser: Serializer[T]): String = YAML.toPrettyYAML[T](v) +} + +abstract class TestYAML extends fm.serializer.TestSerializer[String] { + def serialize[T](v: T)(implicit ser: Serializer[T]): String + final def deserialize[T](yaml: String)(implicit deser: Deserializer[T]): T = YAML.fromYAML[T](yaml) +} \ No newline at end of file diff --git a/src/test/scala/fm/serializer/yaml/TestYAMLFeatures.scala b/src/test/scala/fm/serializer/yaml/TestYAMLFeatures.scala new file mode 100755 index 0000000..08c78a0 --- /dev/null +++ b/src/test/scala/fm/serializer/yaml/TestYAMLFeatures.scala @@ -0,0 +1,188 @@ +/* + * Copyright 2014 Frugal Mechanic (http://frugalmechanic.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fm.serializer.yaml + +import fm.serializer.Serializer + +final class TestYAMLFeatures extends TestYAML { + def serialize[T](v: T)(implicit ser: Serializer[T]): String = YAML.toPrettyYAML[T](v) + + // Example YAML File from: https://learnxinyminutes.com/docs/yaml/ + val testYAML: String = + """ + |# Comments in YAML look like this. + | + |################ + |# SCALAR TYPES # + |################ + | + |# Our root object (which continues for the entire document) will be a map, + |# which is equivalent to a dictionary, hash or object in other languages. + |key: value + |another_key: Another value goes here. + |a_number_value: 100 + |scientific_notation: 1e+12 + |# The number 1 will be interpreted as a number, not a boolean. if you want + |# it to be interpreted as a boolean, use true + |boolean: true + |null_value: null + |key with spaces: value + |# Notice that strings don't need to be quoted. However, they can be. + |however: 'A string, enclosed in quotes.' + |'Keys can be quoted too.': "Useful if you want to put a ':' in your key." + |single quotes: 'have ''one'' escape pattern' + |double quotes: "have many: \", \0, \t, \u263A, \x0d\x0a == \r\n, and more." + | + |# Multiple-line strings can be written either as a 'literal block' (using |), + |# or a 'folded block' (using '>'). + |literal_block: | + | This entire block of text will be the value of the 'literal_block' key, + | with line breaks being preserved. + | + | The literal continues until de-dented, and the leading indentation is + | stripped. + | + | Any lines that are 'more-indented' keep the rest of their indentation - + | these lines will be indented by 4 spaces. + |folded_style: > + | This entire block of text will be the value of 'folded_style', but this + | time, all newlines will be replaced with a single space. + | + | Blank lines, like above, are converted to a newline character. + | + | 'More-indented' lines keep their newlines, too - + | this text will appear over two lines. + | + |#################### + |# COLLECTION TYPES # + |#################### + | + |# Nesting uses indentation. 2 space indent is preferred (but not required). + |a_nested_map: + | key: value + | another_key: Another Value + | another_nested_map: + | hello: hello + | + |# Maps don't have to have string keys. + |0.25: a float key + | + |# Keys can also be complex, like multi-line objects + |# We use ? followed by a space to indicate the start of a complex key. + |? | + | This is a key + | that has multiple lines + |: and this is its value + | + |# YAML also allows mapping between sequences with the complex key syntax + |# Some language parsers might complain + |# An example + |? - Manchester United + | - Real Madrid + |: [2001-01-01, 2002-02-02] + | + |# Sequences (equivalent to lists or arrays) look like this + |# (note that the '-' counts as indentation): + |a_sequence: + | - Item 1 + | - Item 2 + | - 0.5 # sequences can contain disparate types. + | - Item 4 + | - key: value + | another_key: another_value + | - + | - This is a sequence + | - inside another sequence + | - - - Nested sequence indicators + | - can be collapsed + | + |# Since YAML is a superset of JSON, you can also write JSON-style maps and + |# sequences: + |json_map: {"key": "value"} + |json_seq: [3, 2, 1, "takeoff"] + |and quotes are optional: {key: [3, 2, 1, takeoff]} + | + |####################### + |# EXTRA YAML FEATURES # + |####################### + | + |# YAML also has a handy feature called 'anchors', which let you easily duplicate + |# content across your document. Both of these keys will have the same value: + |anchored_content: &anchor_name This string will appear as the value of two keys. + |other_anchor: *anchor_name + | + |# Anchors can be used to duplicate/inherit properties + |base: &base + | name: Everyone has same name + | + |# The regexp << is called Merge Key Language-Independent Type. It is used to + |# indicate that all the keys of one or more specified maps should be inserted + |# into the current map. + | + |foo: &foo + | <<: *base + | age: 10 + | + |bar: &bar + | <<: *base + | age: 20 + | + |# foo and bar would also have name: Everyone has same name + | + |# YAML also has tags, which you can use to explicitly declare types. + |explicit_string: !!str 0.5 + |# Some parsers implement language specific tags, like this one for Python's + |# complex number type. + |python_complex_number: !!python/complex 1+2j + | + |# We can also use yaml complex keys with language specific tags + |? !!python/tuple [5, 7] + |: Fifty Seven + |# Would be {(5, 7): 'Fifty Seven'} in Python + | + |#################### + |# EXTRA YAML TYPES # + |#################### + | + |# Strings and numbers aren't the only scalars that YAML can understand. + |# ISO-formatted date and datetime literals are also parsed. + |datetime: 2001-12-15T02:59:43.1Z + |datetime_with_spaces: 2001-12-14 21:59:43.10 -5 + |date: 2002-12-14 + | + |# The !!binary tag indicates that a string is actually a base64-encoded + |# representation of a binary blob. + |gif_file: !!binary | + | R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5 + | OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+ + | +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC + | AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs= + | + |# YAML also has a set type, which looks like this: + |set: + | ? item1 + | ? item2 + | ? item3 + |or: {item1, item2, item3} + | + |# Sets are just maps with null values; the above is equivalent to: + |set2: + | item1: null + | item2: null + | item3: null + | + """.stripMargin +} \ No newline at end of file