diff --git a/README.md b/README.md index e35cdc8..0487752 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ The generator can be used with any tool that can perform system calls to a comma See example under [example/generate.scala](./example/generate.scala). ```scala -//> using scala 3.7.3 -//> using dep dev.rolang::gcp-codegen::0.0.8 +//> using scala 3.7.4 +//> using dep dev.rolang::gcp-codegen::0.0.12 import gcp.codegen.*, java.nio.file.*, GeneratorConfig.* @@ -55,13 +55,41 @@ See output in `example/out`. | Configuration | Description | Options | Default | | ------------------- | ---------------- | ------- | --- | -| -specs | Can be `stdin` or a path to the JSON file. | | | -| -out-dir | Ouput directory | | | -| -out-pkg | Output package | | | -| -http-source | Generated http source. | [Sttp4](https://sttp.softwaremill.com/en/stable) | | -| -json-codec | Generated JSON codec | [Jsoniter](https://github.com/plokhotnyuk/jsoniter-scala), [ZioJson](https://zio.dev/zio-json) | | -| -array-type | Collection type for JSON arrays | `List`, `Vector`, `Array`, `ZioChunk` | `List` | -| -include-resources | Optional resource filter. | | | +| -specs | Can be `stdin` or a path to the JSON file. | | | +| -out-dir | Output directory | | | +| -out-pkg | Output package | | | +| -http-source | Generated http source. | [Sttp4](https://sttp.softwaremill.com/en/stable) | | +| -json-codec | Generated JSON codec | [Jsoniter](https://github.com/plokhotnyuk/jsoniter-scala), [ZioJson](https://zio.dev/zio-json) | | +| -jsoniter-json-type | In case of Jsoniter a fully qualified name of the custom type that can represent a raw Json value | | +| -array-type | Collection type for JSON arrays | `List`, `Vector`, `Array`, `ZioChunk` | `List` | +| -include-resources | Optional resource filter. | | | + +##### Jsoniter Json type and codec example +Jsoniter doesn't ship with a type that can represent raw Json values to be used for mapping of `any` / `object` types, +but it provides methods to read / write raw values as bytes (related [issue](https://github.com/plokhotnyuk/jsoniter-scala/issues/1257)). +Given that we can create a custom type with a codec which can look for example like [that](modules/example-jsoniter-json/shared/src/main/scala/json.scala): +```scala +package example.jsoniter +import com.github.plokhotnyuk.jsoniter_scala.core.* + +opaque type Json = Array[Byte] +object Json: + def writeToJson[T: JsonValueCodec](v: T): Json = writeToArray[T](v) + + given codec: JsonValueCodec[Json] = new JsonValueCodec[Json]: + override def decodeValue(in: JsonReader, default: Json): Json = in.readRawValAsBytes() + override def encodeValue(x: Json, out: JsonWriter): Unit = out.writeRawVal(x) + override val nullValue: Json = Array[Byte](0) + + extension (v: Json) + def readAsUnsafe[T: JsonValueCodec]: T = readFromArray(v) + def readAs[T: JsonValueCodec]: Either[Throwable, T] = + try Right(readFromArray(v)) + catch case t: Throwable => Left(t) +``` +Then pass it as argument to the code generator like `-jsoniter-json-type=_root_.example.jsoniter.Json`. +Since this type and codec can be shared across generated clients it has to be provided (at least for now) +instead of being generated for each client to avoid duplicated / redundant code. ##### Examples: @@ -79,7 +107,7 @@ curl 'https://pubsub.googleapis.com/$discovery/rest?version=v1' > pubsub_v1.json -specs=./pubsub_v1.json \ -out-pkg=gcp.pubsub.v1 \ -http-source=sttp4 \ - -json-codec=jsoniter \ + -json-codec=ziojson \ -include-resources='projects.*,!projects.snapshots' # optional filters ``` diff --git a/build.sbt b/build.sbt index e14bfa5..04541c6 100644 --- a/build.sbt +++ b/build.sbt @@ -46,7 +46,7 @@ val zioVersion = "2.1.23" val zioJsonVersion = "0.8.0" -val jsoniterVersion = "2.38.6" +val jsoniterVersion = "2.38.8" val munitVersion = "1.2.1" @@ -58,6 +58,8 @@ lazy val root = (project in file(".")) .aggregate( core.native, core.jvm, + exampleJsoniterJson.native, + exampleJsoniterJson.jvm, cli ) .aggregate(testProjects.componentProjects.map(p => LocalProject(p.id)) *) @@ -65,6 +67,7 @@ lazy val root = (project in file(".")) // for supporting code inspection / testing of generated code via test_gen.sh script lazy val testLocal = (project in file("test-local")) + .dependsOn(exampleJsoniterJson.jvm) .settings( libraryDependencies ++= Seq( "com.softwaremill.sttp.client4" %% "core" % sttpClient4Version, @@ -101,6 +104,20 @@ lazy val cli = project nativeConfig := nativeConfig.value.withMultithreading(false) ) +lazy val exampleJsoniterJson = crossProject(JVMPlatform, NativePlatform) + .in(file("modules/example-jsoniter-json")) + .settings(noPublish) + .settings( + name := "example-jsoniter-json", + moduleName := "example-jsoniter-json" + ) + .settings( + libraryDependencies ++= Seq( + "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % jsoniterVersion, + "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % jsoniterVersion % "compile-internal" + ) + ) + def dependencyByConfig(httpSource: String, jsonCodec: String, arrayType: String): Seq[ModuleID] = { (httpSource match { case "Sttp4" => Seq("com.softwaremill.sttp.client4" %% "core" % sttpClient4Version) @@ -136,7 +153,7 @@ lazy val testProjects: CompositeProject = new CompositeProject { arrayType <- Seq("ZioChunk", "List") id = s"test-$apiName-$apiVersion-${httpSource}-${jsonCodec}-${arrayType}".toLowerCase() } yield { - Project + val p = Project .apply( id = id, base = file("modules") / id @@ -155,6 +172,12 @@ lazy val testProjects: CompositeProject = new CompositeProject { "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % jsoniterVersion % Test ) ++ dependencyByConfig(httpSource = httpSource, jsonCodec = jsonCodec, arrayType = arrayType) ) + + if (jsonCodec == "Jsoniter") { + p.dependsOn(exampleJsoniterJson.componentProjects.map(p => ClasspathDependency(p, p.configuration)) *) + } else { + p + } } } @@ -235,7 +258,8 @@ def codegenTask( s"-out-pkg=$basePkgName", s"-http-source=$httpSource", s"-json-codec=$jsonCodec", - s"-array-type=$arrayType" + s"-array-type=$arrayType", + s"-jsoniter-json-type=_root_.example.jsoniter.Json" ).mkString(" ") ! ProcessLogger(l => logger.info(l), e => errs += e)) match { case 0 => () case c => throw new InterruptedException(s"Failure on code generation: ${errs.mkString("\n")}") diff --git a/example/generate.scala b/example/generate.scala index a139e8a..d808485 100644 --- a/example/generate.scala +++ b/example/generate.scala @@ -1,5 +1,5 @@ -//> using scala 3.7.3 -//> using dep dev.rolang::gcp-codegen::0.0.8 +//> using scala 3.7.4 +//> using dep dev.rolang::gcp-codegen::0.0.12 import gcp.codegen.*, java.nio.file.*, GeneratorConfig.* @@ -10,7 +10,7 @@ import gcp.codegen.*, java.nio.file.*, GeneratorConfig.* outDir = Path.of("out"), outPkg = "example.pubsub.v1", httpSource = HttpSource.Sttp4, - jsonCodec = JsonCodec.Jsoniter, + jsonCodec = JsonCodec.ZioJson, arrayType = ArrayType.List, preprocess = specs => specs ) diff --git a/modules/cli/src/main/scala/cli.scala b/modules/cli/src/main/scala/cli.scala index 3544105..5723b76 100644 --- a/modules/cli/src/main/scala/cli.scala +++ b/modules/cli/src/main/scala/cli.scala @@ -31,10 +31,10 @@ import scala.concurrent.duration.* private def argsToTask(args: Seq[String]): Either[String, Task] = val argsMap = args.toList - .flatMap(_.split('=').map(_.trim().toLowerCase())) + .flatMap(_.split('=').map(_.trim())) .sliding(2, 2) .collect { case a :: b :: _ => - a -> b + a.toLowerCase() -> b } .toMap @@ -57,10 +57,17 @@ private def argsToTask(args: Seq[String]): Either[String, Task] = .get("-http-source") .flatMap(v => HttpSource.values.find(_.toString().equalsIgnoreCase(v))) .toRight("Missing or invalid -http-source") - jsonCodec <- argsMap - .get("-json-codec") - .flatMap(v => JsonCodec.values.find(_.toString().equalsIgnoreCase(v))) - .toRight("Missing or invalid -json-codec") + jsonCodec <- ( + argsMap + .get("-json-codec") + .map(_.toLowerCase()), + argsMap.get("-jsoniter-json-type") + ) match { + case (Some("ziojson"), _) => Right(JsonCodec.ZioJson) + case (Some("jsoniter"), Some(jsonType)) => Right(JsonCodec.Jsoniter(jsonType)) + case (Some("jsoniter"), None) => Left("Missing -jsoniter-json-type") + case _ => Left("Missing or invalid -json-codec") + } arrayType <- argsMap.get("-array-type") match case None => Right(ArrayType.List) case Some(v) => ArrayType.values.find(_.toString().equalsIgnoreCase(v)).toRight(s"Invalid array-type $v") diff --git a/modules/core/shared/src/main/scala/codegen.scala b/modules/core/shared/src/main/scala/codegen.scala index 960e8f9..bf76f1c 100644 --- a/modules/core/shared/src/main/scala/codegen.scala +++ b/modules/core/shared/src/main/scala/codegen.scala @@ -30,7 +30,8 @@ object GeneratorConfig: case Sttp4 enum JsonCodec: - case ZioJson, Jsoniter + case ZioJson + case Jsoniter(jsonTypeRef: String) enum ArrayType { case List, Vector, Array, ZioChunk @@ -159,15 +160,15 @@ def generateBySpec( "import sttp.model.*\nimport sttp.client4.*, sttp.client4.ResponseException.{DeserializationException, UnexpectedStatusCode}" }, config.jsonCodec match { - case JsonCodec.ZioJson => "import zio.json.*" - case JsonCodec.Jsoniter => "import com.github.plokhotnyuk.jsoniter_scala.core.*" + case JsonCodec.ZioJson => "import zio.json.*" + case _: JsonCodec.Jsoniter => "import com.github.plokhotnyuk.jsoniter_scala.core.*" }, s"val resourceRequest: PartialRequest[Either[String, String]] = basicRequest.headers(Header.contentType(MediaType.ApplicationJson))", "", s"export ${config.outPkg}.QueryParameters", "", (config.httpSource, config.jsonCodec) match - case (HttpSource.Sttp4, JsonCodec.Jsoniter) => + case (HttpSource.Sttp4, _: JsonCodec.Jsoniter) => """|def asJson[T : JsonValueCodec]: ResponseAs[Either[ResponseException[String], T]] = | asByteArrayAlways.mapWithMetadata((bytes, metadata) => | if metadata.isSuccess then @@ -221,23 +222,13 @@ def generateBySpec( }, // generate schemas with properties for { - (commonCodecs, hasExtraCodecs) <- Future { - commonSchemaCodecs( - schemas = specs.schemas.filter(_._2.properties.nonEmpty), - pkg = schemasPkg, - objName = commonCodecsObj, - jsonCodec = config.jsonCodec, - hasProps = p => specs.hasProps(p), - arrType = config.arrayType - ) match - case None => (Nil, false) - case Some((content, hasExtraCodecs)) => - Files.writeString(commonCodecsPath, content) - (List(commonCodecsPath.toFile()), hasExtraCodecs) - } schemas <- Future .traverse(specs.schemas) { (schemaPath, schema) => Future { + val jsonType = config.jsonCodec match + case JsonCodec.ZioJson => "zio.json.ast.Json" + case JsonCodec.Jsoniter(jsonTypeRef) => jsonTypeRef + val code = (if schema.properties.nonEmpty then schemasCode( @@ -245,16 +236,14 @@ def generateBySpec( pkg = schemasPkg, jsonCodec = config.jsonCodec, hasProps = p => specs.hasProps(p), - arrType = config.arrayType, - commonCodecsPkg = - if commonCodecs.nonEmpty && hasExtraCodecs then Some(commonCodecsPkg) else None + arrType = config.arrayType ) else // create a type alias for objects without properties val comment = toComment(schema.description) s"""|package $schemasPkg | - |${comment}type ${schema.id.scalaName} = Option[""]""".stripMargin + |${comment}type ${schema.id.scalaName} = Option[$jsonType]""".stripMargin ) val path = schemasPath / s"${schemaPath.scalaName}.scala" @@ -262,7 +251,7 @@ def generateBySpec( path.toFile() } } - } yield commonCodecs ::: schemas.toList + } yield schemas.toList ) ) .map(_.flatten) @@ -415,8 +404,7 @@ def schemasCode( pkg: String, jsonCodec: JsonCodec, hasProps: SchemaPath => Boolean, - arrType: ArrayType, - commonCodecsPkg: Option[String] + arrType: ArrayType ): String = { def enums = schema.properties.collect: @@ -426,26 +414,26 @@ def schemasCode( def `def toJsonString`(objName: String) = jsonCodec match case JsonCodec.ZioJson => s"def toJsonString: String = $objName.jsonCodec.encodeJson(this, None).toString()" - case JsonCodec.Jsoniter => + case _: JsonCodec.Jsoniter => s"def toJsonString: String = writeToString(this)" def jsonDecoder(objName: String) = List( s"object $objName {", - // Jsoniter doesn't support derivation from Scala 3 union types - if jsonCodec == JsonCodec.Jsoniter then - enums - .map((k, e) => - s" enum ${toScalaTypeName(k)} {\n${e.values.map(v => s"${toComment(Some(v.enumDescription), " ")} case ${toScalaName(v.value)}").mkString("\n ")}\n }\n" - ) - .mkString("\n") - else "", jsonCodec match case JsonCodec.ZioJson => s" given jsonCodec: JsonCodec[$objName] = JsonCodec.derived[$objName]" - case JsonCodec.Jsoniter => - s"""| given jsonCodec: JsonValueCodec[$objName] = - | JsonCodecMaker.make(CodecMakerConfig.withAllowRecursiveTypes(true).withDiscriminatorFieldName(None))""".stripMargin, + case _: JsonCodec.Jsoniter => + // Jsoniter doesn't support derivation from Scala 3 union types + enums + .map((k, e) => + s" enum ${toScalaTypeName(k)} {\n${e.values.map(v => s"${toComment(Some(v.enumDescription), " ")} case ${toScalaName(v.value)}").mkString("\n ")}\n }\n" + ) + .mkString("\n") + .appendedAll( + s"""| given jsonCodec: JsonValueCodec[$objName] = + | JsonCodecMaker.make(CodecMakerConfig.withAllowRecursiveTypes(true).withDiscriminatorFieldName(None))""".stripMargin + ), "}" ).mkString("\n") @@ -473,115 +461,15 @@ def schemasCode( s"package $pkg", "", jsonCodec match { - case JsonCodec.ZioJson => "import zio.json.*" - case JsonCodec.Jsoniter => + case JsonCodec.ZioJson => "import zio.json.*" + case _: JsonCodec.Jsoniter => """|import com.github.plokhotnyuk.jsoniter_scala.core.* |import com.github.plokhotnyuk.jsoniter_scala.macros.*""".stripMargin }, - commonCodecsPkg match - case Some(codecsPkg) => s"import $codecsPkg.given" - case _ => "", toSchemaClass(schema) ).mkString("\n") } -def commonSchemaCodecs( - schemas: Map[SchemaPath, Schema], - pkg: String, - objName: String, - jsonCodec: JsonCodec, - hasProps: SchemaPath => Boolean, - arrType: ArrayType -): Option[(String, Boolean)] = { - (jsonCodec match - case JsonCodec.ZioJson => Nil - case JsonCodec.Jsoniter => - List( - s"""|package $pkg - | - |import com.github.plokhotnyuk.jsoniter_scala.core.* - |import com.github.plokhotnyuk.jsoniter_scala.macros.* - | - |opaque type Json = Array[Byte] - | - |object Json { - | - | given codec: JsonValueCodec[Json] = new JsonValueCodec[Json] { - | override def decodeValue(in: JsonReader, default: Json): Json = in.readRawValAsBytes() - | - | override def encodeValue(x: Json, out: JsonWriter): Unit = out.writeRawVal(x) - | - | override val nullValue: Json = new Array[Byte](0) - | } - | - | extension (v: Json) - | def readAsUnsafe[T: JsonValueCodec]: T = readFromArray(v) - | def readAs[T: JsonValueCodec]: Either[Throwable, T] = - | try - | Right(readFromArray(v)) - | catch - | case t: Throwable => Left(t) - |}""".stripMargin -> false - ) - ).appendedAll((jsonCodec, arrType) match - case (JsonCodec.Jsoniter, ArrayType.ZioChunk) => - schemas.toList - .flatMap((sk, sv) => - sv.sortedProperties(hasProps) - .collect { case (k, Property(_, SchemaType.Array(typ, _), _)) => - val enumType = - if jsonCodec == JsonCodec.ZioJson then SchemaType.EnumType.Literal - else SchemaType.EnumType.Nominal(s"${sk.lastOption.getOrElse("")}.${toScalaTypeName(k)}") - typ.scalaType(arrType, jsonCodec, enumType) - } - ) - .distinct match - case Nil => Nil - case props => - List( - List( - "", - s"object $objName {", - "", - // to ensure codec for Chunk[Json] is added since it may not be present in props - """| given JsonChunkCodec: JsonValueCodec[zio.Chunk[Json]] = new JsonValueCodec[zio.Chunk[Json]] { - | val arrCodec: JsonValueCodec[Array[Json]] = JsonCodecMaker.make - | - | override val nullValue: zio.Chunk[Json] = zio.Chunk.empty - | - | override def decodeValue(in: JsonReader, default: zio.Chunk[Json]): zio.Chunk[Json] = - | zio.Chunk.fromArray(arrCodec.decodeValue(in, default.toArray)) - | - | override def encodeValue(x: zio.Chunk[Json], out: JsonWriter): Unit = - | arrCodec.encodeValue(x.toArray, out) - | }""".stripMargin, - "", - props - .filterNot(_ == "Json") // to void duplicate codec for Chunk[Json] - .map { t => - val prefix = " given " + toScalaName(t + "ChunkCodec") - s"""|${prefix}: JsonValueCodec[zio.Chunk[$t]] = new JsonValueCodec[zio.Chunk[$t]] { - | val arrCodec: JsonValueCodec[Array[$t]] = JsonCodecMaker.make - | - | override val nullValue: zio.Chunk[$t] = zio.Chunk.empty - | - | override def decodeValue(in: JsonReader, default: zio.Chunk[$t]): zio.Chunk[$t] = - | zio.Chunk.fromArray(arrCodec.decodeValue(in, default.toArray)) - | - | override def encodeValue(x: zio.Chunk[$t], out: JsonWriter): Unit = - | arrCodec.encodeValue(x.toArray, out) - |}""".stripMargin - } - .mkString("\n\n"), - "}" - ).mkString("\n") -> true - ) - case _ => Nil) match - case Nil => None - case codecs => Some((codecs.map(_._1).mkString("\n"), codecs.exists(_._2))) - -} - case class FlatPath(path: String, params: List[String]) case class MediaUploadProtocol(multipart: Boolean, path: String) derives Reader @@ -793,8 +681,8 @@ enum SchemaType(val optional: Boolean): case _: Object => toType( jsonCodec match - case JsonCodec.ZioJson => "zio.json.ast.Json.Obj" - case JsonCodec.Jsoniter => "Json" // assuming the codecs package is imported + case JsonCodec.ZioJson => "zio.json.ast.Json.Obj" + case JsonCodec.Jsoniter(jsonRef) => jsonRef ) case Enum(_, values, _) => enumType match @@ -803,8 +691,8 @@ enum SchemaType(val optional: Boolean): case _ => toType( jsonCodec match - case JsonCodec.ZioJson => "zio.json.ast.Json" - case JsonCodec.Jsoniter => "Json" // assuming the codecs package is imported + case JsonCodec.ZioJson => "zio.json.ast.Json" + case JsonCodec.Jsoniter(jsonRef) => jsonRef ) object SchemaType: diff --git a/modules/example-jsoniter-json/shared/src/main/scala/json.scala b/modules/example-jsoniter-json/shared/src/main/scala/json.scala new file mode 100644 index 0000000..28f0c70 --- /dev/null +++ b/modules/example-jsoniter-json/shared/src/main/scala/json.scala @@ -0,0 +1,18 @@ +package example.jsoniter + +import com.github.plokhotnyuk.jsoniter_scala.core.* + +opaque type Json = Array[Byte] +object Json: + def writeToJson[T: JsonValueCodec](v: T): Json = writeToArray[T](v) + + given codec: JsonValueCodec[Json] = new JsonValueCodec[Json]: + override def decodeValue(in: JsonReader, default: Json): Json = in.readRawValAsBytes() + override def encodeValue(x: Json, out: JsonWriter): Unit = out.writeRawVal(x) + override val nullValue: Json = Array[Byte](0) + + extension (v: Json) + def readAsUnsafe[T: JsonValueCodec]: T = readFromArray(v) + def readAs[T: JsonValueCodec]: Either[Throwable, T] = + try Right(readFromArray(v)) + catch case t: Throwable => Left(t) diff --git a/test_gen.sh b/test_gen.sh index f53a24b..89ec322 100755 --- a/test_gen.sh +++ b/test_gen.sh @@ -8,7 +8,7 @@ spec=aiplatform version=v1 json_codec=jsoniter http_source=sttp4 -array_type=list +array_type=ziochunk out_dir=test-local/src/main/scala/test-$spec-$version/$http_source/$spec/$json_codec rm -rf $out_dir && mkdir -p $out_dir @@ -19,4 +19,5 @@ scala modules/cli/src/main/scala/cli.scala -- \ -out-pkg=gcp.${spec}.${version}.${http_source}.${json_codec}.$array_type \ -http-source=$http_source \ -json-codec=$json_codec \ + -jsoniter-json-type=_root_.custom.jsoniter.Json \ -array-type=$array_type \ No newline at end of file