Add integration tests for K8sDnsNameResolver

build.sbt

@@ -1,4 +1,5 @@
 import Dependencies.*
+import com.typesafe.sbt.packager.docker.{Cmd, ExecCmd}
 import sbt.*
 
 ThisBuild / organization := "com.evolution.jgrpc.tools"
@@ -26,27 +27,53 @@ ThisBuild / scmInfo := Some(ScmInfo(
 // not sure if bincompat check works for Java code, put it here just in case
 ThisBuild / versionPolicyIntention := Compatibility.BinaryCompatible
 
-// this is a Java project, setting a fixed Scala version just in case
-ThisBuild / scalaVersion := "2.13.16"
-
 // setting pure-Java module build settings
 ThisBuild / crossPaths := false // drop off Scala suffix from artifact names.
 ThisBuild / autoScalaLibrary := false // exclude scala-library from dependencies
 ThisBuild / javacOptions := Seq("-source", "17", "-target", "17", "-Werror", "-Xlint:all")
 ThisBuild / doc / javacOptions := Seq("-source", "17", "-Xdoclint:all", "-Werror")
-
-// common test dependencies:
-ThisBuild / libraryDependencies ++= Seq(
-  // to be able to run JUnit 5+ tests:
-  "com.github.sbt.junit" % "jupiter-interface" % JupiterKeys.jupiterVersion.value,
-  Slf4j.simple,
-).map(_ % Test)
+ThisBuild / scalaVersion := "2.13.18"
+ThisBuild / scalacOptions ++= Seq(
+  "-release:17",
+  "-deprecation",
+  "-Xsource:3",
+)
 
 // common compile dependencies:
 ThisBuild / libraryDependencies ++= Seq(
   jspecify, // JSpecify null-check annotations
 )
 
+def asJavaPublishedModule(p: Project): Project = {
+  p.settings(
+    // common test dependencies for Java modules:
+    libraryDependencies ++= Seq(
+      // to be able to run JUnit 5+ tests:
+      "com.github.sbt.junit" % "jupiter-interface" % JupiterKeys.jupiterVersion.value,
+      Slf4j.simple,
+    ).map(_ % Test),
+  )
+}
+
+def asScalaIntegrationTestModule(p: Project): Project = {
+  p.disablePlugins(JupiterPlugin) // using scalatest instead
+    .settings(
+      publish / skip := true,
+      autoScalaLibrary := true, // int tests are written in Scala, returning scala-library dependency
+      Test / parallelExecution := false, // disable parallel execution between test suites
+      Test / fork := true, // disable parallel execution between modules
+      // tests take a long time to run, better to see the process in real time
+      Test / logBuffered := false,
+      // disable scaladoc generation to avoid dealing with annoying warnings
+      Compile / doc / sources := Seq.empty,
+      // common test dependencies for Scala int test modules:
+      libraryDependencies ++= Seq(
+        scalatest,
+        Slf4j.simple,
+      ).map(_ % Test),
+    )
+}
+
 lazy val root = project.in(file("."))
   .settings(
     name := "grpc-java-tools-root",
@@ -55,9 +82,11 @@ lazy val root = project.in(file("."))
   )
   .aggregate(
     k8sDnsNameResolver,
+    k8sDnsNameResolverIt,
   )
 
 lazy val k8sDnsNameResolver = project.in(file("k8s-dns-name-resolver"))
+  .configure(asJavaPublishedModule)
   .settings(
     name := "k8s-dns-name-resolver",
     description := "Evolution grpc-java tools - DNS-based name resolver for Kubernetes services",
@@ -68,6 +97,43 @@ lazy val k8sDnsNameResolver = project.in(file("k8s-dns-name-resolver"))
     ),
   )
 
+lazy val k8sDnsNameResolverIt = project.in(file("k8s-dns-name-resolver-it"))
+  .configure(asScalaIntegrationTestModule)
+  // the module builds its own test app docker container
+  .enablePlugins(JavaAppPackaging, DockerPlugin)
+  .settings(
+    name := "k8s-dns-name-resolver-it",
+    description := "Evolution grpc-java tools - DNS-based name resolver for Kubernetes services - integration tests",
+    Compile / PB.targets := Seq(
+      scalapb.gen() -> (Compile / sourceManaged).value / "scalapb",
+    ),
+    dockerBaseImage := "amazoncorretto:17-alpine",
+    dockerCommands ++= Seq(
+      // root rights are needed to install additional packages, and also test client needs it
+      // to manipulate its DNS settings
+      Cmd("USER", "root"),
+      // bash is needed for testcontainers log watching logic
+      // lsof and coredns are needed for the integration test logic
+      ExecCmd("RUN", "apk", "add", "--no-cache", "bash", "lsof", "coredns"),
+    ),
+    dockerExposedPorts := Seq(9000), // Should match the test app GRPC server port.
+    // The int test here needs the test app docker container staged before running the code.
+    // It's then used in docker compose inside testcontainers.
+    test := {
+      (Docker / stage).value
+      (Test / test).value
+    },
+    libraryDependencies ++= Seq(
+      Slf4j.simple,
+      commonsLang3,
+      "io.grpc" % "grpc-netty" % scalapb.compiler.Version.grpcJavaVersion,
+      "com.thesamet.scalapb" %% "scalapb-runtime-grpc" % scalapb.compiler.Version.scalapbVersion,
+      Testcontainers.core % Test,
+    ),
+  ).dependsOn(
+    k8sDnsNameResolver,
+  )
+
 addCommandAlias("fmt", "all scalafmtAll scalafmtSbt javafmtAll")
 addCommandAlias(
   "build",

k8s-dns-name-resolver-it/src/main/protobuf/test_svc.proto

@@ -0,0 +1,13 @@
+syntax = "proto2";
+
+package k8sdns.it;
+
+service TestSvc {
+  rpc GetId (GetIdRequest) returns (GetIdReply) {}
+}
+
+message GetIdRequest {}
+
+message GetIdReply {
+  required int32 id = 1;
+}

k8s-dns-name-resolver-it/src/main/resources/simplelogger.properties

@@ -0,0 +1,7 @@
+# Configure slf4j-simple to have concise output for tests
+# Supported settings: https://www.slf4j.org/api/org/slf4j/simple/SimpleLogger.html
+org.slf4j.simpleLogger.logFile=System.out
+org.slf4j.simpleLogger.showDateTime=true
+org.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS
+org.slf4j.simpleLogger.showThreadName=false
+org.slf4j.simpleLogger.showShortLogName=true

k8s-dns-name-resolver-it/src/main/scala/com/evolution/jgrpc/tools/k8sdns/it/TestApp.scala

@@ -0,0 +1,36 @@
+package com.evolution.jgrpc.tools.k8sdns.it
+
+/**
+ * `K8sDnsNameResolver` integration test service app entrypoint.
+ *
+ * Depending on the run mode environment variable value could either work as
+ * [[TestServer]] or [[TestClient]].
+ */
+object TestApp extends App {
+  private val runModeEnvVarName = "TEST_SVC_RUN_MODE"
+  private val instanceIdVarName = "TEST_SVC_INSTANCE_ID"
+
+  sys.env.get(runModeEnvVarName) match {
+    case None =>
+      sys.error(s"missing environment variable: $runModeEnvVarName")
+    case Some("server") =>
+      runServer()
+    case Some("client") =>
+      runClient()
+    case Some(unexpectedRunMode) =>
+      sys.error(s"unexpected run mode: $unexpectedRunMode")
+  }
+
+  private def runServer(): Unit = {
+    val instanceId = sys.env.getOrElse(
+      instanceIdVarName,
+      sys.error(s"missing environment variable: $instanceIdVarName"),
+    ).toInt
+
+    new TestServer(instanceId = instanceId).run()
+  }
+
+  private def runClient(): Unit = {
+    new TestClient().run()
+  }
+}

k8s-dns-name-resolver-it/src/main/scala/com/evolution/jgrpc/tools/k8sdns/it/TestAppShared.scala

@@ -0,0 +1,169 @@
+package com.evolution.jgrpc.tools.k8sdns.it
+
+import java.nio.file.*
+
+/**
+ * Common things shared between the `K8sDnsNameResolver` integration test code and the
+ * test service code
+ *
+ * @see
+ *   [[TestApp]]
+ */
+object TestAppShared {
+
+  /**
+   * [[TestApp]] GRPC server port
+   */
+  val ServerPort: Int = 9000
+
+  /**
+   * Docker compose service names for [[TestApp]] containers.
+   *
+   * The names here should match the ones used in the
+   * `src/test/resources/docker/compose-test.yml` file.
+   */
+  object TestAppSvcNames {
+
+    /**
+     * First [[TestApp]] service in a [[TestServer]] mode
+     */
+    val Server1: String = "test-server1"
+
+    /**
+     * Second [[TestApp]] service in a [[TestServer]] mode
+     */
+    val Server2: String = "test-server2"
+
+    /**
+     * [[TestApp]] service in a [[TestClient]] mode
+     */
+    val Client: String = "test-client"
+  }
+
+  /**
+   * `K8sDnsNameResolver` integration test watches for these [[TestApp]] log messages in
+   * the stdout.
+   */
+  object TestAppSpecialLogMsgs {
+
+    /**
+     * [[TestApp]] docker container has been started and ready to proceed with the test
+     */
+    val Ready: String = "TEST CONTAINER READY"
+
+    /**
+     * [[TestApp]] in the [[TestClient]] mode died prematurely, all the tests should be
+     * aborted
+     */
+    val ClientPrematureDeath: String = "TEST CLIENT PANIC"
+
+    /**
+     * [[TestApp]] in the [[TestClient]] mode completed a requested test case successfully
+     *
+     * @see
+     *   [[TestClientControl]] for how to request a test case execution
+     */
+    val ClientTestCaseSuccess: String = "TEST SUCCESS"
+
+    /**
+     * [[TestApp]] in the [[TestClient]] mode ran a requested test case and got a failure
+     *
+     * @see
+     *   [[TestClientControl]] for how to request a test case execution
+     */
+    val ClientTestCaseFailed: String = "TEST FAILED"
+  }
+
+  /**
+   * Defines the way to send commands to the [[TestApp]] container in the [[TestClient]]
+   * mode:
+   *   - create an empty file in the [[CmdDirPath]] directory on the container - the name
+   *     of the file is the command name
+   *   - the [[TestClient]] code deletes the file and queues the command for execution
+   *   - commands are executed on the [[TestClient]] one-by-one
+   *   - monitor [[TestClient]] container stdout for the command progress - see
+   *     [[TestAppSpecialLogMsgs]]
+   *
+   * Currently supported commands:
+   *   - [[RunTestCaseCmdFileName]] for running [[TestClientTestCase]]
+   */
+  object TestClientControl {
+
+    /**
+     * Directory which [[TestApp]] in the [[TestClient]] mode uses for receiving commands
+     *
+     * @see
+     *   [[TestClientControl]]
+     */
+    val CmdDirPath: Path = Paths.get("/tmp/test-client-control")
+
+    /**
+     * [[TestClientControl]] command for running [[TestClientTestCase]].
+     */
+    object RunTestCaseCmdFileName {
+      private val fileNamePrefix = ".run-test-case-"
+
+      /**
+       * Creates a [[TestClientControl]] command file name for running the given
+       * [[TestClientTestCase]]
+       */
+      def apply(testCase: TestClientTestCase): String = {
+        s"$fileNamePrefix${ testCase.name }"
+      }
+
+      /**
+       * Matches [[TestClientControl]] command file name which runs a
+       * [[TestClientTestCase]]
+       */
+      def unapply(fileName: String): Option[TestClientTestCase] = {
+        if (fileName.startsWith(fileNamePrefix)) {
+          val testCaseName = fileName.drop(fileNamePrefix.length)
+          TestClientTestCase.values.find(_.name == testCaseName)
+        } else {
+          None
+        }
+      }
+    }
+  }
+
+  /**
+   * Test case to run on [[TestClient]].
+   *
+   * @see
+   *   [[TestClientControl.RunTestCaseCmdFileName]]
+   */
+  sealed abstract class TestClientTestCase extends Product {
+    final def name: String = productPrefix
+  }
+  object TestClientTestCase {
+    val values: Vector[TestClientTestCase] = Vector(
+      DiscoverNewPod,
+      DnsFailureRecover,
+    )
+
+    /**
+     * [[TestClient]] test case verifying that `K8sDnsNameResolver` live pod discovery
+     * works.
+     *
+     * Test steps overview:
+     *   - point the service host DNS records to one server container
+     *   - create a GRPC client, check that it sees only the first server
+     *   - add the second server to the DNS records
+     *   - check that after the configured reload TTL, the client sees both servers
+     */
+    case object DiscoverNewPod extends TestClientTestCase
+
+    /**
+     * [[TestClient]] test case verifying that `K8sDnsNameResolver` recovers after a DNS
+     * call failure.
+     *
+     * Test steps overview:
+     *   - point the service host DNS records to one server container
+     *   - create a GRPC client, check that it sees only the first server
+     *   - stop the DNS server, wait until the client gets a DNS error
+     *   - start the DNS server back again, with 2 servers in the records
+     *   - check that after the configured reload TTL, the client sees both servers
+     */
+    case object DnsFailureRecover extends TestClientTestCase
+  }
+}