diff --git a/testing/src/main/java/dev/cel/testing/testrunner/CelCoverageIndex.java b/testing/src/main/java/dev/cel/testing/testrunner/CelCoverageIndex.java index b67d5a1e6..e7b08da79 100644 --- a/testing/src/main/java/dev/cel/testing/testrunner/CelCoverageIndex.java +++ b/testing/src/main/java/dev/cel/testing/testrunner/CelCoverageIndex.java @@ -18,6 +18,7 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; import com.google.errorprone.annotations.CanIgnoreReturnValue; import javax.annotation.concurrent.ThreadSafe; import dev.cel.common.CelAbstractSyntaxTree; @@ -28,6 +29,8 @@ import dev.cel.common.types.CelKind; import dev.cel.parser.CelUnparserVisitor; import dev.cel.runtime.CelEvaluationListener; +import java.io.File; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.Map; @@ -156,7 +159,13 @@ public final Builder addUnencounteredBranches(String value) { } } - /** Returns the coverage report for the CEL test suite. */ + /** + * Generates a coverage report for the CEL test suite. + * + *

Note: If the generated graph URL results in a `Request Entity Too Large` error, download the + * `coverage_graph.txt` file from the test artifacts and upload its contents to Graphviz to render the coverage graph. + */ public CoverageReport generateCoverageReport() { CoverageReport.Builder reportBuilder = CoverageReport.builder().setCelExpression(new CelUnparserVisitor(ast).unparse()); @@ -170,6 +179,7 @@ public CoverageReport generateCoverageReport() { dotGraphBuilder); dotGraphBuilder.append("}"); String dotGraph = dotGraphBuilder.toString(); + CoverageReport report = reportBuilder.setDotGraph(dotGraph).build(); logger.info("CEL Expression: " + report.celExpression()); logger.info("Nodes: " + report.nodes()); @@ -180,6 +190,8 @@ public CoverageReport generateCoverageReport() { logger.info("Unencountered Branches: \n" + String.join("\n", report.unencounteredBranches())); logger.info("Dot Graph: " + report.dotGraph()); + + writeDotGraphToArtifact(dotGraph); return report; } @@ -392,4 +404,21 @@ private String escapeSpecialCharacters(String exprText) { .replace("{", "\\{") .replace("}", "\\}"); } + + private void writeDotGraphToArtifact(String dotGraph) { + String testOutputsDir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR"); + if (testOutputsDir == null) { + // Only for non-bazel/blaze users, we write to a subdirectory under the cwd. + testOutputsDir = "cel_artifacts"; + } + File outputDir = new File(testOutputsDir, "cel_test_coverage"); + if (!outputDir.exists()) { + outputDir.mkdirs(); + } + try { + Files.asCharSink(new File(outputDir, "coverage_graph.txt"), UTF_8).write(dotGraph); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } } diff --git a/testing/src/test/java/dev/cel/testing/testrunner/CelCoverageIndexTest.java b/testing/src/test/java/dev/cel/testing/testrunner/CelCoverageIndexTest.java index 400147d35..287d5435e 100644 --- a/testing/src/test/java/dev/cel/testing/testrunner/CelCoverageIndexTest.java +++ b/testing/src/test/java/dev/cel/testing/testrunner/CelCoverageIndexTest.java @@ -14,8 +14,10 @@ package dev.cel.testing.testrunner; import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; import dev.cel.bundle.Cel; import dev.cel.bundle.CelFactory; import dev.cel.common.CelAbstractSyntaxTree; @@ -27,7 +29,7 @@ import dev.cel.runtime.CelEvaluationListener; import dev.cel.runtime.CelRuntime; import dev.cel.testing.testrunner.CelCoverageIndex.CoverageReport; -import org.junit.Before; +import java.io.File; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -35,23 +37,15 @@ @RunWith(JUnit4.class) public final class CelCoverageIndexTest { - private Cel cel; - private CelAbstractSyntaxTree ast; - private CelRuntime.Program program; - - @Before - public void setUp() throws Exception { - cel = + @Test + public void getCoverageReport_fullCoverage() throws Exception { + Cel cel = CelFactory.standardCelBuilder() .addVar("x", SimpleType.INT) .addVar("y", SimpleType.INT) .build(); - ast = cel.compile("x > 1 && y > 1").getAst(); - program = cel.createProgram(ast); - } - - @Test - public void getCoverageReport_fullCoverage() throws Exception { + CelAbstractSyntaxTree ast = cel.compile("x > 1 && y > 1").getAst(); + CelRuntime.Program program = cel.createProgram(ast); CelCoverageIndex coverageIndex = new CelCoverageIndex(); coverageIndex.init(ast); CelEvaluationListener listener = coverageIndex.newEvaluationListener(); @@ -74,6 +68,13 @@ public void getCoverageReport_fullCoverage() throws Exception { @Test public void getCoverageReport_partialCoverage_shortCircuit() throws Exception { + Cel cel = + CelFactory.standardCelBuilder() + .addVar("x", SimpleType.INT) + .addVar("y", SimpleType.INT) + .build(); + CelAbstractSyntaxTree ast = cel.compile("x > 1 && y > 1").getAst(); + CelRuntime.Program program = cel.createProgram(ast); CelCoverageIndex coverageIndex = new CelCoverageIndex(); coverageIndex.init(ast); CelEvaluationListener listener = coverageIndex.newEvaluationListener(); @@ -97,15 +98,15 @@ public void getCoverageReport_partialCoverage_shortCircuit() throws Exception { @Test public void getCoverageReport_comprehension_generatesDotGraph() throws Exception { - cel = CelFactory.standardCelBuilder().build(); + Cel cel = CelFactory.standardCelBuilder().build(); CelCompiler compiler = cel.toCompilerBuilder() .setOptions(CelOptions.newBuilder().populateMacroCalls(true).build()) .setStandardMacros(CelStandardMacro.STANDARD_MACROS) .addLibraries(CelExtensions.comprehensions()) .build(); - ast = compiler.compile("[1, 2, 3].all(i, i % 2 != 0)").getAst(); - program = cel.createProgram(ast); + CelAbstractSyntaxTree ast = compiler.compile("[1, 2, 3].all(i, i % 2 != 0)").getAst(); + CelRuntime.Program program = cel.createProgram(ast); CelCoverageIndex coverageIndex = new CelCoverageIndex(); coverageIndex.init(ast); CelEvaluationListener listener = coverageIndex.newEvaluationListener(); @@ -125,4 +126,32 @@ public void getCoverageReport_comprehension_generatesDotGraph() throws Exception .contains("label=\"{<1> exprID: 16 | <2> LoopStep} | <3> @result && i % 2 != 0\""); assertThat(report.dotGraph()).contains("label=\"{<1> exprID: 17 | <2> Result} | <3> @result\""); } + + @Test + public void getCoverageReport_fullCoverage_writesToUndeclaredOutputs() throws Exception { + // Setup for a more complex graph to write. + Cel cel = CelFactory.standardCelBuilder().build(); + CelCompiler compiler = + cel.toCompilerBuilder() + .setOptions(CelOptions.newBuilder().populateMacroCalls(true).build()) + .setStandardMacros(CelStandardMacro.STANDARD_MACROS) + .addLibraries(CelExtensions.comprehensions()) + .build(); + CelAbstractSyntaxTree ast = compiler.compile("[1, 2, 3].all(i, i % 2 != 0)").getAst(); + CelRuntime.Program program = cel.createProgram(ast); + CelCoverageIndex coverageIndex = new CelCoverageIndex(); + coverageIndex.init(ast); + CelEvaluationListener listener = coverageIndex.newEvaluationListener(); + program.trace(ImmutableMap.of(), listener); + + CoverageReport report = coverageIndex.generateCoverageReport(); + + String undeclaredOutputsDir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR"); + assertThat(undeclaredOutputsDir).isNotNull(); + + File outputFile = new File(undeclaredOutputsDir, "cel_test_coverage/coverage_graph.txt"); + + String fileContent = Files.asCharSource(outputFile, UTF_8).read(); + assertThat(fileContent).isEqualTo(report.dotGraph()); + } }