From aa4f403c71a7a114ddfd81a38aa5cc6244e839f2 Mon Sep 17 00:00:00 2001 From: Jason Brazile Date: Thu, 27 Dec 2018 10:13:45 +0100 Subject: [PATCH] original pull-request but re-based to latest HEAD --- README.md | 16 ++++++ .../java/jp/vmi/junit/result/ITestTarget.java | 27 ++++++++++ .../java/jp/vmi/junit/result/JUnitResult.java | 20 +++++--- .../jp/vmi/junit/result/TestCaseResult.java | 12 +++-- .../vmi/selenium/selenese/ErrorTestCase.java | 11 ++++ .../vmi/selenium/selenese/ErrorTestSuite.java | 11 ++++ .../jp/vmi/selenium/selenese/TestCase.java | 12 +++++ .../jp/vmi/selenium/selenese/TestSuite.java | 12 +++++ .../selenium/selenese/command/Comment.java | 7 +++ .../jp/vmi/junit/result/JUnitResultTest.java | 50 +++++++++++++++++-- .../selenese/DriverIndependentTest.java | 12 +++++ 11 files changed, 175 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 07cc19bc4..35d52afa1 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,22 @@ You can use variables in FILENAME. See [the test-case example](src/test/resources/selenese/testcase_include.html). +Tests in DRAFT mode +------------------- + +Sometimes tests exist before the application-under-test has completely +implemented the feature being tested (in Test-Driven-Development, for +example). You can mark such tests to be in DRAFT (as opposed to FINAL) +mode, whereby they are executed normally and their results are shown. +But for build purposes, they will be accounted as "skipped" rather than +"success" or "fail" - thereby allowing builds including them to succeed. + +This is done by way of a ###-tagged "lifecycle" directive in a comment: + + + +The default lifecycle state is FINAL. + License ------- diff --git a/src/main/java/jp/vmi/junit/result/ITestTarget.java b/src/main/java/jp/vmi/junit/result/ITestTarget.java index 0d2feccfe..bfcbc2879 100644 --- a/src/main/java/jp/vmi/junit/result/ITestTarget.java +++ b/src/main/java/jp/vmi/junit/result/ITestTarget.java @@ -7,6 +7,19 @@ */ public interface ITestTarget { + /** + * Selenese lifecycle types. + */ + @SuppressWarnings("javadoc") + public enum Lifecycle { + FINAL, DRAFT + } + + /** + * Flag for triggering LIFECYCLE_DRAFT in a selenese test. + */ + public static final String FLAG_LIFECYCLE_DRAFT = "### lifecycle=" + Lifecycle.DRAFT.name() + " ###"; + /** * Get test-target file base name. * @@ -36,4 +49,18 @@ public interface ITestTarget { * @return stop watch. */ StopWatch getStopWatch(); + + /** + * Get the lifecycle type. + * + * @return lifecycle type. + */ + Lifecycle getLifecycle(); + + /** + * Set the lifecycle type. + * + * @param lifecycle + */ + void setLifecycle(Lifecycle lifecycle); } diff --git a/src/main/java/jp/vmi/junit/result/JUnitResult.java b/src/main/java/jp/vmi/junit/result/JUnitResult.java index 731bfec12..b7b67f7bc 100644 --- a/src/main/java/jp/vmi/junit/result/JUnitResult.java +++ b/src/main/java/jp/vmi/junit/result/JUnitResult.java @@ -11,6 +11,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import jp.vmi.junit.result.ITestTarget.Lifecycle; import jp.vmi.selenium.selenese.utils.CommandLineUtils; import static jp.vmi.junit.result.ObjectFactory.*; @@ -102,7 +103,7 @@ public void startTestSuite(ITestSuite testSuite) { /** * End test-suite. * - * @param testSuite test-suite instatnce. + * @param testSuite test-suite instance. */ public void endTestSuite(ITestSuite testSuite) { TestSuiteResult suiteResult = (TestSuiteResult) map.remove(testSuite); @@ -125,7 +126,7 @@ public void endTestSuite(ITestSuite testSuite) { /** * Add property in test-suite. * - * @param testSuite test-suite instatnce. + * @param testSuite test-suite instance. * @param name property name. * @param value property value. */ @@ -166,7 +167,8 @@ public void endTestCase(ITestCase testCase) { public void setSuccess(ITestCase testCase) { TestCaseResult caseResult = (TestCaseResult) map.get(testCase); caseResult.setSuccess(); - failsafeSummary.completed++; + if (testCase.getLifecycle() != Lifecycle.DRAFT) + failsafeSummary.completed++; } /** @@ -179,8 +181,10 @@ public void setSuccess(ITestCase testCase) { public void setError(ITestCase testCase, String message, String trace) { TestCaseResult caseResult = (TestCaseResult) map.get(testCase); caseResult.setError(message, trace); - failsafeSummary.completed++; - failsafeSummary.errors++; + if (testCase.getLifecycle() != Lifecycle.DRAFT) { + failsafeSummary.completed++; + failsafeSummary.errors++; + } } /** @@ -193,8 +197,10 @@ public void setError(ITestCase testCase, String message, String trace) { public void setFailure(ITestCase testCase, String message, String trace) { TestCaseResult caseResult = (TestCaseResult) map.get(testCase); caseResult.setFailure(message, trace); - failsafeSummary.completed++; - failsafeSummary.failures++; + if (testCase.getLifecycle() != Lifecycle.DRAFT) { + failsafeSummary.completed++; + failsafeSummary.failures++; + } } /** diff --git a/src/main/java/jp/vmi/junit/result/TestCaseResult.java b/src/main/java/jp/vmi/junit/result/TestCaseResult.java index ab441f4d7..36f4a58f0 100644 --- a/src/main/java/jp/vmi/junit/result/TestCaseResult.java +++ b/src/main/java/jp/vmi/junit/result/TestCaseResult.java @@ -12,6 +12,7 @@ import org.apache.commons.lang3.StringUtils; +import jp.vmi.junit.result.ITestTarget.Lifecycle; import jp.vmi.selenium.selenese.utils.LogRecorder.LogMessage; import static jp.vmi.junit.result.ObjectFactory.*; @@ -44,7 +45,8 @@ public class TestCaseResult extends TestResult { * Set success result. */ public void setSuccess() { - this.success = true; + if (this.testTarget.getLifecycle() != Lifecycle.DRAFT) + this.success = true; } /** @@ -54,7 +56,8 @@ public void setSuccess() { * @param value error value. */ public void setError(String message, String value) { - error = factory.createError(message, value); + if (this.testTarget.getLifecycle() != Lifecycle.DRAFT) + error = factory.createError(message, value); } /** @@ -64,7 +67,8 @@ public void setError(String message, String value) { * @param value failure value. */ public void setFailure(String message, String value) { - failure = factory.createFailure(message, value); + if (this.testTarget.getLifecycle() != Lifecycle.DRAFT) + failure = factory.createFailure(message, value); } /** @@ -93,7 +97,7 @@ public int getFailures() { @XmlElementRef @XmlJavaTypeAdapter(SkippedAdapter.class) public Integer getSkipped() { - return (!success && error == null && failure == null) ? 1 : 0; + return ((this.testTarget.getLifecycle() == Lifecycle.DRAFT) || (!success && error == null && failure == null)) ? 1 : 0; } /** diff --git a/src/main/java/jp/vmi/selenium/selenese/ErrorTestCase.java b/src/main/java/jp/vmi/selenium/selenese/ErrorTestCase.java index 05864e803..38f690ce7 100644 --- a/src/main/java/jp/vmi/selenium/selenese/ErrorTestCase.java +++ b/src/main/java/jp/vmi/selenium/selenese/ErrorTestCase.java @@ -14,6 +14,7 @@ public class ErrorTestCase extends ErrorSource implements ITestCase, IHtmlResultTestCase { private LogRecorder logRecorder = null; + private Lifecycle lifecycle = Lifecycle.FINAL; @Override public void setLogRecorder(LogRecorder logRecorder) { @@ -35,6 +36,16 @@ public Type getType() { return Type.TEST_CASE; } + @Override + public Lifecycle getLifecycle() { + return lifecycle; + } + + @Override + public void setLifecycle(Lifecycle lifecycle) { + this.lifecycle = lifecycle; + } + @ExecuteTestCase @Override public Result execute(Selenese parent, Context context) throws InvalidSeleneseException { diff --git a/src/main/java/jp/vmi/selenium/selenese/ErrorTestSuite.java b/src/main/java/jp/vmi/selenium/selenese/ErrorTestSuite.java index c55b59ab6..9cf63b807 100644 --- a/src/main/java/jp/vmi/selenium/selenese/ErrorTestSuite.java +++ b/src/main/java/jp/vmi/selenium/selenese/ErrorTestSuite.java @@ -12,6 +12,7 @@ public class ErrorTestSuite extends ErrorSource implements ITreedFileGenerator, private ITreedFileGenerator parent = null; private int index = 0; + private Lifecycle lifecycle = Lifecycle.FINAL; @Override public ErrorTestSuite initialize(String filename, InvalidSeleneseException e) { @@ -23,6 +24,16 @@ public Type getType() { return Type.TEST_SUITE; } + @Override + public Lifecycle getLifecycle() { + return lifecycle; + } + + @Override + public void setLifecycle(Lifecycle lifecycle) { + this.lifecycle = lifecycle; + } + @Override public ITreedFileGenerator getParent() { return parent; diff --git a/src/main/java/jp/vmi/selenium/selenese/TestCase.java b/src/main/java/jp/vmi/selenium/selenese/TestCase.java index 39a536e47..25a980047 100644 --- a/src/main/java/jp/vmi/selenium/selenese/TestCase.java +++ b/src/main/java/jp/vmi/selenium/selenese/TestCase.java @@ -47,6 +47,8 @@ public class TestCase implements Selenese, ITestCase, IHtmlResultTestCase { private final StopWatch stopWatch = new StopWatch(); private LogRecorder logRecorder = null; + private Lifecycle lifecycle = Lifecycle.FINAL; + /** * Initialize after constructed. * @@ -74,6 +76,16 @@ public Type getType() { return Type.TEST_CASE; } + @Override + public Lifecycle getLifecycle() { + return lifecycle; + } + + @Override + public void setLifecycle(Lifecycle lifecycle) { + this.lifecycle = lifecycle; + } + /** * Test-case source type. * diff --git a/src/main/java/jp/vmi/selenium/selenese/TestSuite.java b/src/main/java/jp/vmi/selenium/selenese/TestSuite.java index f8a428574..4e05f952d 100644 --- a/src/main/java/jp/vmi/selenium/selenese/TestSuite.java +++ b/src/main/java/jp/vmi/selenium/selenese/TestSuite.java @@ -39,6 +39,8 @@ public class TestSuite implements Selenese, ITreedFileGenerator, ITestSuite, IHt private final StopWatch stopWatch = new StopWatch(); private Result result = UNEXECUTED; + private Lifecycle lifecycle = Lifecycle.FINAL; + /** * Initialize after constructed. * @@ -78,6 +80,16 @@ public Type getType() { return Type.TEST_SUITE; } + @Override + public Lifecycle getLifecycle() { + return lifecycle; + } + + @Override + public void setLifecycle(Lifecycle lifecycle) { + this.lifecycle = lifecycle; + } + @Override public boolean isError() { return false; diff --git a/src/main/java/jp/vmi/selenium/selenese/command/Comment.java b/src/main/java/jp/vmi/selenium/selenese/command/Comment.java index 8a376fb9a..15f190c80 100644 --- a/src/main/java/jp/vmi/selenium/selenese/command/Comment.java +++ b/src/main/java/jp/vmi/selenium/selenese/command/Comment.java @@ -1,5 +1,7 @@ package jp.vmi.selenium.selenese.command; +import jp.vmi.junit.result.ITestTarget; +import jp.vmi.junit.result.ITestTarget.Lifecycle; import jp.vmi.selenium.selenese.Context; import jp.vmi.selenium.selenese.result.Result; @@ -28,6 +30,11 @@ public boolean mayUpdateScreen() { @Override protected Result executeImpl(Context context, String... curArgs) { + // if a comment contains FLAG_LIFECYCLE_DRAFT, set this testcase to DRAFT + String comment = getArguments()[MESSAGE]; + if (comment.contains(ITestTarget.FLAG_LIFECYCLE_DRAFT)) { + context.getCurrentTestCase().setLifecycle(Lifecycle.DRAFT); + } return SUCCESS; } diff --git a/src/test/java/jp/vmi/junit/result/JUnitResultTest.java b/src/test/java/jp/vmi/junit/result/JUnitResultTest.java index d08c22325..a0768ee10 100644 --- a/src/test/java/jp/vmi/junit/result/JUnitResultTest.java +++ b/src/test/java/jp/vmi/junit/result/JUnitResultTest.java @@ -24,6 +24,7 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; +import jp.vmi.junit.result.ITestTarget.Lifecycle; import jp.vmi.selenium.selenese.ITreedFileGenerator; import jp.vmi.selenium.selenese.utils.LogRecorder; import jp.vmi.selenium.selenese.utils.StopWatch; @@ -103,6 +104,7 @@ public void testJUnitResult() throws Exception { final ITestSuite testSuite = new ITestSuite() { private int index = 0; private final StopWatch stopWatch = new StopWatch(); + private Lifecycle lifecycle = Lifecycle.FINAL; @Override public ITreedFileGenerator getParent() { @@ -119,6 +121,16 @@ public void setIndex(int index) { this.index = index; } + @Override + public Lifecycle getLifecycle() { + return lifecycle; + } + + @Override + public void setLifecycle(Lifecycle lifecycle) { + this.lifecycle = lifecycle; + } + @Override public String getBaseName() { return getName(); @@ -139,12 +151,13 @@ public StopWatch getStopWatch() { return stopWatch; } }; - ITestCase[] testCases = new ITestCase[4]; + ITestCase[] testCases = new ITestCase[5]; for (int i = 0; i < testCases.length; i++) { final int num = i; testCases[i] = new ITestCase() { private final StopWatch stopWatch = new StopWatch(); private final LogRecorder logRecorder = new LogRecorder(System.out); + private Lifecycle lifecycle = Lifecycle.FINAL; @Override public String getBaseName() { @@ -156,6 +169,16 @@ public String getName() { return "test-case" + num; } + @Override + public Lifecycle getLifecycle() { + return lifecycle; + } + + @Override + public void setLifecycle(Lifecycle lifecycle) { + this.lifecycle = lifecycle; + } + @Override public boolean isError() { return false; @@ -228,6 +251,14 @@ public LogRecorder getLogRecorder() { sw.start(); sw.end(); jur.endTestCase(tc); + tc = testCases[4]; + tc.setLifecycle(Lifecycle.DRAFT); + sw = tc.getStopWatch(); + jur.startTestCase(testSuite, tc); + sw.start(); + jur.setFailure(tc, "defailt4", "trace4"); + sw.end(); + jur.endTestCase(tc); testSuite.getStopWatch().end(); jur.endTestSuite(testSuite); jur.generateFailsafeSummary(); @@ -244,7 +275,7 @@ public LogRecorder getLogRecorder() { // test-case test. NodeList caseResults = getChild(suiteResult, "testcase", NODESET); - assertThat("test-case:count", caseResults.getLength(), is(4)); + assertThat("test-case:count", caseResults.getLength(), is(5)); Element caseResult; Element error; @@ -303,7 +334,18 @@ public LogRecorder getLogRecorder() { skipped = getChild(caseResult, "skipped", BOOLEAN); assertThat("test-case[3]:error", error, is(nullValue())); assertThat("test-case[3]:failure", failure, is(nullValue())); - assertThat("test-case[2]:skipped", skipped, is(TRUE)); + assertThat("test-case[3]:skipped", skipped, is(TRUE)); + + // test-case 4 test. + caseResult = (Element) caseResults.item(4); + assertThat("test-case[4]:name", caseResult.getAttribute("name"), is("test-case4")); + + error = getChild(caseResult, "error", NODE); + failure = getChild(caseResult, "failure", NODE); + skipped = getChild(caseResult, "skipped", BOOLEAN); + assertThat("test-case[4]:error", error, is(nullValue())); + assertThat("test-case[4]:failure", failure, is(nullValue())); + assertThat("test-case[4]:skipped", skipped, is(TRUE)); // failsafe summary. File summaryFile = new File(tmp.getRoot(), JUnitResult.FAILSAFE_SUMMARY_FILENAME); @@ -314,7 +356,7 @@ public LogRecorder getLogRecorder() { assertThat("failsafe-summary/completed", (String) getChild(summary, "completed", STRING), is("3")); assertThat("failsafe-summary/errors", (String) getChild(summary, "errors", STRING), is("1")); assertThat("failsafe-summary/failures", (String) getChild(summary, "failures", STRING), is("1")); - assertThat("failsafe-summary/skipped", (String) getChild(summary, "skipped", STRING), is("1")); + assertThat("failsafe-summary/skipped", (String) getChild(summary, "skipped", STRING), is("2")); assertThat("failsafe-summary/failureMessage", (String) getChild(summary, "failureMessage", STRING), isEmptyString()); } diff --git a/src/test/java/jp/vmi/selenium/selenese/DriverIndependentTest.java b/src/test/java/jp/vmi/selenium/selenese/DriverIndependentTest.java index 471dbd5fe..fe3b1be85 100644 --- a/src/test/java/jp/vmi/selenium/selenese/DriverIndependentTest.java +++ b/src/test/java/jp/vmi/selenium/selenese/DriverIndependentTest.java @@ -6,6 +6,7 @@ import org.apache.commons.lang3.time.StopWatch; import org.junit.Test; +import jp.vmi.junit.result.ITestTarget.Lifecycle; import jp.vmi.selenium.selenese.command.CommandFactory; import jp.vmi.selenium.selenese.inject.Binder; import jp.vmi.selenium.selenese.result.Error; @@ -126,4 +127,15 @@ public void issue163() { execute("issue163"); assertThat(result, is(instanceOf(Success.class))); } + + @Test + public void lifecycleDraft() { + Runner runner = new Runner(); + runner.setDriver(driver); + CommandFactory cf = runner.getCommandFactory(); + TestCase testCase = Binder.newTestCase(SourceType.SELENESE, "dummy", "dummy", wsr.getBaseURL()); + testCase.addCommand(cf, "comment", ""); + runner.execute(testCase); + assertThat(testCase.getLifecycle(), is(Lifecycle.DRAFT)); + } }