diff --git a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java index 5c38caa70d..02d168da25 100644 --- a/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java +++ b/surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java @@ -24,6 +24,7 @@ import org.apache.maven.surefire.api.provider.CommandListener; import org.apache.maven.surefire.api.report.TestOutputReportEntry; import org.apache.maven.surefire.api.report.TestReportListener; +import org.apache.maven.surefire.api.testset.ResolvedTest; import org.apache.maven.surefire.common.junit4.JUnit4RunListener; import org.apache.maven.surefire.common.junit4.JUnit4TestChecker; import org.apache.maven.surefire.common.junit4.JUnitTestFailureListener; @@ -50,11 +51,17 @@ import org.junit.runner.manipulation.Filter; import org.junit.runner.notification.StoppedByUserException; +import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; +import java.util.List; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static java.lang.reflect.Modifier.isAbstract; import static java.lang.reflect.Modifier.isInterface; +import static java.util.Collections.emptySet; import static org.apache.maven.surefire.api.report.RunMode.NORMAL_RUN; import static org.apache.maven.surefire.api.report.RunMode.RERUN_TEST_AFTER_FAILURE; import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.createMatchAnyDescriptionFilter; @@ -78,6 +85,7 @@ public class JUnit4Provider extends AbstractProvider { + private static final Pattern JUNIT_TEST_DESCRIPTION_PATTERN = Pattern.compile( "(.*)\\((.*)\\)" ); private static final String UNDETERMINED_TESTS_DESCRIPTION = "cannot determine test in forked JVM with surefire"; private final ClassMethodIndexer classMethodIndexer = new ClassMethodIndexer(); @@ -275,13 +283,14 @@ private void executeWithRerun( Class clazz, Notifier notifier, RunModeSetter JUnitTestFailureListener failureListener = new JUnitTestFailureListener(); notifier.addListener( failureListener ); boolean hasMethodFilter = testResolver != null && testResolver.hasMethodPatterns(); + Set includedPatterns = testResolver != null ? testResolver.getIncludedPatterns() : emptySet(); try { try { notifier.asFailFast( isFailFast() ); - execute( clazz, notifier, hasMethodFilter ? createMethodFilter() : null ); + execute( clazz, notifier, hasMethodFilter ? createMethodFilter() : null, includedPatterns ); } finally { @@ -299,7 +308,7 @@ private void executeWithRerun( Class clazz, Notifier notifier, RunModeSetter Set failures = generateFailingTestDescriptions( failureListener.getAllFailures() ); failureListener.reset(); Filter failureDescriptionFilter = createMatchAnyDescriptionFilter( failures ); - execute( clazz, rerunNotifier, failureDescriptionFilter ); + execute( clazz, rerunNotifier, failureDescriptionFilter, includedPatterns ); } } } @@ -361,12 +370,19 @@ private static boolean isJUnit4UpgradeCheck() return System.getProperty( "surefire.junit4.upgradecheck" ) != null; } - private static void execute( Class testClass, Notifier notifier, Filter filter ) + private static void execute( Class testClass, Notifier notifier, Filter filter, + Set includedPatterns ) { final int classModifiers = testClass.getModifiers(); if ( !isAbstract( classModifiers ) && !isInterface( classModifiers ) ) { Request request = aClass( testClass ); + + if ( !includedPatterns.isEmpty() ) + { + request = sortByTestOrder( includedPatterns, request ); + } + if ( filter != null ) { request = request.filterWith( filter ); @@ -379,6 +395,50 @@ private static void execute( Class testClass, Notifier notifier, Filter filte } } + private static Request sortByTestOrder( Set includedPatterns, Request request ) + { + // This relies on the set being a LinkedHashSet (predictable iteration order) + List testOrder = new ArrayList<>( includedPatterns ); + Comparator descriptionComparator = ( d1, d2 ) -> + { + int i1 = indexOf( testOrder, d1 ); + int i2 = indexOf( testOrder, d2 ); + return i1 - i2; + }; + return request.sortWith( descriptionComparator ); + } + + /** + * Finds the position of the ResolvedTest that matches the JUnit description + * @param orderedTests a test order + * @param description JUnit Description + * @return the position, or Integer.MAX_VALUE if not found + */ + private static int indexOf( List orderedTests, Description description ) + { + if ( description.isSuite() ) + { + return Integer.MAX_VALUE; + } + Matcher m = JUNIT_TEST_DESCRIPTION_PATTERN.matcher( description.toString() ); + if ( !m.matches() ) + { + return Integer.MAX_VALUE; + } + String methodName = m.group( 1 ); + String className = m.group( 2 ); + String classFileName = className.replace( ".", "/" ) + ".class"; + for ( int i = 0; i < orderedTests.size(); i++ ) + { + ResolvedTest resolvedTest = orderedTests.get( i ); + if ( resolvedTest.matchAsInclusive( classFileName, methodName ) ) + { + return i; + } + } + return Integer.MAX_VALUE; + } + /** * JUnit error: test count includes one test-class as a suite which has filtered out all children. * Then the child test has a description "initializationError0(org.junit.runner.manipulation.Filter)" diff --git a/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderOrderingTest.java b/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderOrderingTest.java new file mode 100644 index 0000000000..fb32361b01 --- /dev/null +++ b/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderOrderingTest.java @@ -0,0 +1,158 @@ +package org.apache.maven.surefire.junit4; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.api.booter.BaseProviderFactory; +import org.apache.maven.surefire.api.report.ReporterFactory; +import org.apache.maven.surefire.api.report.TestOutputReportEntry; +import org.apache.maven.surefire.api.report.TestReportListener; +import org.apache.maven.surefire.api.suite.RunResult; +import org.apache.maven.surefire.api.testset.RunOrderParameters; +import org.apache.maven.surefire.api.testset.TestListResolver; +import org.apache.maven.surefire.api.testset.TestRequest; +import org.apache.maven.surefire.api.testset.TestSetFailedException; +import org.apache.maven.surefire.api.util.TestsToRun; +import org.junit.Ignore; +import org.junit.Test; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.Arrays.asList; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +/** + * @author Aslak Hellesøy + */ +public class JUnit4ProviderOrderingTest +{ + /** + */ + public static class TestX + { + @Test + public void testA() + { + } + + @Test + public void testB() + { + } + + @Test + public void testC() + { + } + } + + /** + */ + public static class TestY + { + @Test + public void testD() + { + } + + @Test + public void testE() + { + } + + @Test + public void testF() + { + } + + } + + @Test + public void testShouldOrderWithinSingleTestClass() throws TestSetFailedException + { + assertTestOrder( + TestX.class.getName() + "#testC", + TestX.class.getName() + "#testA", + TestX.class.getName() + "#testB" + ); + } + + @Test + @Ignore( "this is currently not possible" ) + public void testShouldOrderWithinInterleavedTestClasses() throws TestSetFailedException + { + assertTestOrder( + TestX.class.getName() + "#testC", + TestY.class.getName() + "#testE", + TestX.class.getName() + "#testA", + TestY.class.getName() + "#testD", + TestX.class.getName() + "#testB", + TestY.class.getName() + "#testF" + ); + } + + private void assertTestOrder( String... tests ) throws TestSetFailedException + { + List testOrder = asList( tests ); + BaseProviderFactory providerParameters = new BaseProviderFactory( true ); + providerParameters.setProviderProperties( new HashMap<>() ); + providerParameters.setClassLoaders( getClass().getClassLoader() ); + TestListResolver requestedTests = new TestListResolver( String.join( ",", testOrder ) ); + TestRequest testRequest = new TestRequest( null, null, requestedTests ); + providerParameters.setTestRequest( testRequest ); + providerParameters.setRunOrderParameters( RunOrderParameters.alphabetical() ); + + MockReporter testReportListener = new MockReporter(); + providerParameters.setReporterFactory( new StubReporterFactory( testReportListener ) ); + JUnit4Provider provider = new JUnit4Provider( providerParameters ); + TestsToRun testsToRun = new TestsToRun( new HashSet<>( asList( TestX.class, TestY.class ) ) ); + provider.invoke( testsToRun ); + + List actualOrder = testReportListener.getReports().stream() + .map( r -> String.format( "%s#%s", r.getSourceName(), r.getName() ) ) + .collect( Collectors.toList() ); + + assertThat( actualOrder, is( testOrder ) ); + } + + private static class StubReporterFactory implements ReporterFactory + { + private final TestReportListener testReportListener; + + private StubReporterFactory( TestReportListener testReportListener ) + { + this.testReportListener = testReportListener; + } + + @Override + public TestReportListener createTestReportListener() + { + return testReportListener; + } + + @Override + public RunResult close() + { + return null; + } + } +} diff --git a/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4SuiteTest.java b/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4SuiteTest.java index 2d4fb72882..4118b1a5c6 100644 --- a/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4SuiteTest.java +++ b/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4SuiteTest.java @@ -22,6 +22,7 @@ import junit.framework.JUnit4TestAdapter; import junit.framework.Test; import junit.framework.TestCase; +import junit.framework.TestSuite; /** * Adapt the JUnit4 tests which use only annotations to the JUnit3 test suite. @@ -30,6 +31,9 @@ public class JUnit4SuiteTest extends TestCase { public static Test suite() { - return new JUnit4TestAdapter( JUnit4ProviderTest.class ); + TestSuite testSuite = new TestSuite(); + testSuite.addTest( new JUnit4TestAdapter( JUnit4ProviderTest.class ) ); + testSuite.addTest( new JUnit4TestAdapter( JUnit4ProviderOrderingTest.class ) ); + return testSuite; } } diff --git a/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/MockReporter.java b/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/MockReporter.java new file mode 100644 index 0000000000..aa4ac668bb --- /dev/null +++ b/surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/MockReporter.java @@ -0,0 +1,147 @@ +package org.apache.maven.surefire.junit4; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.api.report.ReportEntry; +import org.apache.maven.surefire.api.report.TestOutputReportEntry; +import org.apache.maven.surefire.api.report.TestReportListener; +import org.apache.maven.surefire.api.report.TestSetReportEntry; + +import java.util.ArrayList; +import java.util.List; + +/** + * Internal tests use only. + */ +final class MockReporter + implements TestReportListener +{ + private final List reports = new ArrayList<>(); + + @Override + public void testSetStarting( TestSetReportEntry report ) + { + } + + @Override + public void testSetCompleted( TestSetReportEntry report ) + { + } + + @Override + public void testStarting( ReportEntry report ) + { + reports.add( report ); + } + + @Override + public void testSucceeded( ReportEntry report ) + { + } + + @Override + public void testSkipped( ReportEntry report ) + { + } + + @Override + public void testExecutionSkippedByUser() + { + } + + @Override + public void testError( ReportEntry report ) + { + } + + @Override + public void testFailed( ReportEntry report ) + { + } + + @Override + public void testAssumptionFailure( ReportEntry report ) + { + } + + @Override + public void writeTestOutput( TestOutputReportEntry reportEntry ) + { + } + + @Override + public boolean isDebugEnabled() + { + return false; + } + + @Override + public void debug( String message ) + { + } + + @Override + public boolean isInfoEnabled() + { + return false; + } + + @Override + public void info( String message ) + { + } + + @Override + public boolean isWarnEnabled() + { + return false; + } + + @Override + public void warning( String message ) + { + } + + @Override + public boolean isErrorEnabled() + { + return false; + } + + @Override + public void error( String message ) + { + } + + @Override + public void error( String message, Throwable t ) + { + } + + @Override + public void error( Throwable t ) + { + } + + public List getReports() + { + return reports; + } +}