diff --git a/docs/Configuration.md b/docs/Configuration.md index 2557cc8..0cb908a 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -60,3 +60,5 @@ In addition to: there is also: - `timeout(Duration timeout)` - make the test fail if it takes too long - see [Timeout](Timeout.md) +- `randomOrder()` - put the tests into a different order each run - this outputs the seed used, allowing you to repeat a particular run +- `randomOrder(long seed)` - put the tests into a pseudo-random order, using the given seed to fix that order - most commonly used to diagnose a particular order having an effect on the result diff --git a/src/main/java/com/greghaskins/spectrum/Configure.java b/src/main/java/com/greghaskins/spectrum/Configure.java index bcbe257..9997a41 100644 --- a/src/main/java/com/greghaskins/spectrum/Configure.java +++ b/src/main/java/com/greghaskins/spectrum/Configure.java @@ -3,6 +3,7 @@ import com.greghaskins.spectrum.internal.DeclarationState; import com.greghaskins.spectrum.internal.configuration.BlockFocused; import com.greghaskins.spectrum.internal.configuration.BlockIgnore; +import com.greghaskins.spectrum.internal.configuration.BlockRandomOrder; import com.greghaskins.spectrum.internal.configuration.BlockTagging; import com.greghaskins.spectrum.internal.configuration.BlockTimeout; import com.greghaskins.spectrum.internal.configuration.ConfiguredBlock; @@ -109,6 +110,24 @@ static BlockConfigurationChain timeout(Duration timeout) { return new BlockConfigurationChain().with(new BlockTimeout(timeout)); } + /** + * Apply random order to all parents from this point in the hierarchy down. + * @return a chainable configuration that will apply random test order to all parent nodes below + */ + static BlockConfigurationChain randomOrder() { + return new BlockConfigurationChain().with(new BlockRandomOrder()); + } + + /** + * Apply random order to all parents from this point in the hierarchy down. + * Using the given seed to make that random order the same every time. + * @param seed a fixed random seed to (temporarily) fix the order for diagnostics. + * @return a chainable configuration that will apply random test order to all parent nodes below + */ + static BlockConfigurationChain randomOrder(long seed) { + return new BlockConfigurationChain().with(new BlockRandomOrder(seed)); + } + /** * Filter which tests in the current suite will run. * diff --git a/src/main/java/com/greghaskins/spectrum/internal/ExecutionSequenceApplier.java b/src/main/java/com/greghaskins/spectrum/internal/ExecutionSequenceApplier.java new file mode 100644 index 0000000..576edc5 --- /dev/null +++ b/src/main/java/com/greghaskins/spectrum/internal/ExecutionSequenceApplier.java @@ -0,0 +1,12 @@ +package com.greghaskins.spectrum.internal; + +/** + * Tagging interface for a {@link Parent} that sequences its children. + */ +public interface ExecutionSequenceApplier { + /** + * Attache the sequencing strategy. + * @param sequencer execution sequencer that orders the children. + */ + void setSequencer(ExecutionSequencer sequencer); +} diff --git a/src/main/java/com/greghaskins/spectrum/internal/ExecutionSequencer.java b/src/main/java/com/greghaskins/spectrum/internal/ExecutionSequencer.java new file mode 100644 index 0000000..2aefaaf --- /dev/null +++ b/src/main/java/com/greghaskins/spectrum/internal/ExecutionSequencer.java @@ -0,0 +1,18 @@ +package com.greghaskins.spectrum.internal; + +import java.util.List; + +/** + * Put the children into an execution order. + */ +@FunctionalInterface +public interface ExecutionSequencer { + ExecutionSequencer DEFAULT = list -> list; + + /** + * Apply the ordering to the list of children. + * @param originalOrder order they are stored + * @return re-ordered + */ + List sequence(List originalOrder); +} diff --git a/src/main/java/com/greghaskins/spectrum/internal/Suite.java b/src/main/java/com/greghaskins/spectrum/internal/Suite.java index efc3507..16bcd8d 100644 --- a/src/main/java/com/greghaskins/spectrum/internal/Suite.java +++ b/src/main/java/com/greghaskins/spectrum/internal/Suite.java @@ -1,5 +1,6 @@ package com.greghaskins.spectrum.internal; +import static com.greghaskins.spectrum.internal.ExecutionSequencer.DEFAULT; import static com.greghaskins.spectrum.internal.configuration.BlockConfiguration.merge; import com.greghaskins.spectrum.Block; @@ -18,7 +19,7 @@ import java.util.List; import java.util.Set; -public class Suite implements Parent, Child { +public class Suite implements Parent, Child, ExecutionSequenceApplier { private Hooks hooks = new Hooks(); protected final List children = new ArrayList<>(); @@ -34,6 +35,8 @@ public class Suite implements Parent, Child { private BlockConfiguration configuration = BlockConfiguration.defaultConfiguration(); private NameSanitiser nameSanitiser = new NameSanitiser(); + private ExecutionSequencer sequencer = DEFAULT; + /** * The strategy for running the children within the suite. */ @@ -152,6 +155,11 @@ public Hooks getInheritableHooks() { return this.parent.getInheritableHooks().plus(this.hooks.forAtomic()); } + @Override + public void setSequencer(ExecutionSequencer sequencer) { + this.sequencer = sequencer; + } + /** * Set the suite to require certain tags of all tests below. * @@ -283,7 +291,8 @@ public void removeAllChildren() { private static void defaultChildRunner(final Suite suite, final RunReporting reporting) { - suite.children.forEach((child) -> suite.runChild(child, reporting)); + suite.sequencer.sequence(suite.children) + .forEach((child) -> suite.runChild(child, reporting)); } private String sanitise(final String name) { diff --git a/src/main/java/com/greghaskins/spectrum/internal/configuration/BlockRandomOrder.java b/src/main/java/com/greghaskins/spectrum/internal/configuration/BlockRandomOrder.java new file mode 100644 index 0000000..a6ca25a --- /dev/null +++ b/src/main/java/com/greghaskins/spectrum/internal/configuration/BlockRandomOrder.java @@ -0,0 +1,53 @@ +package com.greghaskins.spectrum.internal.configuration; + +import com.greghaskins.spectrum.internal.Child; +import com.greghaskins.spectrum.internal.ExecutionSequenceApplier; +import com.greghaskins.spectrum.internal.ExecutionSequencer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; + +/** + * Configuration block that randomises the order of tests in their parent. + */ +public class BlockRandomOrder implements BlockConfigurable, ExecutionSequencer { + private Random random; + + public BlockRandomOrder() { + this(System.currentTimeMillis()); + } + + public BlockRandomOrder(long seed) { + random = new Random(seed); + System.out.println("Random execution order set using seed: " + seed); + } + + @Override + public boolean inheritedByChild() { + return true; + } + + @Override + public void applyTo(Child child, TaggingFilterCriteria state) { + if (child instanceof ExecutionSequenceApplier) { + ((ExecutionSequenceApplier) child).setSequencer(this); + } + } + + @Override + public BlockConfigurable merge(BlockConfigurable other) { + // my value supersedes any inherited value + + return this; + } + + @Override + public List sequence(List originalOrder) { + List result = new ArrayList<>(originalOrder); + Collections.shuffle(result, random); + + return result; + } +} diff --git a/src/test/java/specs/RandomOrderSpecs.java b/src/test/java/specs/RandomOrderSpecs.java new file mode 100644 index 0000000..51752fa --- /dev/null +++ b/src/test/java/specs/RandomOrderSpecs.java @@ -0,0 +1,120 @@ +package specs; + +import static com.greghaskins.spectrum.Configure.randomOrder; +import static com.greghaskins.spectrum.Configure.with; +import static com.greghaskins.spectrum.Spectrum.describe; +import static com.greghaskins.spectrum.Spectrum.it; +import static com.greghaskins.spectrum.dsl.gherkin.Gherkin.*; +import static java.util.stream.Collectors.toList; +import static junit.framework.TestCase.fail; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +import com.greghaskins.spectrum.BlockConfigurationChain; +import com.greghaskins.spectrum.Spectrum; +import com.greghaskins.spectrum.SpectrumHelper; + +import org.junit.runner.Result; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.List; + +@RunWith(Spectrum.class) +public class RandomOrderSpecs { + { + describe("Random order specs", () -> { + it("can execute at all", () -> { + Result result = SpectrumHelper.run(() -> { + buildSuite(randomOrder()); + }); + assertThat(result.getRunCount(), is(6)); + }); + + it("will have different execution order each time", () -> { + List originalFailures = failureList(SpectrumHelper.run(() -> { + buildSuite(randomOrder()); + })); + + // as we are dealing with random numbers, it may take a few goes before a new + // permutation comes up + int iteration = 0; + Result nextResult; + do { + nextResult = SpectrumHelper.run(() -> { + buildSuite(randomOrder()); + }); + iteration++; + } while (iteration < 100 && failureList(nextResult).equals(originalFailures)); + + // should not have reached the limit where we gave up finding a new one + assertThat(iteration, not(is(100))); + }); + + it("can have the same execution order each time with a seed", () -> { + Result result1 = SpectrumHelper.run(() -> { + buildSuite(randomOrder(12345)); + }); + Result result2 = SpectrumHelper.run(() -> { + buildSuite(randomOrder(12345)); + }); + + assertThat(failureList(result1), is(failureList(result2))); + }); + + describe("composite tests", with(randomOrder(), () -> { + scenario("a test that relies on order in a random order tree", () -> { + final List strings = new ArrayList<>(); + given("first step adds first", () -> { + strings.add("first"); + }); + when("second step adds second", () -> { + strings.add("second"); + }); + and("third step adds third", () -> { + strings.add("third"); + }); + then("the order is correct", () -> { + assertThat(strings, contains("first", "second", "third")); + }); + }); + })); + }); + } + + private List failureList(Result result) { + return result.getFailures().stream() + .map(failure -> failure.getDescription().getMethodName()) + .collect(toList()); + } + + private static void buildSuite(BlockConfigurationChain order) { + // each test fails so we can use the failures to determine if they were random + describe("Tests in random order", with(order, () -> { + describe("scramble each level of the hierarchy", () -> { + it("happens whenever", () -> { + fail(); + }); + it("happens after or before", () -> { + fail(); + }); + it("can happen when it likes", () -> { + fail(); + }); + }); + describe("make each level of the hierarchy different", () -> { + it("then happens whenever", () -> { + fail(); + }); + it("then happens after or before", () -> { + fail(); + }); + it("then can happen when it likes", () -> { + fail(); + }); + }); + })); + } +}