diff --git a/src/main/java/junit/framework/JUnit4TestAdapter.java b/src/main/java/junit/framework/JUnit4TestAdapter.java index cbb66dbb064c..4c9a9e3fcd78 100644 --- a/src/main/java/junit/framework/JUnit4TestAdapter.java +++ b/src/main/java/junit/framework/JUnit4TestAdapter.java @@ -81,6 +81,7 @@ public void filter(Filter filter) throws NoTestsRemainException { } public void sort(Sorter sorter) { - sorter.apply(fRunner); + if(sorter!=null) + sorter.apply(fRunner); } } \ No newline at end of file diff --git a/src/main/java/org/junit/SortWith.java b/src/main/java/org/junit/SortWith.java new file mode 100644 index 000000000000..0d2376412dee --- /dev/null +++ b/src/main/java/org/junit/SortWith.java @@ -0,0 +1,67 @@ +package org.junit; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.Comparator; + +import org.junit.runners.Sorters; + +/** + * The SortWith Annotation that tells JUnit to execute test methods + * or test classes defined in a test suite in a order specified by users. + * It also supports randomness of test by designating a random sorting method + * to either a test class or a test suite. + * + *

+ * A sortable test class with its test methods executed in name ascending order looks like: + * 

+ * 
+ * @RunWith(JUnit4.class)
+ * @SortWith(Sorters.NAME_ASCENDING)
+ * public class Example {
+ *    @Test
+ *    public void testMethod1() {
+ *       
+ *    }
+ *    
+ *    @Test
+ *    public void testMethod2() {
+ *     
+ *    }
+ * }
+ *
+ *

+ * + *

+ * A sortable test suite with its test classes executed in name ascending order looks like: + * 

+ * 
+ * @RunWith(Suite.class)
+ * @Suite.SuiteClasses({ ClassA.class, ClassB.class })
+ * @SortWith(Sorters.NAME_ASCENDING)
+ * public class Example {
+ * 
+ * }
+ *
+ *

+ * + * The SortWith Annotation takes one input parameter, see {@link org.junit.runner.manipulation.Sorter}. + * The parameter declares which sorting method JUnit should adopt when + * starts running tests. If no sorting method is specified for the + * Annotation, then by default the default sorting method which compares + * hash codes of names of the two methods or names of two test classes is + * used. + * + * Be aware that using {@link org.junit.runner.Request#sortWith(Comparator)} to manually specify a sorting + * method for JUnit overrides all the declarative sorting methods specified through SortWith Annotation. + * + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface SortWith { + + Sorters value() default Sorters.DEFAULT; + +} diff --git a/src/main/java/org/junit/internal/requests/ClassRequest.java b/src/main/java/org/junit/internal/requests/ClassRequest.java index af136e71ebc1..5fb13789089b 100644 --- a/src/main/java/org/junit/internal/requests/ClassRequest.java +++ b/src/main/java/org/junit/internal/requests/ClassRequest.java @@ -3,6 +3,9 @@ import org.junit.internal.builders.AllDefaultPossibilitiesBuilder; import org.junit.runner.Request; import org.junit.runner.Runner; +import org.junit.runner.manipulation.Sortable; +import org.junit.runner.manipulation.Sorter; +import org.junit.runners.ParentRunner; public class ClassRequest extends Request { private final Object fRunnerLock = new Object(); @@ -25,6 +28,8 @@ public Runner getRunner() { synchronized (fRunnerLock) { if (fRunner == null) { fRunner = new AllDefaultPossibilitiesBuilder(fCanUseSuiteMethod).safeRunnerForClass(fTestClass); + if(fRunner instanceof Sortable) + ((Sortable)fRunner).sort(Sorter.ANNOTATED_SORTER); } } } diff --git a/src/main/java/org/junit/runner/manipulation/Sorter.java b/src/main/java/org/junit/runner/manipulation/Sorter.java index 0bf5534acd34..fde483d25bef 100644 --- a/src/main/java/org/junit/runner/manipulation/Sorter.java +++ b/src/main/java/org/junit/runner/manipulation/Sorter.java @@ -5,8 +5,11 @@ import org.junit.runner.Description; /** - * A Sorter orders tests. In general you will not need - * to use a Sorter directly. Instead, use {@link org.junit.runner.Request#sortWith(Comparator)}. + * A Sorter orders tests. In general you should specify the Sorter you + * want using SortWith Annotation. + * To use a Sorter directly, use {@link org.junit.runner.Request#sortWith(Comparator)}. + * Be aware, as long as a Sorter is manually specified, all declarative sorting methods defined by + * SortWith Annotation will be overridden. * * @since 4.0 */ @@ -19,6 +22,59 @@ public int compare(Description o1, Description o2) { return 0; } }); + + public static final Sorter ANNOTATED_SORTER=null; + + /** + * DEFAULT sorting method. It compares hash codes of either names of two test methods or names of two test classes, + * depending on whether it is used for sorting execution order for test methods or sorting execution order for + * test classes defined in a test suite. + */ + public static final Sorter DEFAULT=new Sorter(new Comparator(){ + + public int compare(Description o1, Description o2) { + String[] items = getComparisonItems(o1,o2); + int hash1 = items[0].hashCode(); + int hash2 = items[1].hashCode(); + if (hash1 != hash2) { + return hash1 < hash2 ? -1 : 1; + } + return NAME_ASCENDING.compare(o1, o2); + } + + }); + + /** + * Sorting method based on name ascending order. It compares names of two test methods or names of two test classes, + * depending on whether it is used for sorting execution order for test methods or sorting execution order for + * test classes defined in a test suite. + */ + public static final Sorter NAME_ASCENDING = new Sorter(new Comparator(){ + + public int compare(Description o1, Description o2) { + String[] items = getComparisonItems(o1,o2); + final int comparison = items[0].compareTo(items[1]); + if (comparison != 0) { + return comparison; + } + return items[0].toString().compareTo(items[1].toString()); + } + + }); + + /** + * Random sorting method. It generates a random double value between 0 and 1 during runtime such that + * a set of test methods or a set of test classes will always run in a random order. + */ + public static final Sorter RANDOM = new Sorter(new Comparator(){ + + public int compare(Description o1, Description o2) { + return Math.random() - 0.5 >0 ? 1 : -1; + } + + }); + + //public static final Sorter JVM private final Comparator fComparator; @@ -45,4 +101,15 @@ public void apply(Object object) { public int compare(Description o1, Description o2) { return fComparator.compare(o1, o2); } + + private final static String[] getComparisonItems(Description o1,Description o2){ + if(o1.isSuite()){ + final String[] items={o1.getClassName(),o2.getClassName()}; + return items; + }else{ + final String[] items={o1.getMethodName(),o2.getMethodName()}; + return items; + } + } + } diff --git a/src/main/java/org/junit/runners/ParentRunner.java b/src/main/java/org/junit/runners/ParentRunner.java index 8d9bf8ac7939..0c7b8a5a1dc1 100755 --- a/src/main/java/org/junit/runners/ParentRunner.java +++ b/src/main/java/org/junit/runners/ParentRunner.java @@ -17,6 +17,7 @@ import org.junit.ClassRule; import org.junit.Ignore; import org.junit.Rule; +import org.junit.SortWith; import org.junit.internal.AssumptionViolatedException; import org.junit.internal.runners.model.EachTestNotifier; import org.junit.internal.runners.statements.RunAfters; @@ -384,13 +385,25 @@ public void filter(Filter filter) throws NoTestsRemainException { } public void sort(Sorter sorter) { - synchronized (fChildrenLock) { - for (T each : getFilteredChildren()) { - sorter.apply(each); - } + if(sorter==Sorter.ANNOTATED_SORTER){ List sortedChildren = new ArrayList(getFilteredChildren()); - Collections.sort(sortedChildren, comparator(sorter)); + Sorter annotatedSorter=getSorter(); + if(annotatedSorter!=null) + Collections.sort(sortedChildren, comparator(annotatedSorter)); fFilteredChildren = Collections.unmodifiableCollection(sortedChildren); + for (T each : getFilteredChildren()) { + if(each instanceof Sortable) + ((Sortable)each).sort(null); + } + }else{ + synchronized (fChildrenLock) { + for (T each : getFilteredChildren()) { + sorter.apply(each); + } + List sortedChildren = new ArrayList(getFilteredChildren()); + Collections.sort(sortedChildren, comparator(sorter)); + fFilteredChildren = Collections.unmodifiableCollection(sortedChildren); + } } } @@ -436,4 +449,14 @@ public int compare(T o1, T o2) { public void setScheduler(RunnerScheduler scheduler) { this.fScheduler = scheduler; } + + private Sorter getSorter(){ + final TestClass clazz = getTestClass(); + if(clazz==null) + return null; + SortWith sortMethod=clazz.getJavaClass().getAnnotation(SortWith.class); + if(sortMethod==null) + return null; + return sortMethod.value().getSorter(); + } } diff --git a/src/main/java/org/junit/runners/Sorters.java b/src/main/java/org/junit/runners/Sorters.java new file mode 100644 index 000000000000..1f3b36557a25 --- /dev/null +++ b/src/main/java/org/junit/runners/Sorters.java @@ -0,0 +1,44 @@ +package org.junit.runners; + +import org.junit.runner.manipulation.Sorter; + + +/** + * Sorters enumerator used for defining sorting + * method for {@link SortWith} + * + */ +public enum Sorters { + + /** + * default sorting method, using hash code + * of a method name o a class name to compare + */ + DEFAULT(Sorter.DEFAULT), + + /** + * random sorting method. + */ + RANDOM(Sorter.RANDOM), + + /** + * sorting method based on name ascending order + */ + NAME_ASCENDING(Sorter.NAME_ASCENDING), + + /** + * JVM sorting method + */ + JVM(null); + + private final Sorter sorter; + + private Sorters(Sorter sorter){ + this.sorter=sorter; + } + + public Sorter getSorter(){ + return sorter; + } + +} diff --git a/src/main/java/org/junit/runners/model/TestClass.java b/src/main/java/org/junit/runners/model/TestClass.java index e02ec9bb87db..6acfdc6af084 100755 --- a/src/main/java/org/junit/runners/model/TestClass.java +++ b/src/main/java/org/junit/runners/model/TestClass.java @@ -61,7 +61,7 @@ public TestClass(Class klass) { protected void scanAnnotatedMembers(Map, List> methodsForAnnotations, Map, List> fieldsForAnnotations) { for (Class eachClass : getSuperClasses(fClass)) { - for (Method eachMethod : MethodSorter.getDeclaredMethods(eachClass)) { + for (Method eachMethod : eachClass.getDeclaredMethods()) { addToAnnotationLists(new FrameworkMethod(eachMethod), methodsForAnnotations); } // ensuring fields are sorted to make sure that entries are inserted diff --git a/src/test/java/org/junit/tests/manipulation/ClassSorterTest.java b/src/test/java/org/junit/tests/manipulation/ClassSorterTest.java new file mode 100644 index 000000000000..c0d27e845826 --- /dev/null +++ b/src/test/java/org/junit/tests/manipulation/ClassSorterTest.java @@ -0,0 +1,317 @@ +package org.junit.tests.manipulation; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import org.junit.SortWith; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.Description; +import org.junit.runner.JUnitCore; +import org.junit.runner.Request; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.junit.runners.Sorters; +import org.junit.runners.Suite; + +@RunWith(Enclosed.class) +public class ClassSorterTest { + + private static Comparator nameDescending() { + return new Comparator() { + public int compare(Description o1, Description o2) { + return o2.getClassName().compareTo(o1.getClassName()); + } + }; + } + + public static class SortClassInSuiteByName { + + private static String str = ""; + + @RunWith(JUnit4.class) + public static class ClassE { + + @Test + public void testE() { + str += "e"; + } + + } + + @RunWith(JUnit4.class) + public static class ClassB { + + @Test + public void testB() { + str += "b"; + } + + } + + @RunWith(JUnit4.class) + public static class ClassC { + + @Test + public void testC() { + str += "c"; + } + + } + + @RunWith(Suite.class) + @Suite.SuiteClasses({ ClassE.class, ClassB.class, ClassC.class }) + @SortWith(Sorters.NAME_ASCENDING) + public static class ClassSuiteSortByName { + + } + + @Test + public void testSortClassByName() { + Request request = Request.aClass(ClassSuiteSortByName.class); + new JUnitCore().run(request); + assertEquals("bce", str); + } + + } + + public static class SortClassInSuiteByDefault { + + private static List list = new ArrayList(); + + @RunWith(JUnit4.class) + public static class Fantastic { + + @Test + public void testAdd() { + list.add(Fantastic.class.getName()); + } + + } + + @RunWith(JUnit4.class) + public static class Fabulent { + + @Test + public void testAdd() { + list.add(Fabulent.class.getName()); + } + + } + + @RunWith(JUnit4.class) + public static class Splendid { + + @Test + public void testAdd() { + list.add(Splendid.class.getName()); + } + + } + + @RunWith(Suite.class) + @Suite.SuiteClasses({ Fantastic.class, Fabulent.class, Splendid.class }) + @SortWith(Sorters.DEFAULT) + public static class ClassSuiteSortByDefault { + + } + + @Test + public void testSortClassByDefault() { + Request request = Request.aClass(ClassSuiteSortByDefault.class); + new JUnitCore().run(request); + String[] strs = { Fantastic.class.getName(), + Fabulent.class.getName(), Splendid.class.getName() }; + Arrays.sort(strs, new Comparator() { + + public int compare(String o1, String o2) { + int item1 = o1.hashCode(); + int item2 = o2.hashCode(); + if (item1 != item2) { + return item1 < item2 ? -1 : 1; + } + return 0; + } + + }); + String[] results = new String[3]; + assertEquals(strs, SortClassInSuiteByDefault.list.toArray(results)); + } + + } + + public static class SortClassInSuiteByRandom { + + public static String str = ""; + + @RunWith(JUnit4.class) + public static class ClassRA { + + @Test + public void testAdd() { + str += "RA"; + } + + } + + @RunWith(JUnit4.class) + public static class ClassRB { + + @Test + public void testAdd() { + str += "RB"; + } + + } + + @RunWith(JUnit4.class) + public static class ClassRC { + + @Test + public void testAdd() { + str += "RC"; + } + + } + + @RunWith(JUnit4.class) + public static class ClassRD { + + @Test + public void testAdd() { + str += "RD"; + } + + } + + @RunWith(JUnit4.class) + public static class ClassRE { + + @Test + public void testAdd() { + str += "RE"; + } + + } + + @RunWith(Suite.class) + @Suite.SuiteClasses({ ClassRA.class, ClassRB.class, ClassRC.class, + ClassRD.class, ClassRE.class }) + @SortWith(Sorters.RANDOM) + public static class ClassSuiteSortByRandom { + + } + + @Test + public void testSortClassByRandom() { + String lastExecutionOrder = null; + int i = 0; + for (; i < 10; i++, lastExecutionOrder = SortClassInSuiteByRandom.str, lastExecutionOrder = SortClassInSuiteByRandom.str = "") { + Request request = Request.aClass(ClassSuiteSortByRandom.class); + new JUnitCore().run(request); + if (lastExecutionOrder == null) { + lastExecutionOrder = SortClassInSuiteByRandom.str; + continue; + } + if (!lastExecutionOrder.equals(SortClassInSuiteByRandom.str)) + break; + } + assertNotEquals(10, i); + } + } + + public static class SortClassInSuiteByJVM { + + public static String str = ""; + + @RunWith(JUnit4.class) + public static class ClassJVMA { + + @Test + public void testAdd() { + str += "A"; + } + + } + + @RunWith(JUnit4.class) + public static class ClassJVMB { + + @Test + public void testAdd() { + str += "B"; + } + + } + + @RunWith(Suite.class) + @Suite.SuiteClasses({ ClassJVMA.class, ClassJVMB.class }) + @SortWith(Sorters.JVM) + public static class ClassSuiteSortByJVM { + + } + + @Test + public void testSortClassByJVM() { + Request request = Request.aClass(ClassSuiteSortByJVM.class); + new JUnitCore().run(request); + String test1 = SortClassInSuiteByJVM.str; + SortClassInSuiteByJVM.str = ""; + request = Request.aClass(ClassSuiteSortByJVM.class); + new JUnitCore().run(request); + String test2 = SortClassInSuiteByJVM.str; + assertEquals(test1, test2); + } + + } + + public static class OverrideSortingMethodInSuite { + + public static String str = ""; + + @RunWith(JUnit4.class) + public static class ClassA { + + @Test + public void testAdd() { + str += "a"; + } + + } + + @RunWith(JUnit4.class) + public static class ClassB { + + @Test + public void testAdd() { + str += "b"; + } + + } + + @RunWith(Suite.class) + @Suite.SuiteClasses({ ClassA.class, ClassB.class }) + @SortWith(Sorters.NAME_ASCENDING) + public static class ClassSuiteSortByNameAscending { + + } + + @Test + public void testOverrideClassOrder() { + Request request = Request.aClass(ClassSuiteSortByNameAscending.class); + new JUnitCore().run(request); + assertEquals("ab",OverrideSortingMethodInSuite.str); + OverrideSortingMethodInSuite.str=""; + request = Request.aClass(ClassSuiteSortByNameAscending.class).sortWith(nameDescending()); + new JUnitCore().run(request); + assertEquals("ba",OverrideSortingMethodInSuite.str); + } + + } + +} diff --git a/src/test/java/org/junit/tests/manipulation/MethodSorterTest.java b/src/test/java/org/junit/tests/manipulation/MethodSorterTest.java new file mode 100644 index 000000000000..4a346f41259b --- /dev/null +++ b/src/test/java/org/junit/tests/manipulation/MethodSorterTest.java @@ -0,0 +1,262 @@ +package org.junit.tests.manipulation; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.SortWith; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.Description; +import org.junit.runner.JUnitCore; +import org.junit.runner.Request; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.junit.runners.Sorters; + +@RunWith(Enclosed.class) +public class MethodSorterTest { + + private static Comparator nameDescending() { + return new Comparator() { + public int compare(Description o1, Description o2) { + return o2.getDisplayName().compareTo(o1.getDisplayName()); + } + }; + } + + public static class TestSortByName { + + @RunWith(JUnit4.class) + @SortWith(Sorters.NAME_ASCENDING) + public static class SortByName { + + public static String str = ""; + + @Test + public void testA() { + str += "a"; + } + + @Test + public void testB() { + str += "b"; + } + + @Test + public void testC() { + str += "c"; + } + + @Test + public void testD() { + str += "d"; + } + + @Test + public void testE() { + str += "e"; + } + } + + @Test + public void testNameAscendingOrder() { + Request request = Request.aClass(SortByName.class); + new JUnitCore().run(request); + assertEquals("abcde", SortByName.str); + } + } + + public static class TestSortByDefault { + + @RunWith(JUnit4.class) + @SortWith + public static class SortByDefault { + + public static List list = new ArrayList(); + + @Test + public void fun() { + list.add("fun"); + } + + @Test + public void ecstasy() { + list.add("ecstasy"); + } + + @Test + public void happy() { + list.add("happy"); + } + + @Test + public void meow() { + list.add("meow"); + } + + @Test + public void halirious() { + list.add("halirious"); + } + } + + @Test + public void testDefaultOrder() { + Request request = Request.aClass(SortByDefault.class); + new JUnitCore().run(request); + String[] strs = { "fun", "ecstasy", "happy", "meow" ,"halirious"}; + Arrays.sort(strs, new Comparator() { + + public int compare(String o1, String o2) { + int item1 = o1.hashCode(); + int item2 = o2.hashCode(); + if (item1 != item2) { + return item1 < item2 ? -1 : 1; + } + return 0; + } + + }); + String[] results = new String[4]; + assertEquals(strs, SortByDefault.list.toArray(results)); + } + } + + public static class TestSortByRandom { + + @RunWith(JUnit4.class) + @SortWith(Sorters.RANDOM) + public static class SortByRandom{ + + public static String str=""; + + @Test + public void testA(){ + str += "a"; + } + + @Test + public void testB(){ + str += "b"; + } + + @Test + public void testC(){ + str += "c"; + } + + @Test + public void testD(){ + str += "d"; + } + + @Test + public void testE(){ + str += "e"; + } + + @Test + public void testF(){ + str += "f"; + } + + } + + @Test + public void testRandomOrder() { + String lastExecutionOrder=null; + int i=0; + for(;i<10;i++,lastExecutionOrder=SortByRandom.str,SortByRandom.str=""){ + Request request = Request.aClass(SortByRandom.class); + new JUnitCore().run(request); + if(lastExecutionOrder==null){ + lastExecutionOrder=SortByRandom.str; + continue; + } + if(!lastExecutionOrder.equals(SortByRandom.str)) + break; + } + assertNotEquals(10,i); + } + + } + + public static class TestSortByJVM{ + + @RunWith(JUnit4.class) + @SortWith(Sorters.JVM) + public static class SortByJVM{ + + public static String str=""; + + @Test + public void testA(){ + str+="a"; + } + + @Test + public void testB(){ + str+="b"; + } + + } + + @Test + public void testJVMOrder(){ + Request request = Request.aClass(SortByJVM.class); + new JUnitCore().run(request); + String test1=SortByJVM.str; + SortByJVM.str=""; + request = Request.aClass(SortByJVM.class); + new JUnitCore().run(request); + String test2=SortByJVM.str; + assertEquals(test1,test2); + } + } + + public static class TestOverrideSorting{ + + @RunWith(JUnit4.class) + @SortWith(Sorters.NAME_ASCENDING) + public static class SortByNameAscending{ + + public static String str=""; + + @Test + public void testA(){ + str+="a"; + } + + @Test + public void testB(){ + str+="b"; + } + + @Test + public void testC(){ + str+="c"; + } + + } + + @Test + public void testOverideSorting(){ + Request ascending=Request.aClass(SortByNameAscending.class); + new JUnitCore().run(ascending); + assertEquals("abc",SortByNameAscending.str); + SortByNameAscending.str=""; + Request descending = Request.aClass(SortByNameAscending.class).sortWith(nameDescending()); + new JUnitCore().run(descending); + assertEquals("cba",SortByNameAscending.str); + } + + } + +} diff --git a/src/test/java/org/junit/tests/running/classes/ParentRunnerTest.java b/src/test/java/org/junit/tests/running/classes/ParentRunnerTest.java index 4ce0140fe1c9..14dc47e53f4e 100644 --- a/src/test/java/org/junit/tests/running/classes/ParentRunnerTest.java +++ b/src/test/java/org/junit/tests/running/classes/ParentRunnerTest.java @@ -22,10 +22,17 @@ import org.junit.runners.model.RunnerScheduler; import org.junit.tests.experimental.rules.RuleFieldValidatorTest.TestWithNonStaticClassRule; import org.junit.tests.experimental.rules.RuleFieldValidatorTest.TestWithProtectedClassRule; +import org.junit.SortWith; public class ParentRunnerTest { public static String log = ""; + /* + * Adding SortWith here because MethodSorter is no longer used when creating a test case. + * So if no sorting method is specified, the test methods will be executed in JVM order, + * which has chance to produce randomness to fail this test case. + */ + @SortWith public static class FruitTest { @Test public void apple() { @@ -41,7 +48,8 @@ public void apple() { @Test public void useChildHarvester() throws InitializationError { log = ""; - ParentRunner runner = new BlockJUnit4ClassRunner(FruitTest.class); + Request request = Request.aClass(FruitTest.class); + ParentRunner runner = (ParentRunner)request.getRunner(); runner.setScheduler(new RunnerScheduler() { public void schedule(Runnable childStatement) { log += "before ";