001 // This file is part of the program FRYSK.
002 //
003 // Copyright 2005, 2006, 2007, 2008, Red Hat Inc.
004 //
005 // FRYSK is free software; you can redistribute it and/or modify it
006 // under the terms of the GNU General Public License as published by
007 // the Free Software Foundation; version 2 of the License.
008 //
009 // FRYSK is distributed in the hope that it will be useful, but
010 // WITHOUT ANY WARRANTY; without even the implied warranty of
011 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
012 // General Public License for more details.
013 //
014 // You should have received a copy of the GNU General Public License
015 // along with FRYSK; if not, write to the Free Software Foundation,
016 // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
017 //
018 // In addition, as a special exception, Red Hat, Inc. gives You the
019 // additional right to link the code of FRYSK with code not covered
020 // under the GNU General Public License ("Non-GPL Code") and to
021 // distribute linked combinations including the two, subject to the
022 // limitations in this paragraph. Non-GPL Code permitted under this
023 // exception must only link to the code of FRYSK through those well
024 // defined interfaces identified in the file named EXCEPTION found in
025 // the source code files (the "Approved Interfaces"). The files of
026 // Non-GPL Code may instantiate templates or use macros or inline
027 // functions from the Approved Interfaces without causing the
028 // resulting work to be covered by the GNU General Public
029 // License. Only Red Hat, Inc. may make changes or additions to the
030 // list of Approved Interfaces. You must obey the GNU General Public
031 // License in all respects for all of the FRYSK code and other code
032 // used in conjunction with FRYSK except the Non-GPL Code covered by
033 // this exception. If you modify this file, you may extend this
034 // exception to your version of the file, but you are not obligated to
035 // do so. If you do not wish to provide this exception without
036 // modification, you must delete this exception statement from your
037 // version and license this file solely under the GPL without
038 // exception.
039
040 package frysk.junit;
041
042 import frysk.config.FryskVersion;
043 import frysk.config.Prefix;
044 import frysk.config.Host;
045 import frysk.rsl.Log;
046 import frysk.rsl.LogOption;
047 import frysk.expunit.Expect;
048 import gnu.classpath.tools.getopt.FileArgumentCallback;
049 import gnu.classpath.tools.getopt.Option;
050 import gnu.classpath.tools.getopt.OptionException;
051 import gnu.classpath.tools.getopt.Parser;
052 import java.util.ArrayList;
053 import java.util.Collection;
054 import java.util.Enumeration;
055 import java.util.Iterator;
056 import java.util.LinkedList;
057 import java.util.regex.PatternSyntaxException;
058 import junit.framework.Test;
059 import junit.framework.TestResult;
060 import junit.framework.TestSuite;
061 import junit.textui.TestRunner;
062
063 /**
064 * <em>frysk</em> specific extension to the JUnit test framework.
065 */
066
067 public class Runner extends TestRunner {
068 private static final Log fine = Log.fine(Runner.class);
069
070 // Repeat once by default.
071 private int repeatValue = 1;
072 private Collection testCases = null;
073 private boolean listClassesOnly = false;
074 // Put all tests through a filter; by default exclude all Stress.*
075 // classes.
076 private String testFilter = "^(|.*\\.)(?!Stress)[^\\.]*$";
077 private ArrayList excludeTests = new ArrayList();
078 private ArrayList includeTests = new ArrayList();
079
080 private LinkedList otherArgs;
081
082 public static void usage(String message, int exitVal) {
083 System.out.println (message);
084 System.exit (exitVal);
085 }
086
087 public void setTestCases(Collection testCases) {
088 this.testCases = testCases;
089 }
090
091 public Collection getTestCases() {
092 return this.testCases;
093 }
094
095 private int runCases(Collection testClasses) {
096 // Create the testsuite to be run, either as specified on the
097 // command line, or from the provided list of classes.
098
099 TestSuite testSuite = new TestSuite ();
100
101 // Construct the testsuite from the list of names.
102 for (Iterator i = otherArgs.iterator(); i.hasNext(); ) {
103 String arg = (String) i.next();
104
105 // A -N argument specifying a repeat count.
106 if (arg.matches("^-[0-9]*$")) {
107 repeatValue = - Integer.parseInt(arg);
108 fine.log(this, "repeat", repeatValue);
109 continue;
110 }
111
112 String testMethod;
113 String testClass;
114 if (arg.matches("^test.*\\(.*\\)$")) {
115 // testFoo(frysk.class)
116 String[] testTuple = arg.split("[\\(\\)]");
117 testMethod = testTuple[0];
118 testClass = testTuple[1];
119 } else if (arg.matches(".*\\.test[A-Z][^.]*")) {
120 // frysk.class.testMethod
121 int lidot = arg.lastIndexOf('.');
122 testClass = arg.substring(0, lidot);
123 testMethod = arg.substring(lidot + 1);
124 } else {
125 // frysk.class
126 testClass = arg;
127 testMethod = null;
128 }
129 fine.log(this, "testClass", testClass, "testMethod", testMethod);
130
131 for (Iterator c = testClasses.iterator(); c.hasNext (); ) {
132 Class testKlass = (Class) c.next ();
133 if (testMethod == null) {
134 // class only
135 if (testKlass.getName().startsWith(testClass)) {
136 testSuite.addTest(new TestSuite(testKlass));
137 fine.log(this, "adding", testKlass);
138 }
139 } else {
140 // class.name or name(class)
141 if (testKlass.getName().equals(testClass)) {
142 try {
143 // Probe the class to see if the method
144 // exists.
145 testKlass.getMethod(testMethod, null);
146 TestCase test = (TestCase)testKlass.newInstance();
147 test.setName(testMethod);
148 testSuite.addTest(test);
149 fine.log(this, "adding", testKlass);
150 } catch (NoSuchMethodException e) {
151 System.out.println("Couldn't find test method: " + testClass + "." + testMethod);
152 } catch (InstantiationException e) {
153 System.out.println("Couldn't instantiate class with name: "
154 + testClass);
155 } catch (IllegalAccessException e) {
156 System.out.println("Couldn't access class with name: "
157 + testClass);
158 }
159 }
160 }
161 }
162 }
163
164 if (otherArgs.size() == 0) {
165 for (Iterator i = testClasses.iterator (); i.hasNext (); ) {
166 Class testKlass = (Class) i.next ();
167 // Only include tests that gets by both filters.
168 if (testKlass.getName ().matches (testFilter)) {
169 boolean addit = true;
170 for (int j = 0; j < excludeTests.size(); j++) {
171 try {
172 if (testKlass.getName ()
173 .matches ((String)excludeTests.get (j))) {
174 addit = false;
175 break;
176 }
177 } catch (PatternSyntaxException p) {
178 System.out.println(p.getMessage());
179 }
180 }
181 if (!addit) {
182 for (int j = 0; j < includeTests.size(); j++) {
183 try {
184 if (testKlass.getName ()
185 .matches ((String)includeTests.get (j))) {
186 addit = true;
187 break;
188 }
189 } catch (PatternSyntaxException p) {
190 System.out.println(p.getMessage());
191 }
192 }
193 }
194 if (addit) {
195 testSuite.addTest (new TestSuite (testKlass));
196 } else {
197 System.out.println ("Omitting " + testKlass.getName());
198 }
199 }
200 }
201 }
202
203 if (listClassesOnly) {
204 for (Enumeration e = testSuite.tests (); e.hasMoreElements (); ) {
205 Test test = (Test) e.nextElement ();
206 System.out.println (test.toString ());
207 }
208 return SUCCESS_EXIT;
209 }
210
211 // Run the TestSuite <<repeat>> times.
212 try {
213 for (int i = 0; i < this.repeatValue; i++) {
214 TestResult testResult = doRun (testSuite);
215
216 if (!testResult.wasSuccessful()) {
217 System.out.println("Failed after run #" + i);
218 return FAILURE_EXIT;
219 }
220 }
221 } catch(Exception e) {
222 System.err.println(e.getMessage());
223 return EXCEPTION_EXIT;
224 }
225 return SUCCESS_EXIT;
226 }
227
228 /**
229 * Create and return the command line parser used by frysk's JUnit
230 * tests.
231 */
232 private Parser createCommandLineParser (String programName) {
233 Parser parser = new Parser(programName, FryskVersion.getVersion(),
234 true);
235
236 parser.add(new LogOption("debug", 'c'));
237
238 parser.add(new Option("unbreak", 'u', "Run broken tests") {
239 public void parsed (String arg) throws OptionException {
240 skipUnresolvedTests = false;
241 }
242 });
243
244 // Determine the number of times that the testsuite should be
245 // run.
246 parser.add (new Option ("repeat", 'r',
247 "Set the count of repeating the test.",
248 "<repeat-count>") {
249 public void parsed (String arg0) throws OptionException {
250 try {
251 repeatValue = Integer.parseInt (arg0);
252 } catch (NumberFormatException e) {
253 throw new OptionException ("Argument: " + arg0
254 + " was not a number");
255 }
256 }
257 });
258
259 parser.add (new Option ("arch",
260 ("On 64-bit systems,"
261 + " only use test programs with the"
262 + " specified word-size (32, 64, all)."
263 + " By default, both 32-bit and 64-bit"
264 + " test programs are used"),
265 "<arch>") {
266 public void parsed (String arg0) throws OptionException {
267 if (arg0.equals("32"))
268 Prefix.set(config32);
269 else if (arg0.equals("64")) {
270 if (Host.wordSize() != 64)
271 throw new OptionException("-arch requires 64-bit");
272 Prefix.set(config64);
273 } else if (arg0.equals("all"))
274 Prefix.set(configAll);
275 else
276 throw new OptionException( "Invalid arch value: "
277 + arg0);
278 }
279 });
280
281 parser.add (new Option ("list-classes-only", 'n',
282 "Do not run any tests, instead list the"
283 + " classes that would have been tested") {
284 public void parsed (String nullArgument)
285 throws OptionException
286 {
287 listClassesOnly = true;
288 }
289 });
290
291 parser.add (new Option ("stress",
292 "Run only stress tests "
293 + "(by default stress tests are excluded).") {
294 public void parsed (String nullArgument)
295 throws OptionException
296 {
297 testFilter = "^(|.*)Stress.*$";
298 }
299 });
300
301 parser.add (new Option ("all",
302 "Run all tests "
303 + "(by default stress tests are excluded).") {
304 public void parsed (String nullArgument)
305 throws OptionException
306 {
307 testFilter = "^.*$";
308 }
309 });
310
311 // Specify tests to omit.
312 parser.add (new Option ("exclude", 'e',
313 "Specify a test to exclude. Each passed"
314 + " option will be interpreted as the"
315 + " regex specification of a test to omit."
316 + " This option may be used multiple"
317 + " times.",
318 "<test-spec>") {
319 public void parsed (String arg0) {
320 excludeTests.add (arg0);
321 }
322 });
323
324 // Specify tests to include, overriding omit.
325 parser.add (new Option ("include", 'i',
326 "Specify a test to include, ovirriding an"
327 + " omit specification. Each passed"
328 + " option will be interpreted as the"
329 + " regex specification of a test to include."
330 + " This option may be used multiple"
331 + " times.",
332 "<test-spec>") {
333 public void parsed (String arg0) {
334 includeTests.add (arg0);
335 }
336 });
337
338 parser.add(new Option ("timeout",
339 "Specify a timeout (in seconds) to use for " +
340 "assertRunUntilStop", "<timeout>") {
341 public void parsed (String arg0) {
342 int timeout = Integer.parseInt(arg0);
343 TestCase.setTimeoutSeconds (timeout);
344 Expect.setDefaultTimeoutSeconds (timeout);
345 }
346 });
347
348 parser.setHeader ("Usage:"
349 + " [ --console <LOG=LEVEL> ]"
350 + " [ --log <LOG=LEVEL> ]"
351 + " [ -r <repeat-count> ]"
352 + " [ --arch <arch>]"
353 + " [ -n ]"
354 + " [ --stress ]"
355 + " [ --all ]"
356 + " [-o spec...]"
357 + " [-i spec...]"
358 + " [--timeout <timeout>]"
359 + " [--unbreak ]"
360 + " [ class ... ]");
361 return parser;
362 }
363
364 private static String programBasename;
365 /**
366 * Possible configurations.
367 */
368 private final Prefix configAll;
369 private final Prefix config32;
370 private final Prefix config64;
371
372 /**
373 * Return the TestRunner's true basename - it could be "funit" or
374 * it could be "TestRunner".
375 *
376 * XXX: Hack, shouldn't be using static storage for this. Should
377 * this go in frysk.Prefix?
378 */
379 public static String getProgramBasename ()
380 {
381 return programBasename;
382 }
383
384 /**
385 * Create a JUnit TestRunner, using command-line arguments args,
386 * and the supplied testClasses.
387 */
388 public Runner(String programBasename, String[] args,
389 Prefix configAll, Prefix config32, Prefix config64) {
390 // Override the print methods.
391 super(new Results(System.out));
392
393 Prefix.set(configAll); // default
394 this.configAll = configAll;
395 this.config32 = config32;
396 this.config64 = config64;
397
398 // Tell expect the default timeout.
399 Expect.setDefaultTimeoutSeconds (TestCase.getTimeoutSeconds ());
400
401 // Create the command line parser, and use it to parse all
402 // command line options.
403 Runner.programBasename = programBasename;
404 Parser parser = createCommandLineParser(programBasename);
405
406 otherArgs = new LinkedList();
407
408 parser.parse(args, new FileArgumentCallback() {
409 public void notifyFile(String arg) throws OptionException {
410 otherArgs.add(arg);
411 }
412 });
413
414 }
415
416 /**
417 * Merge two TestRunner results returning the most fatal.
418 */
419 private int worstResult (int lhs, int rhs) {
420 if (lhs == SUCCESS_EXIT) {
421 return rhs;
422 }
423 if (lhs == FAILURE_EXIT) {
424 if (rhs == SUCCESS_EXIT) {
425 return FAILURE_EXIT;
426 } else {
427 return rhs;
428 }
429 }
430 return EXCEPTION_EXIT;
431 }
432
433 /**
434 * Run the testcases carried by testClasses.
435 *
436 * @param testClasses
437 * @return int the value of exit.
438 */
439 public int runTestCases (Collection tests) {
440 int result = SUCCESS_EXIT;
441 result = worstResult(runCases(tests), result);
442 return result;
443 }
444
445 /**
446 * Should the known-to-be-broken tests run?
447 */
448 private static boolean skipUnresolvedTests = true;
449
450 /**
451 * A method that returns true, and reports "UNRESOLVED". Used by
452 * test cases that want to be skipped (vis: if (broken()) return)
453 * while trying to avoid the compiler's optimizer realizing that
454 * the rest of the function is dead.
455 */
456 static boolean unresolved(int bug, boolean unresolved) {
457 String msg = "http://sourceware.org/bugzilla/show_bug.cgi?id=" + bug;
458 return unresolved(msg, unresolved);
459 }
460
461 /**
462 * A method that returns true, and reports "UNRESOLVED". Used by
463 * test cases that want to be skipped (vis: if (broken()) return)
464 * while trying to avoid the compiler's optimizer realizing that
465 * the rest of the function is dead.
466 */
467 static boolean unresolved(String bug, boolean unresolved) {
468 if (skipUnresolvedTests) {
469 if (unresolved) {
470 Results.addUnresolved(bug);
471 }
472 } else {
473 Results.addResolved(bug);
474 }
475 return skipUnresolvedTests && unresolved;
476 }
477
478 /**
479 * An unsupported feature; can't test.
480 */
481 static boolean unsupported(String reason, boolean unsupported)
482 {
483 if (unsupported) {
484 Results.addUnsupported(reason);
485 }
486 // XXX: For moment do not enable unsupported tests when -u was
487 // specified as it seems to cause cascading problems.
488 return unsupported;
489 }
490 }